Upload model (depth) to HuggingFace Hub.

checkpoint checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/added_tokens.json

@@ -0,0 +1,23 @@
+{
+  "<eosen>": 32120,
+  "<sent_0>": 32100,
+  "<sent_10>": 32110,
+  "<sent_11>": 32111,
+  "<sent_12>": 32112,
+  "<sent_13>": 32113,
+  "<sent_14>": 32114,
+  "<sent_15>": 32115,
+  "<sent_16>": 32116,
+  "<sent_17>": 32117,
+  "<sent_18>": 32118,
+  "<sent_19>": 32119,
+  "<sent_1>": 32101,
+  "<sent_2>": 32102,
+  "<sent_3>": 32103,
+  "<sent_4>": 32104,
+  "<sent_5>": 32105,
+  "<sent_6>": 32106,
+  "<sent_7>": 32107,
+  "<sent_8>": 32108,
+  "<sent_9>": 32109
+}

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "google/t5-v1_1-base",
+  "architectures": [
+    "DepthForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.0,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_bf16": true,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 32128
+}

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.39.3"
+}

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/global_step32000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04023b143fa0e230378c6b93f169f463317e89d565bf35fe95b8faf31a1741d1
+size 742743344

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/global_step32000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b116074a2b0070c6396d496473e00b4e880c0d9b95529a5857c50d2b6fb58ae5
+size 742745648

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/global_step32000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c319b5a6cc609d7c9fb22dc6af671833c257c29d857fc02826fa9ee87915866
+size 742746032

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/global_step32000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eb9ab9b7d0e61b7b85168ccbc5478b82513f275f73819eab34439436d5ed334
+size 742743728

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/global_step32000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c53b1ccb0213a888dc69c8e46a662de9a175fc8f218b4977eb3b2346bd6caa2
+size 495255352

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/latest

@@ -0,0 +1 @@
+global_step32000

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:969c2828c573e11627ffc20e8ec161523a4deece18d7ba740a269fb24627aefb
+size 593886976

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:292fb0ad574b1a3a8916a46a83343976fc3bfce51bc3ebe5a5f37c68901aa3f1
+size 14960

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b60a06c853511d8533d43d90b453f8bab65fb69b88c4857a6b108bf6d0dd2b1f
+size 14960

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:565a85418f89f1c21bd80d9e1b44664b6bd4a93b1b13a7b1abed3960db1b28e3
+size 14960

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c15219e878685684ea566a3dd012361286a347b1313e162192bbdd73b0dfd092
+size 14960

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53f5c933052ccdd970779ae792f976bf2831a3a92b2b4ce5dfe471c11c436501
+size 1064

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/special_tokens_map.json

@@ -0,0 +1,872 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<extra_id_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<extra_id_99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sent_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<eosen>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/tokenizer_config.json

@@ -0,0 +1,1130 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<extra_id_99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<extra_id_98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<extra_id_97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<extra_id_96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<extra_id_95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<extra_id_94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<extra_id_93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<extra_id_92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<extra_id_91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<extra_id_90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<extra_id_89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32011": {
+      "content": "<extra_id_88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32012": {
+      "content": "<extra_id_87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32013": {
+      "content": "<extra_id_86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32014": {
+      "content": "<extra_id_85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32015": {
+      "content": "<extra_id_84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32016": {
+      "content": "<extra_id_83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32017": {
+      "content": "<extra_id_82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32018": {
+      "content": "<extra_id_81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32019": {
+      "content": "<extra_id_80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32020": {
+      "content": "<extra_id_79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32021": {
+      "content": "<extra_id_78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32022": {
+      "content": "<extra_id_77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32023": {
+      "content": "<extra_id_76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32024": {
+      "content": "<extra_id_75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32025": {
+      "content": "<extra_id_74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32026": {
+      "content": "<extra_id_73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32027": {
+      "content": "<extra_id_72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32028": {
+      "content": "<extra_id_71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32029": {
+      "content": "<extra_id_70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32030": {
+      "content": "<extra_id_69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32031": {
+      "content": "<extra_id_68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32032": {
+      "content": "<extra_id_67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32033": {
+      "content": "<extra_id_66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32034": {
+      "content": "<extra_id_65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32035": {
+      "content": "<extra_id_64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32036": {
+      "content": "<extra_id_63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32037": {
+      "content": "<extra_id_62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32038": {
+      "content": "<extra_id_61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32039": {
+      "content": "<extra_id_60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32040": {
+      "content": "<extra_id_59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32041": {
+      "content": "<extra_id_58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32042": {
+      "content": "<extra_id_57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32043": {
+      "content": "<extra_id_56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32044": {
+      "content": "<extra_id_55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32045": {
+      "content": "<extra_id_54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32046": {
+      "content": "<extra_id_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32047": {
+      "content": "<extra_id_52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32048": {
+      "content": "<extra_id_51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32049": {
+      "content": "<extra_id_50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32050": {
+      "content": "<extra_id_49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32051": {
+      "content": "<extra_id_48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32052": {
+      "content": "<extra_id_47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32053": {
+      "content": "<extra_id_46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32054": {
+      "content": "<extra_id_45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32055": {
+      "content": "<extra_id_44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32056": {
+      "content": "<extra_id_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32057": {
+      "content": "<extra_id_42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32058": {
+      "content": "<extra_id_41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32059": {
+      "content": "<extra_id_40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32060": {
+      "content": "<extra_id_39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32061": {
+      "content": "<extra_id_38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32062": {
+      "content": "<extra_id_37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32063": {
+      "content": "<extra_id_36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32064": {
+      "content": "<extra_id_35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32065": {
+      "content": "<extra_id_34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32066": {
+      "content": "<extra_id_33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32067": {
+      "content": "<extra_id_32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32068": {
+      "content": "<extra_id_31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32069": {
+      "content": "<extra_id_30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32070": {
+      "content": "<extra_id_29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32071": {
+      "content": "<extra_id_28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32072": {
+      "content": "<extra_id_27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32073": {
+      "content": "<extra_id_26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32074": {
+      "content": "<extra_id_25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32075": {
+      "content": "<extra_id_24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32076": {
+      "content": "<extra_id_23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32077": {
+      "content": "<extra_id_22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32078": {
+      "content": "<extra_id_21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32079": {
+      "content": "<extra_id_20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32080": {
+      "content": "<extra_id_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32081": {
+      "content": "<extra_id_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32082": {
+      "content": "<extra_id_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32083": {
+      "content": "<extra_id_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32084": {
+      "content": "<extra_id_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32085": {
+      "content": "<extra_id_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32086": {
+      "content": "<extra_id_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32087": {
+      "content": "<extra_id_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32088": {
+      "content": "<extra_id_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32089": {
+      "content": "<extra_id_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32090": {
+      "content": "<extra_id_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32091": {
+      "content": "<extra_id_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32092": {
+      "content": "<extra_id_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32093": {
+      "content": "<extra_id_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32094": {
+      "content": "<extra_id_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32095": {
+      "content": "<extra_id_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32096": {
+      "content": "<extra_id_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32097": {
+      "content": "<extra_id_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32098": {
+      "content": "<extra_id_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32099": {
+      "content": "<extra_id_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32100": {
+      "content": "<sent_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32101": {
+      "content": "<sent_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32102": {
+      "content": "<sent_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32103": {
+      "content": "<sent_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32104": {
+      "content": "<sent_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32105": {
+      "content": "<sent_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32106": {
+      "content": "<sent_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32107": {
+      "content": "<sent_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32108": {
+      "content": "<sent_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32109": {
+      "content": "<sent_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32110": {
+      "content": "<sent_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32111": {
+      "content": "<sent_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32112": {
+      "content": "<sent_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32113": {
+      "content": "<sent_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32114": {
+      "content": "<sent_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32115": {
+      "content": "<sent_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32116": {
+      "content": "<sent_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32117": {
+      "content": "<sent_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32118": {
+      "content": "<sent_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32119": {
+      "content": "<sent_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32120": {
+      "content": "<eosen>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>",
+    "<sent_0>",
+    "<sent_1>",
+    "<sent_2>",
+    "<sent_3>",
+    "<sent_4>",
+    "<sent_5>",
+    "<sent_6>",
+    "<sent_7>",
+    "<sent_8>",
+    "<sent_9>",
+    "<sent_10>",
+    "<sent_11>",
+    "<sent_12>",
+    "<sent_13>",
+    "<sent_14>",
+    "<sent_15>",
+    "<sent_16>",
+    "<sent_17>",
+    "<sent_18>",
+    "<sent_19>",
+    "<eosen>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "legacy": true,
+  "model_max_length": 1000000000,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "DepthTokenizer",
+  "unk_token": "<unk>",
+  "use_fast": true
+}

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/trainer_state.json

@@ -0,0 +1,3197 @@
+{
+  "best_metric": 0.35371094942092896,
+  "best_model_checkpoint": "checkpoints/pre_train/from_pretrained/depth/allenai,c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000",
+  "epoch": 0.032,
+  "eval_steps": 1000,
+  "global_step": 32000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 117027.92446249742,
+      "learning_rate": 1e-08,
+      "loss": 48.4062,
+      "num_input_tokens_seen": 102400,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 98187.21383153716,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 48.4508,
+      "num_input_tokens_seen": 10240000,
+      "step": 100
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 55606.95898896109,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 47.7316,
+      "num_input_tokens_seen": 20480000,
+      "step": 200
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 50086.50245325581,
+      "learning_rate": 3e-06,
+      "loss": 46.0,
+      "num_input_tokens_seen": 30720000,
+      "step": 300
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 49593.95317979804,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 43.4838,
+      "num_input_tokens_seen": 40960000,
+      "step": 400
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 9189.89336173168,
+      "learning_rate": 5e-06,
+      "loss": 40.5959,
+      "num_input_tokens_seen": 51200000,
+      "step": 500
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 6370.043445691716,
+      "learning_rate": 6e-06,
+      "loss": 37.7531,
+      "num_input_tokens_seen": 61440000,
+      "step": 600
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 5259.458859996911,
+      "learning_rate": 7.000000000000001e-06,
+      "loss": 33.2895,
+      "num_input_tokens_seen": 71680000,
+      "step": 700
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 4966.0240635744,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 24.192,
+      "num_input_tokens_seen": 81920000,
+      "step": 800
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 608.0843318878558,
+      "learning_rate": 9e-06,
+      "loss": 11.0091,
+      "num_input_tokens_seen": 92160000,
+      "step": 900
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 323.02566171281194,
+      "learning_rate": 1e-05,
+      "loss": 4.5075,
+      "num_input_tokens_seen": 102400000,
+      "step": 1000
+    },
+    {
+      "epoch": 0.0,
+      "eval_average_loss_on_non_sentence_tokens": 3.735890426314327,
+      "eval_average_loss_on_sentence_tokens": 6.916936108694429,
+      "eval_average_shuffling_prob": 0.525,
+      "eval_loss": 3.879453182220459,
+      "eval_non_padding_tokens_in_labels": 133.5083,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.39165,
+      "eval_padding_tokens_in_labels": 378.4917,
+      "eval_reconstruction_accuracy": 0.7531481443581489,
+      "eval_runtime": 172.2869,
+      "eval_samples_per_second": 29.021,
+      "eval_sentence_accuracy": 0.0019918531412062377,
+      "eval_steps_per_second": 0.075,
+      "eval_variance_shuffling_prob": 0.24937499999999996,
+      "num_input_tokens_seen": 102400000,
+      "step": 1000
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 72.39298574285789,
+      "learning_rate": 1.1000000000000001e-05,
+      "loss": 2.8069,
+      "num_input_tokens_seen": 112640000,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 159.50111125690472,
+      "learning_rate": 1.2e-05,
+      "loss": 1.8009,
+      "num_input_tokens_seen": 122880000,
+      "step": 1200
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 68.30627491221296,
+      "learning_rate": 1.3000000000000001e-05,
+      "loss": 1.6564,
+      "num_input_tokens_seen": 133120000,
+      "step": 1300
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 35.400115354533206,
+      "learning_rate": 1.4000000000000001e-05,
+      "loss": 1.6037,
+      "num_input_tokens_seen": 143360000,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 10.18265947358931,
+      "learning_rate": 1.5e-05,
+      "loss": 1.5582,
+      "num_input_tokens_seen": 153600000,
+      "step": 1500
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 9.040887356645733,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 1.5222,
+      "num_input_tokens_seen": 163840000,
+      "step": 1600
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 5.339145796173222,
+      "learning_rate": 1.7000000000000003e-05,
+      "loss": 1.4793,
+      "num_input_tokens_seen": 174080000,
+      "step": 1700
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 2.342113304871639,
+      "learning_rate": 1.8e-05,
+      "loss": 1.4391,
+      "num_input_tokens_seen": 184320000,
+      "step": 1800
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.773095253340904,
+      "learning_rate": 1.9e-05,
+      "loss": 1.4118,
+      "num_input_tokens_seen": 194560000,
+      "step": 1900
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.5889313696050641,
+      "learning_rate": 2e-05,
+      "loss": 1.3452,
+      "num_input_tokens_seen": 204800000,
+      "step": 2000
+    },
+    {
+      "epoch": 0.0,
+      "eval_average_loss_on_non_sentence_tokens": 1.279003297678963,
+      "eval_average_loss_on_sentence_tokens": 2.3100904416252224,
+      "eval_average_shuffling_prob": 0.52,
+      "eval_loss": 1.325351595878601,
+      "eval_non_padding_tokens_in_labels": 133.54125,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3952,
+      "eval_padding_tokens_in_labels": 378.45875,
+      "eval_reconstruction_accuracy": 0.7939266267005077,
+      "eval_runtime": 158.3282,
+      "eval_samples_per_second": 31.58,
+      "eval_sentence_accuracy": 0.04539989592118722,
+      "eval_steps_per_second": 0.082,
+      "eval_variance_shuffling_prob": 0.2496,
+      "num_input_tokens_seen": 204800000,
+      "step": 2000
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.4533874803817849,
+      "learning_rate": 2.1e-05,
+      "loss": 1.3164,
+      "num_input_tokens_seen": 215040000,
+      "step": 2100
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.5023528114557592,
+      "learning_rate": 2.2000000000000003e-05,
+      "loss": 1.3003,
+      "num_input_tokens_seen": 225280000,
+      "step": 2200
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.7179893117125062,
+      "learning_rate": 2.3000000000000003e-05,
+      "loss": 1.2726,
+      "num_input_tokens_seen": 235520000,
+      "step": 2300
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.23368749714892442,
+      "learning_rate": 2.4e-05,
+      "loss": 1.2449,
+      "num_input_tokens_seen": 245760000,
+      "step": 2400
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.26528883095495337,
+      "learning_rate": 2.5e-05,
+      "loss": 1.2168,
+      "num_input_tokens_seen": 256000000,
+      "step": 2500
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.4369897591811637,
+      "learning_rate": 2.6000000000000002e-05,
+      "loss": 1.171,
+      "num_input_tokens_seen": 266240000,
+      "step": 2600
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.3964361903785269,
+      "learning_rate": 2.7000000000000002e-05,
+      "loss": 1.1204,
+      "num_input_tokens_seen": 276480000,
+      "step": 2700
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.5518399607852702,
+      "learning_rate": 2.8000000000000003e-05,
+      "loss": 1.0646,
+      "num_input_tokens_seen": 286720000,
+      "step": 2800
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.5533210889041299,
+      "learning_rate": 2.9e-05,
+      "loss": 1.0273,
+      "num_input_tokens_seen": 296960000,
+      "step": 2900
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.4948192241590541,
+      "learning_rate": 3e-05,
+      "loss": 0.9912,
+      "num_input_tokens_seen": 307200000,
+      "step": 3000
+    },
+    {
+      "epoch": 0.0,
+      "eval_average_loss_on_non_sentence_tokens": 0.9183126312714729,
+      "eval_average_loss_on_sentence_tokens": 2.071033007189449,
+      "eval_average_shuffling_prob": 0.53,
+      "eval_loss": 0.9700586199760437,
+      "eval_non_padding_tokens_in_labels": 133.5311,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3945,
+      "eval_padding_tokens_in_labels": 378.4689,
+      "eval_reconstruction_accuracy": 0.8350565051675188,
+      "eval_runtime": 159.6707,
+      "eval_samples_per_second": 31.314,
+      "eval_sentence_accuracy": 0.047616056848565326,
+      "eval_steps_per_second": 0.081,
+      "eval_variance_shuffling_prob": 0.2490999999999999,
+      "num_input_tokens_seen": 307200000,
+      "step": 3000
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.5204243719766946,
+      "learning_rate": 3.1e-05,
+      "loss": 0.9669,
+      "num_input_tokens_seen": 317440000,
+      "step": 3100
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.8025626302801746,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 0.9417,
+      "num_input_tokens_seen": 327680000,
+      "step": 3200
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.6290059040323912,
+      "learning_rate": 3.3e-05,
+      "loss": 0.923,
+      "num_input_tokens_seen": 337920000,
+      "step": 3300
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.84089911930044,
+      "learning_rate": 3.4000000000000007e-05,
+      "loss": 0.9124,
+      "num_input_tokens_seen": 348160000,
+      "step": 3400
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.6866744351545562,
+      "learning_rate": 3.5e-05,
+      "loss": 0.8843,
+      "num_input_tokens_seen": 358400000,
+      "step": 3500
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.104960914671423,
+      "learning_rate": 3.6e-05,
+      "loss": 0.8719,
+      "num_input_tokens_seen": 368640000,
+      "step": 3600
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.6965317376806954,
+      "learning_rate": 3.7e-05,
+      "loss": 0.8381,
+      "num_input_tokens_seen": 378880000,
+      "step": 3700
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.4779527584566995,
+      "learning_rate": 3.8e-05,
+      "loss": 0.8079,
+      "num_input_tokens_seen": 389120000,
+      "step": 3800
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.5748217588916886,
+      "learning_rate": 3.9000000000000006e-05,
+      "loss": 0.7808,
+      "num_input_tokens_seen": 399360000,
+      "step": 3900
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.7680719160790532,
+      "learning_rate": 4e-05,
+      "loss": 0.7668,
+      "num_input_tokens_seen": 409600000,
+      "step": 4000
+    },
+    {
+      "epoch": 0.0,
+      "eval_average_loss_on_non_sentence_tokens": 0.7154172288004359,
+      "eval_average_loss_on_sentence_tokens": 1.6378286597585627,
+      "eval_average_shuffling_prob": 0.515,
+      "eval_loss": 0.757031261920929,
+      "eval_non_padding_tokens_in_labels": 133.50705,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.38365,
+      "eval_padding_tokens_in_labels": 378.49295,
+      "eval_reconstruction_accuracy": 0.8574579187521558,
+      "eval_runtime": 159.6769,
+      "eval_samples_per_second": 31.313,
+      "eval_sentence_accuracy": 0.06032982216878712,
+      "eval_steps_per_second": 0.081,
+      "eval_variance_shuffling_prob": 0.249775,
+      "num_input_tokens_seen": 409600000,
+      "step": 4000
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.7187184554931695,
+      "learning_rate": 4.1e-05,
+      "loss": 0.7605,
+      "num_input_tokens_seen": 419840000,
+      "step": 4100
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.3479579302403603,
+      "learning_rate": 4.2e-05,
+      "loss": 0.742,
+      "num_input_tokens_seen": 430080000,
+      "step": 4200
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.5810996991843084,
+      "learning_rate": 4.3e-05,
+      "loss": 0.733,
+      "num_input_tokens_seen": 440320000,
+      "step": 4300
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.42764991205025665,
+      "learning_rate": 4.4000000000000006e-05,
+      "loss": 0.7189,
+      "num_input_tokens_seen": 450560000,
+      "step": 4400
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.36334677074604205,
+      "learning_rate": 4.5e-05,
+      "loss": 0.7045,
+      "num_input_tokens_seen": 460800000,
+      "step": 4500
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.326550478120764,
+      "learning_rate": 4.600000000000001e-05,
+      "loss": 0.6995,
+      "num_input_tokens_seen": 471040000,
+      "step": 4600
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.3517289297561798,
+      "learning_rate": 4.7e-05,
+      "loss": 0.6818,
+      "num_input_tokens_seen": 481280000,
+      "step": 4700
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.4094795639507133,
+      "learning_rate": 4.8e-05,
+      "loss": 0.6712,
+      "num_input_tokens_seen": 491520000,
+      "step": 4800
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.3860976693769952,
+      "learning_rate": 4.9e-05,
+      "loss": 0.6608,
+      "num_input_tokens_seen": 501760000,
+      "step": 4900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.3933630540632164,
+      "learning_rate": 5e-05,
+      "loss": 0.6505,
+      "num_input_tokens_seen": 512000000,
+      "step": 5000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.6011862853229497,
+      "eval_average_loss_on_sentence_tokens": 1.5440813649998053,
+      "eval_average_shuffling_prob": 0.48,
+      "eval_loss": 0.6436914205551147,
+      "eval_non_padding_tokens_in_labels": 133.50815,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3919,
+      "eval_padding_tokens_in_labels": 378.49185,
+      "eval_reconstruction_accuracy": 0.8786358199055293,
+      "eval_runtime": 159.8392,
+      "eval_samples_per_second": 31.281,
+      "eval_sentence_accuracy": 0.07920756545301201,
+      "eval_steps_per_second": 0.081,
+      "eval_variance_shuffling_prob": 0.2496,
+      "num_input_tokens_seen": 512000000,
+      "step": 5000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.35213323995132506,
+      "learning_rate": 5.1000000000000006e-05,
+      "loss": 0.6432,
+      "num_input_tokens_seen": 522240000,
+      "step": 5100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.34561932081416386,
+      "learning_rate": 5.2000000000000004e-05,
+      "loss": 0.6402,
+      "num_input_tokens_seen": 532480000,
+      "step": 5200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.2883404722609347,
+      "learning_rate": 5.300000000000001e-05,
+      "loss": 0.6321,
+      "num_input_tokens_seen": 542720000,
+      "step": 5300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.416875358314017,
+      "learning_rate": 5.4000000000000005e-05,
+      "loss": 0.6242,
+      "num_input_tokens_seen": 552960000,
+      "step": 5400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.4017204252435866,
+      "learning_rate": 5.500000000000001e-05,
+      "loss": 0.6143,
+      "num_input_tokens_seen": 563200000,
+      "step": 5500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.4867774006513271,
+      "learning_rate": 5.6000000000000006e-05,
+      "loss": 0.6114,
+      "num_input_tokens_seen": 573440000,
+      "step": 5600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.24656542353836802,
+      "learning_rate": 5.6999999999999996e-05,
+      "loss": 0.6014,
+      "num_input_tokens_seen": 583680000,
+      "step": 5700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.24945153856235952,
+      "learning_rate": 5.8e-05,
+      "loss": 0.6003,
+      "num_input_tokens_seen": 593920000,
+      "step": 5800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.32042545559953334,
+      "learning_rate": 5.9e-05,
+      "loss": 0.5956,
+      "num_input_tokens_seen": 604160000,
+      "step": 5900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.20149177282537045,
+      "learning_rate": 6e-05,
+      "loss": 0.5882,
+      "num_input_tokens_seen": 614400000,
+      "step": 6000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.5351263822580051,
+      "eval_average_loss_on_sentence_tokens": 1.5069590978345084,
+      "eval_average_shuffling_prob": 0.495,
+      "eval_loss": 0.5788671970367432,
+      "eval_non_padding_tokens_in_labels": 133.53385,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.38385,
+      "eval_padding_tokens_in_labels": 378.46615,
+      "eval_reconstruction_accuracy": 0.889190764406181,
+      "eval_runtime": 159.1864,
+      "eval_samples_per_second": 31.41,
+      "eval_sentence_accuracy": 0.08739928580400882,
+      "eval_steps_per_second": 0.082,
+      "eval_variance_shuffling_prob": 0.24997499999999995,
+      "num_input_tokens_seen": 614400000,
+      "step": 6000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.49233812721792247,
+      "learning_rate": 6.1e-05,
+      "loss": 0.5747,
+      "num_input_tokens_seen": 624640000,
+      "step": 6100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.8520129526042493,
+      "learning_rate": 6.2e-05,
+      "loss": 0.5693,
+      "num_input_tokens_seen": 634880000,
+      "step": 6200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.363399404364346,
+      "learning_rate": 6.3e-05,
+      "loss": 0.5569,
+      "num_input_tokens_seen": 645120000,
+      "step": 6300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.4110389203574224,
+      "learning_rate": 6.400000000000001e-05,
+      "loss": 0.5385,
+      "num_input_tokens_seen": 655360000,
+      "step": 6400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.48985495455641515,
+      "learning_rate": 6.500000000000001e-05,
+      "loss": 0.5313,
+      "num_input_tokens_seen": 665600000,
+      "step": 6500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.6040708463144516,
+      "learning_rate": 6.6e-05,
+      "loss": 0.5147,
+      "num_input_tokens_seen": 675840000,
+      "step": 6600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.6612252828597202,
+      "learning_rate": 6.7e-05,
+      "loss": 0.5058,
+      "num_input_tokens_seen": 686080000,
+      "step": 6700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.42919798624080896,
+      "learning_rate": 6.800000000000001e-05,
+      "loss": 0.4994,
+      "num_input_tokens_seen": 696320000,
+      "step": 6800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.21242321394189362,
+      "learning_rate": 6.9e-05,
+      "loss": 0.4902,
+      "num_input_tokens_seen": 706560000,
+      "step": 6900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.44660943216480237,
+      "learning_rate": 7e-05,
+      "loss": 0.4895,
+      "num_input_tokens_seen": 716800000,
+      "step": 7000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.4361637280436824,
+      "eval_average_loss_on_sentence_tokens": 1.4670178340323676,
+      "eval_average_shuffling_prob": 0.445,
+      "eval_loss": 0.4828124940395355,
+      "eval_non_padding_tokens_in_labels": 133.5463,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3933,
+      "eval_padding_tokens_in_labels": 378.4537,
+      "eval_reconstruction_accuracy": 0.9147457630502843,
+      "eval_runtime": 160.0763,
+      "eval_samples_per_second": 31.235,
+      "eval_sentence_accuracy": 0.10527213020618371,
+      "eval_steps_per_second": 0.081,
+      "eval_variance_shuffling_prob": 0.24697499999999992,
+      "num_input_tokens_seen": 716800000,
+      "step": 7000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5076886025918936,
+      "learning_rate": 7.1e-05,
+      "loss": 0.4845,
+      "num_input_tokens_seen": 727040000,
+      "step": 7100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.3608466110317592,
+      "learning_rate": 7.2e-05,
+      "loss": 0.4801,
+      "num_input_tokens_seen": 737280000,
+      "step": 7200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.19827263227182243,
+      "learning_rate": 7.3e-05,
+      "loss": 0.4757,
+      "num_input_tokens_seen": 747520000,
+      "step": 7300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.31662446315149845,
+      "learning_rate": 7.4e-05,
+      "loss": 0.474,
+      "num_input_tokens_seen": 757760000,
+      "step": 7400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5578082637465724,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.4688,
+      "num_input_tokens_seen": 768000000,
+      "step": 7500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.6317062952444751,
+      "learning_rate": 7.6e-05,
+      "loss": 0.4668,
+      "num_input_tokens_seen": 778240000,
+      "step": 7600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.2758965748218114,
+      "learning_rate": 7.7e-05,
+      "loss": 0.4642,
+      "num_input_tokens_seen": 788480000,
+      "step": 7700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.3153156363678135,
+      "learning_rate": 7.800000000000001e-05,
+      "loss": 0.4646,
+      "num_input_tokens_seen": 798720000,
+      "step": 7800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.2456978118091552,
+      "learning_rate": 7.900000000000001e-05,
+      "loss": 0.4636,
+      "num_input_tokens_seen": 808960000,
+      "step": 7900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5619706205956216,
+      "learning_rate": 8e-05,
+      "loss": 0.4603,
+      "num_input_tokens_seen": 819200000,
+      "step": 8000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.409034057849692,
+      "eval_average_loss_on_sentence_tokens": 1.4305329217038258,
+      "eval_average_shuffling_prob": 0.495,
+      "eval_loss": 0.45506834983825684,
+      "eval_non_padding_tokens_in_labels": 133.52385,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3843,
+      "eval_padding_tokens_in_labels": 378.47615,
+      "eval_reconstruction_accuracy": 0.9193214830199541,
+      "eval_runtime": 160.2918,
+      "eval_samples_per_second": 31.193,
+      "eval_sentence_accuracy": 0.1088251655391462,
+      "eval_steps_per_second": 0.081,
+      "eval_variance_shuffling_prob": 0.24997499999999995,
+      "num_input_tokens_seen": 819200000,
+      "step": 8000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5302097429832601,
+      "learning_rate": 8.1e-05,
+      "loss": 0.459,
+      "num_input_tokens_seen": 829440000,
+      "step": 8100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.23994803615503887,
+      "learning_rate": 8.2e-05,
+      "loss": 0.456,
+      "num_input_tokens_seen": 839680000,
+      "step": 8200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5010205671529753,
+      "learning_rate": 8.3e-05,
+      "loss": 0.4542,
+      "num_input_tokens_seen": 849920000,
+      "step": 8300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.8509608251741911,
+      "learning_rate": 8.4e-05,
+      "loss": 0.4531,
+      "num_input_tokens_seen": 860160000,
+      "step": 8400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.4627523899902831,
+      "learning_rate": 8.5e-05,
+      "loss": 0.4514,
+      "num_input_tokens_seen": 870400000,
+      "step": 8500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.25954602216126105,
+      "learning_rate": 8.6e-05,
+      "loss": 0.4477,
+      "num_input_tokens_seen": 880640000,
+      "step": 8600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.35809980002546543,
+      "learning_rate": 8.7e-05,
+      "loss": 0.4497,
+      "num_input_tokens_seen": 890880000,
+      "step": 8700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5127755165944277,
+      "learning_rate": 8.800000000000001e-05,
+      "loss": 0.4492,
+      "num_input_tokens_seen": 901120000,
+      "step": 8800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.183436392719888,
+      "learning_rate": 8.900000000000001e-05,
+      "loss": 0.4443,
+      "num_input_tokens_seen": 911360000,
+      "step": 8900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.4957735800668283,
+      "learning_rate": 9e-05,
+      "loss": 0.4412,
+      "num_input_tokens_seen": 921600000,
+      "step": 9000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.39479284220034744,
+      "eval_average_loss_on_sentence_tokens": 1.20901018469223,
+      "eval_average_shuffling_prob": 0.505,
+      "eval_loss": 0.43144530057907104,
+      "eval_non_padding_tokens_in_labels": 133.5015,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.382,
+      "eval_padding_tokens_in_labels": 378.4985,
+      "eval_reconstruction_accuracy": 0.920657961678137,
+      "eval_runtime": 160.5519,
+      "eval_samples_per_second": 31.143,
+      "eval_sentence_accuracy": 0.2915283435318607,
+      "eval_steps_per_second": 0.081,
+      "eval_variance_shuffling_prob": 0.24997499999999995,
+      "num_input_tokens_seen": 921600000,
+      "step": 9000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.25962383301178327,
+      "learning_rate": 9.1e-05,
+      "loss": 0.4351,
+      "num_input_tokens_seen": 931840000,
+      "step": 9100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5270242782977608,
+      "learning_rate": 9.200000000000001e-05,
+      "loss": 0.4213,
+      "num_input_tokens_seen": 942080000,
+      "step": 9200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.35660468940981066,
+      "learning_rate": 9.300000000000001e-05,
+      "loss": 0.4109,
+      "num_input_tokens_seen": 952320000,
+      "step": 9300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.581907323150522,
+      "learning_rate": 9.4e-05,
+      "loss": 0.4076,
+      "num_input_tokens_seen": 962560000,
+      "step": 9400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.4161059500910824,
+      "learning_rate": 9.5e-05,
+      "loss": 0.4062,
+      "num_input_tokens_seen": 972800000,
+      "step": 9500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.21968142363703586,
+      "learning_rate": 9.6e-05,
+      "loss": 0.4096,
+      "num_input_tokens_seen": 983040000,
+      "step": 9600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.1782771876270999,
+      "learning_rate": 9.7e-05,
+      "loss": 0.4029,
+      "num_input_tokens_seen": 993280000,
+      "step": 9700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.20054026866622782,
+      "learning_rate": 9.8e-05,
+      "loss": 0.4011,
+      "num_input_tokens_seen": 1003520000,
+      "step": 9800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.5120583710342788,
+      "learning_rate": 9.900000000000001e-05,
+      "loss": 0.3995,
+      "num_input_tokens_seen": 1013760000,
+      "step": 9900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.31092720613740565,
+      "learning_rate": 0.0001,
+      "loss": 0.4017,
+      "num_input_tokens_seen": 1024000000,
+      "step": 10000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.3854121313597723,
+      "eval_average_loss_on_sentence_tokens": 0.5901794197149325,
+      "eval_average_shuffling_prob": 0.48,
+      "eval_loss": 0.3946484327316284,
+      "eval_non_padding_tokens_in_labels": 133.52655,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.37465,
+      "eval_padding_tokens_in_labels": 378.47345,
+      "eval_reconstruction_accuracy": 0.9222960739914927,
+      "eval_runtime": 162.0886,
+      "eval_samples_per_second": 30.847,
+      "eval_sentence_accuracy": 0.6509636262493944,
+      "eval_steps_per_second": 0.08,
+      "eval_variance_shuffling_prob": 0.2496,
+      "num_input_tokens_seen": 1024000000,
+      "step": 10000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.24302657196773422,
+      "learning_rate": 9.9989898989899e-05,
+      "loss": 0.3991,
+      "num_input_tokens_seen": 1034240000,
+      "step": 10100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.27559064135196976,
+      "learning_rate": 9.997979797979799e-05,
+      "loss": 0.3987,
+      "num_input_tokens_seen": 1044480000,
+      "step": 10200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.28406574154044006,
+      "learning_rate": 9.996969696969698e-05,
+      "loss": 0.3934,
+      "num_input_tokens_seen": 1054720000,
+      "step": 10300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.1485544264944323,
+      "learning_rate": 9.995959595959596e-05,
+      "loss": 0.3955,
+      "num_input_tokens_seen": 1064960000,
+      "step": 10400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.16967436423488208,
+      "learning_rate": 9.994949494949496e-05,
+      "loss": 0.3966,
+      "num_input_tokens_seen": 1075200000,
+      "step": 10500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.24514603540038796,
+      "learning_rate": 9.993939393939394e-05,
+      "loss": 0.3965,
+      "num_input_tokens_seen": 1085440000,
+      "step": 10600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.27024939136630677,
+      "learning_rate": 9.992929292929294e-05,
+      "loss": 0.392,
+      "num_input_tokens_seen": 1095680000,
+      "step": 10700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.1418517401981391,
+      "learning_rate": 9.991919191919193e-05,
+      "loss": 0.3929,
+      "num_input_tokens_seen": 1105920000,
+      "step": 10800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.14761725092349237,
+      "learning_rate": 9.990909090909092e-05,
+      "loss": 0.3905,
+      "num_input_tokens_seen": 1116160000,
+      "step": 10900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.18238314762766905,
+      "learning_rate": 9.98989898989899e-05,
+      "loss": 0.391,
+      "num_input_tokens_seen": 1126400000,
+      "step": 11000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.3800238027200128,
+      "eval_average_loss_on_sentence_tokens": 0.49753954927434607,
+      "eval_average_shuffling_prob": 0.46,
+      "eval_loss": 0.3853222727775574,
+      "eval_non_padding_tokens_in_labels": 133.54435,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.38005,
+      "eval_padding_tokens_in_labels": 378.45565,
+      "eval_reconstruction_accuracy": 0.9232105626133866,
+      "eval_runtime": 154.8497,
+      "eval_samples_per_second": 32.289,
+      "eval_sentence_accuracy": 0.6823532578462863,
+      "eval_steps_per_second": 0.084,
+      "eval_variance_shuffling_prob": 0.24839999999999995,
+      "num_input_tokens_seen": 1126400000,
+      "step": 11000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.17781057663046584,
+      "learning_rate": 9.98888888888889e-05,
+      "loss": 0.3881,
+      "num_input_tokens_seen": 1136640000,
+      "step": 11100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.2376840878722443,
+      "learning_rate": 9.987878787878788e-05,
+      "loss": 0.3877,
+      "num_input_tokens_seen": 1146880000,
+      "step": 11200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.21930933440270067,
+      "learning_rate": 9.986868686868687e-05,
+      "loss": 0.4112,
+      "num_input_tokens_seen": 1157120000,
+      "step": 11300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.18168312006356727,
+      "learning_rate": 9.985858585858587e-05,
+      "loss": 0.3898,
+      "num_input_tokens_seen": 1167360000,
+      "step": 11400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.22021381606308368,
+      "learning_rate": 9.984848484848486e-05,
+      "loss": 0.389,
+      "num_input_tokens_seen": 1177600000,
+      "step": 11500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.17201460240652908,
+      "learning_rate": 9.983838383838384e-05,
+      "loss": 0.3882,
+      "num_input_tokens_seen": 1187840000,
+      "step": 11600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.16659289859635742,
+      "learning_rate": 9.982828282828284e-05,
+      "loss": 0.3876,
+      "num_input_tokens_seen": 1198080000,
+      "step": 11700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.17389107753651245,
+      "learning_rate": 9.981818181818182e-05,
+      "loss": 0.3826,
+      "num_input_tokens_seen": 1208320000,
+      "step": 11800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.17306721160945812,
+      "learning_rate": 9.980808080808081e-05,
+      "loss": 0.3852,
+      "num_input_tokens_seen": 1218560000,
+      "step": 11900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.15314627142025644,
+      "learning_rate": 9.97979797979798e-05,
+      "loss": 0.3851,
+      "num_input_tokens_seen": 1228800000,
+      "step": 12000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.37589579345651064,
+      "eval_average_loss_on_sentence_tokens": 0.46593042849939187,
+      "eval_average_shuffling_prob": 0.465,
+      "eval_loss": 0.37996092438697815,
+      "eval_non_padding_tokens_in_labels": 133.5443,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.37965,
+      "eval_padding_tokens_in_labels": 378.4557,
+      "eval_reconstruction_accuracy": 0.9237285910487355,
+      "eval_runtime": 154.7147,
+      "eval_samples_per_second": 32.318,
+      "eval_sentence_accuracy": 0.6995845819800097,
+      "eval_steps_per_second": 0.084,
+      "eval_variance_shuffling_prob": 0.248775,
+      "num_input_tokens_seen": 1228800000,
+      "step": 12000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.2591230504491204,
+      "learning_rate": 9.97878787878788e-05,
+      "loss": 0.385,
+      "num_input_tokens_seen": 1239040000,
+      "step": 12100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.18060719909697132,
+      "learning_rate": 9.977777777777779e-05,
+      "loss": 0.3852,
+      "num_input_tokens_seen": 1249280000,
+      "step": 12200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.13639603743169598,
+      "learning_rate": 9.976767676767678e-05,
+      "loss": 0.3839,
+      "num_input_tokens_seen": 1259520000,
+      "step": 12300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.23837905971298684,
+      "learning_rate": 9.975757575757576e-05,
+      "loss": 0.3851,
+      "num_input_tokens_seen": 1269760000,
+      "step": 12400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.15068487388081905,
+      "learning_rate": 9.974747474747475e-05,
+      "loss": 0.384,
+      "num_input_tokens_seen": 1280000000,
+      "step": 12500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.22705913241215203,
+      "learning_rate": 9.973737373737374e-05,
+      "loss": 0.3793,
+      "num_input_tokens_seen": 1290240000,
+      "step": 12600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.18650120707548257,
+      "learning_rate": 9.972727272727273e-05,
+      "loss": 0.3832,
+      "num_input_tokens_seen": 1300480000,
+      "step": 12700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.22771822134343706,
+      "learning_rate": 9.971717171717173e-05,
+      "loss": 0.3791,
+      "num_input_tokens_seen": 1310720000,
+      "step": 12800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.3151917283377611,
+      "learning_rate": 9.970707070707072e-05,
+      "loss": 0.3925,
+      "num_input_tokens_seen": 1320960000,
+      "step": 12900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.1499099046123697,
+      "learning_rate": 9.96969696969697e-05,
+      "loss": 0.3834,
+      "num_input_tokens_seen": 1331200000,
+      "step": 13000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.3719757643866865,
+      "eval_average_loss_on_sentence_tokens": 0.5306737159842966,
+      "eval_average_shuffling_prob": 0.54,
+      "eval_loss": 0.37910157442092896,
+      "eval_non_padding_tokens_in_labels": 133.46615,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3731,
+      "eval_padding_tokens_in_labels": 378.53385,
+      "eval_reconstruction_accuracy": 0.9241808904556105,
+      "eval_runtime": 169.2783,
+      "eval_samples_per_second": 29.537,
+      "eval_sentence_accuracy": 0.6539065444039693,
+      "eval_steps_per_second": 0.077,
+      "eval_variance_shuffling_prob": 0.2483999999999999,
+      "num_input_tokens_seen": 1331200000,
+      "step": 13000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.20315309935395925,
+      "learning_rate": 9.968686868686869e-05,
+      "loss": 0.3822,
+      "num_input_tokens_seen": 1341440000,
+      "step": 13100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.15078756042478625,
+      "learning_rate": 9.967676767676768e-05,
+      "loss": 0.379,
+      "num_input_tokens_seen": 1351680000,
+      "step": 13200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.14908412582543276,
+      "learning_rate": 9.966666666666667e-05,
+      "loss": 0.3805,
+      "num_input_tokens_seen": 1361920000,
+      "step": 13300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.19913608575034622,
+      "learning_rate": 9.965656565656566e-05,
+      "loss": 0.3822,
+      "num_input_tokens_seen": 1372160000,
+      "step": 13400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.2315431822506722,
+      "learning_rate": 9.964646464646466e-05,
+      "loss": 0.3785,
+      "num_input_tokens_seen": 1382400000,
+      "step": 13500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.16156626978724312,
+      "learning_rate": 9.963636363636363e-05,
+      "loss": 0.3761,
+      "num_input_tokens_seen": 1392640000,
+      "step": 13600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.11581804472711785,
+      "learning_rate": 9.962626262626264e-05,
+      "loss": 0.3757,
+      "num_input_tokens_seen": 1402880000,
+      "step": 13700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.15559662467345098,
+      "learning_rate": 9.961616161616162e-05,
+      "loss": 0.3785,
+      "num_input_tokens_seen": 1413120000,
+      "step": 13800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.2567611230009723,
+      "learning_rate": 9.960606060606061e-05,
+      "loss": 0.3756,
+      "num_input_tokens_seen": 1423360000,
+      "step": 13900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.13475141070218874,
+      "learning_rate": 9.95959595959596e-05,
+      "loss": 0.3743,
+      "num_input_tokens_seen": 1433600000,
+      "step": 14000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.3689672788563913,
+      "eval_average_loss_on_sentence_tokens": 0.44648890796985713,
+      "eval_average_shuffling_prob": 0.46,
+      "eval_loss": 0.3724609315395355,
+      "eval_non_padding_tokens_in_labels": 133.4928,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3707,
+      "eval_padding_tokens_in_labels": 378.5072,
+      "eval_reconstruction_accuracy": 0.9244660977726237,
+      "eval_runtime": 157.1527,
+      "eval_samples_per_second": 31.816,
+      "eval_sentence_accuracy": 0.7164929029016455,
+      "eval_steps_per_second": 0.083,
+      "eval_variance_shuffling_prob": 0.2484,
+      "num_input_tokens_seen": 1433600000,
+      "step": 14000
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.1473000355949265,
+      "learning_rate": 9.95858585858586e-05,
+      "loss": 0.375,
+      "num_input_tokens_seen": 1443840000,
+      "step": 14100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.23647135736851135,
+      "learning_rate": 9.957575757575757e-05,
+      "loss": 0.3787,
+      "num_input_tokens_seen": 1454080000,
+      "step": 14200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.11819577382708915,
+      "learning_rate": 9.956565656565658e-05,
+      "loss": 0.3766,
+      "num_input_tokens_seen": 1464320000,
+      "step": 14300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.17736090732544665,
+      "learning_rate": 9.955555555555556e-05,
+      "loss": 0.3729,
+      "num_input_tokens_seen": 1474560000,
+      "step": 14400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.1456958896154653,
+      "learning_rate": 9.954545454545455e-05,
+      "loss": 0.3777,
+      "num_input_tokens_seen": 1484800000,
+      "step": 14500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.14435721080607852,
+      "learning_rate": 9.953535353535354e-05,
+      "loss": 0.3801,
+      "num_input_tokens_seen": 1495040000,
+      "step": 14600
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.16245784419041476,
+      "learning_rate": 9.952525252525253e-05,
+      "loss": 0.3799,
+      "num_input_tokens_seen": 1505280000,
+      "step": 14700
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.15716643694271826,
+      "learning_rate": 9.951515151515151e-05,
+      "loss": 0.3742,
+      "num_input_tokens_seen": 1515520000,
+      "step": 14800
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.12310718957919906,
+      "learning_rate": 9.950505050505052e-05,
+      "loss": 0.3753,
+      "num_input_tokens_seen": 1525760000,
+      "step": 14900
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 0.14458194694097115,
+      "learning_rate": 9.94949494949495e-05,
+      "loss": 0.377,
+      "num_input_tokens_seen": 1536000000,
+      "step": 15000
+    },
+    {
+      "epoch": 0.01,
+      "eval_average_loss_on_non_sentence_tokens": 0.36717536831858294,
+      "eval_average_loss_on_sentence_tokens": 0.4820282370375661,
+      "eval_average_shuffling_prob": 0.53,
+      "eval_loss": 0.3723437488079071,
+      "eval_non_padding_tokens_in_labels": 133.54825,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.38605,
+      "eval_padding_tokens_in_labels": 378.45175,
+      "eval_reconstruction_accuracy": 0.9248219286504614,
+      "eval_runtime": 160.4988,
+      "eval_samples_per_second": 31.153,
+      "eval_sentence_accuracy": 0.6813887343657473,
+      "eval_steps_per_second": 0.081,
+      "eval_variance_shuffling_prob": 0.2490999999999999,
+      "num_input_tokens_seen": 1536000000,
+      "step": 15000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.2644652087776747,
+      "learning_rate": 9.948484848484849e-05,
+      "loss": 0.3761,
+      "num_input_tokens_seen": 1546240000,
+      "step": 15100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.13911153713651134,
+      "learning_rate": 9.947474747474748e-05,
+      "loss": 0.3755,
+      "num_input_tokens_seen": 1556480000,
+      "step": 15200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1586721496636001,
+      "learning_rate": 9.946464646464647e-05,
+      "loss": 0.3752,
+      "num_input_tokens_seen": 1566720000,
+      "step": 15300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09363102565853981,
+      "learning_rate": 9.945454545454545e-05,
+      "loss": 0.3745,
+      "num_input_tokens_seen": 1576960000,
+      "step": 15400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14030144808834033,
+      "learning_rate": 9.944444444444446e-05,
+      "loss": 0.3735,
+      "num_input_tokens_seen": 1587200000,
+      "step": 15500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12466339895415234,
+      "learning_rate": 9.943434343434343e-05,
+      "loss": 0.3737,
+      "num_input_tokens_seen": 1597440000,
+      "step": 15600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1544825728133745,
+      "learning_rate": 9.942424242424243e-05,
+      "loss": 0.3748,
+      "num_input_tokens_seen": 1607680000,
+      "step": 15700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11560570766941806,
+      "learning_rate": 9.941414141414142e-05,
+      "loss": 0.369,
+      "num_input_tokens_seen": 1617920000,
+      "step": 15800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.16851998686948277,
+      "learning_rate": 9.940404040404041e-05,
+      "loss": 0.3718,
+      "num_input_tokens_seen": 1628160000,
+      "step": 15900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1013062067671787,
+      "learning_rate": 9.939393939393939e-05,
+      "loss": 0.3739,
+      "num_input_tokens_seen": 1638400000,
+      "step": 16000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.3650758731006156,
+      "eval_average_loss_on_sentence_tokens": 0.44875857155687715,
+      "eval_average_shuffling_prob": 0.48,
+      "eval_loss": 0.36884766817092896,
+      "eval_non_padding_tokens_in_labels": 133.5264,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3892,
+      "eval_padding_tokens_in_labels": 378.4736,
+      "eval_reconstruction_accuracy": 0.9250055008312146,
+      "eval_runtime": 156.6637,
+      "eval_samples_per_second": 31.915,
+      "eval_sentence_accuracy": 0.7146760098336533,
+      "eval_steps_per_second": 0.083,
+      "eval_variance_shuffling_prob": 0.2496,
+      "num_input_tokens_seen": 1638400000,
+      "step": 16000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.7206209544244442,
+      "learning_rate": 9.93838383838384e-05,
+      "loss": 0.3741,
+      "num_input_tokens_seen": 1648640000,
+      "step": 16100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12656907348574992,
+      "learning_rate": 9.937373737373737e-05,
+      "loss": 0.3724,
+      "num_input_tokens_seen": 1658880000,
+      "step": 16200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12169840807974559,
+      "learning_rate": 9.936363636363636e-05,
+      "loss": 0.3716,
+      "num_input_tokens_seen": 1669120000,
+      "step": 16300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14111603004919837,
+      "learning_rate": 9.935353535353536e-05,
+      "loss": 0.3739,
+      "num_input_tokens_seen": 1679360000,
+      "step": 16400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11221000785299119,
+      "learning_rate": 9.934343434343435e-05,
+      "loss": 0.3693,
+      "num_input_tokens_seen": 1689600000,
+      "step": 16500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08991521475715944,
+      "learning_rate": 9.933333333333334e-05,
+      "loss": 0.3698,
+      "num_input_tokens_seen": 1699840000,
+      "step": 16600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14364552381185702,
+      "learning_rate": 9.932323232323233e-05,
+      "loss": 0.3692,
+      "num_input_tokens_seen": 1710080000,
+      "step": 16700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12436604860875718,
+      "learning_rate": 9.931313131313131e-05,
+      "loss": 0.3703,
+      "num_input_tokens_seen": 1720320000,
+      "step": 16800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.2905394479490077,
+      "learning_rate": 9.93030303030303e-05,
+      "loss": 0.3751,
+      "num_input_tokens_seen": 1730560000,
+      "step": 16900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11363640761171379,
+      "learning_rate": 9.92929292929293e-05,
+      "loss": 0.3722,
+      "num_input_tokens_seen": 1740800000,
+      "step": 17000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.3628784865518768,
+      "eval_average_loss_on_sentence_tokens": 0.43488322014603975,
+      "eval_average_shuffling_prob": 0.485,
+      "eval_loss": 0.36610352993011475,
+      "eval_non_padding_tokens_in_labels": 133.52035,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3739,
+      "eval_padding_tokens_in_labels": 378.47965,
+      "eval_reconstruction_accuracy": 0.9254036716381047,
+      "eval_runtime": 155.4863,
+      "eval_samples_per_second": 32.157,
+      "eval_sentence_accuracy": 0.7182066143879986,
+      "eval_steps_per_second": 0.084,
+      "eval_variance_shuffling_prob": 0.24977499999999994,
+      "num_input_tokens_seen": 1740800000,
+      "step": 17000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10632912830829595,
+      "learning_rate": 9.928282828282829e-05,
+      "loss": 0.3665,
+      "num_input_tokens_seen": 1751040000,
+      "step": 17100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.16322306333159942,
+      "learning_rate": 9.927272727272728e-05,
+      "loss": 0.3702,
+      "num_input_tokens_seen": 1761280000,
+      "step": 17200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12100661511431879,
+      "learning_rate": 9.926262626262627e-05,
+      "loss": 0.3687,
+      "num_input_tokens_seen": 1771520000,
+      "step": 17300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.2946207264746856,
+      "learning_rate": 9.925252525252525e-05,
+      "loss": 0.3662,
+      "num_input_tokens_seen": 1781760000,
+      "step": 17400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.15342596690055987,
+      "learning_rate": 9.924242424242425e-05,
+      "loss": 0.3711,
+      "num_input_tokens_seen": 1792000000,
+      "step": 17500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12659288243319242,
+      "learning_rate": 9.923232323232323e-05,
+      "loss": 0.3686,
+      "num_input_tokens_seen": 1802240000,
+      "step": 17600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09637991471513421,
+      "learning_rate": 9.922222222222222e-05,
+      "loss": 0.367,
+      "num_input_tokens_seen": 1812480000,
+      "step": 17700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11752431666397788,
+      "learning_rate": 9.921212121212122e-05,
+      "loss": 0.3674,
+      "num_input_tokens_seen": 1822720000,
+      "step": 17800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.16606818096549658,
+      "learning_rate": 9.920202020202021e-05,
+      "loss": 0.3709,
+      "num_input_tokens_seen": 1832960000,
+      "step": 17900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.20761453361286383,
+      "learning_rate": 9.919191919191919e-05,
+      "loss": 0.3674,
+      "num_input_tokens_seen": 1843200000,
+      "step": 18000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.36206278400693837,
+      "eval_average_loss_on_sentence_tokens": 0.3958933896889189,
+      "eval_average_shuffling_prob": 0.46,
+      "eval_loss": 0.36356446146965027,
+      "eval_non_padding_tokens_in_labels": 133.5544,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.37155,
+      "eval_padding_tokens_in_labels": 378.4456,
+      "eval_reconstruction_accuracy": 0.9256239212958267,
+      "eval_runtime": 156.4083,
+      "eval_samples_per_second": 31.968,
+      "eval_sentence_accuracy": 0.7412385378721266,
+      "eval_steps_per_second": 0.083,
+      "eval_variance_shuffling_prob": 0.24839999999999995,
+      "num_input_tokens_seen": 1843200000,
+      "step": 18000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1083439440834149,
+      "learning_rate": 9.918181818181819e-05,
+      "loss": 0.3662,
+      "num_input_tokens_seen": 1853440000,
+      "step": 18100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.25975168520589126,
+      "learning_rate": 9.917171717171717e-05,
+      "loss": 0.3718,
+      "num_input_tokens_seen": 1863680000,
+      "step": 18200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.15694849825372573,
+      "learning_rate": 9.916161616161616e-05,
+      "loss": 0.367,
+      "num_input_tokens_seen": 1873920000,
+      "step": 18300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.19287524399939732,
+      "learning_rate": 9.915151515151515e-05,
+      "loss": 0.3657,
+      "num_input_tokens_seen": 1884160000,
+      "step": 18400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14395940263956455,
+      "learning_rate": 9.914141414141415e-05,
+      "loss": 0.367,
+      "num_input_tokens_seen": 1894400000,
+      "step": 18500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.19642979999644156,
+      "learning_rate": 9.913131313131314e-05,
+      "loss": 0.3697,
+      "num_input_tokens_seen": 1904640000,
+      "step": 18600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12190618543958091,
+      "learning_rate": 9.912121212121213e-05,
+      "loss": 0.3656,
+      "num_input_tokens_seen": 1914880000,
+      "step": 18700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11996760009180743,
+      "learning_rate": 9.911111111111112e-05,
+      "loss": 0.3623,
+      "num_input_tokens_seen": 1925120000,
+      "step": 18800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.16029858370289582,
+      "learning_rate": 9.91010101010101e-05,
+      "loss": 0.3667,
+      "num_input_tokens_seen": 1935360000,
+      "step": 18900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14008933937810145,
+      "learning_rate": 9.909090909090911e-05,
+      "loss": 0.3662,
+      "num_input_tokens_seen": 1945600000,
+      "step": 19000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.36037543821358486,
+      "eval_average_loss_on_sentence_tokens": 0.4375366560188371,
+      "eval_average_shuffling_prob": 0.49,
+      "eval_loss": 0.3639160096645355,
+      "eval_non_padding_tokens_in_labels": 133.5372,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.39125,
+      "eval_padding_tokens_in_labels": 378.4628,
+      "eval_reconstruction_accuracy": 0.9256722739034521,
+      "eval_runtime": 155.4586,
+      "eval_samples_per_second": 32.163,
+      "eval_sentence_accuracy": 0.7188346761892799,
+      "eval_steps_per_second": 0.084,
+      "eval_variance_shuffling_prob": 0.2499,
+      "num_input_tokens_seen": 1945600000,
+      "step": 19000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11805541190048278,
+      "learning_rate": 9.908080808080809e-05,
+      "loss": 0.3677,
+      "num_input_tokens_seen": 1955840000,
+      "step": 19100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11050390808702572,
+      "learning_rate": 9.907070707070708e-05,
+      "loss": 0.3697,
+      "num_input_tokens_seen": 1966080000,
+      "step": 19200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11662551316010913,
+      "learning_rate": 9.906060606060607e-05,
+      "loss": 0.3676,
+      "num_input_tokens_seen": 1976320000,
+      "step": 19300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1223301601413596,
+      "learning_rate": 9.905050505050506e-05,
+      "loss": 0.3691,
+      "num_input_tokens_seen": 1986560000,
+      "step": 19400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10207303274583229,
+      "learning_rate": 9.904040404040404e-05,
+      "loss": 0.3662,
+      "num_input_tokens_seen": 1996800000,
+      "step": 19500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08686027577031438,
+      "learning_rate": 9.903030303030305e-05,
+      "loss": 0.3668,
+      "num_input_tokens_seen": 2007040000,
+      "step": 19600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.19016857340805707,
+      "learning_rate": 9.902020202020202e-05,
+      "loss": 0.3655,
+      "num_input_tokens_seen": 2017280000,
+      "step": 19700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.16012871319435779,
+      "learning_rate": 9.901010101010102e-05,
+      "loss": 0.3639,
+      "num_input_tokens_seen": 2027520000,
+      "step": 19800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10128884632500922,
+      "learning_rate": 9.900000000000001e-05,
+      "loss": 0.3678,
+      "num_input_tokens_seen": 2037760000,
+      "step": 19900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.16727620505609542,
+      "learning_rate": 9.8989898989899e-05,
+      "loss": 0.3644,
+      "num_input_tokens_seen": 2048000000,
+      "step": 20000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.36006847868498015,
+      "eval_average_loss_on_sentence_tokens": 0.4308708401505413,
+      "eval_average_shuffling_prob": 0.5,
+      "eval_loss": 0.36324217915534973,
+      "eval_non_padding_tokens_in_labels": 133.51725,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.36155,
+      "eval_padding_tokens_in_labels": 378.48275,
+      "eval_reconstruction_accuracy": 0.9257211964914828,
+      "eval_runtime": 156.4197,
+      "eval_samples_per_second": 31.965,
+      "eval_sentence_accuracy": 0.719543488793583,
+      "eval_steps_per_second": 0.083,
+      "eval_variance_shuffling_prob": 0.25,
+      "num_input_tokens_seen": 2048000000,
+      "step": 20000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11343092320689546,
+      "learning_rate": 9.897979797979798e-05,
+      "loss": 0.3686,
+      "num_input_tokens_seen": 2058240000,
+      "step": 20100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10904926945300837,
+      "learning_rate": 9.896969696969698e-05,
+      "loss": 0.3666,
+      "num_input_tokens_seen": 2068480000,
+      "step": 20200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08824138717195555,
+      "learning_rate": 9.895959595959596e-05,
+      "loss": 0.3659,
+      "num_input_tokens_seen": 2078720000,
+      "step": 20300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10982444712618332,
+      "learning_rate": 9.894949494949495e-05,
+      "loss": 0.3664,
+      "num_input_tokens_seen": 2088960000,
+      "step": 20400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09810357164314008,
+      "learning_rate": 9.893939393939395e-05,
+      "loss": 0.3644,
+      "num_input_tokens_seen": 2099200000,
+      "step": 20500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.13117198825374618,
+      "learning_rate": 9.892929292929294e-05,
+      "loss": 0.3663,
+      "num_input_tokens_seen": 2109440000,
+      "step": 20600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.37286179130355385,
+      "learning_rate": 9.891919191919192e-05,
+      "loss": 0.3679,
+      "num_input_tokens_seen": 2119680000,
+      "step": 20700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10047485875458267,
+      "learning_rate": 9.890909090909092e-05,
+      "loss": 0.3665,
+      "num_input_tokens_seen": 2129920000,
+      "step": 20800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12476892085669669,
+      "learning_rate": 9.88989898989899e-05,
+      "loss": 0.3638,
+      "num_input_tokens_seen": 2140160000,
+      "step": 20900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12365876297594075,
+      "learning_rate": 9.888888888888889e-05,
+      "loss": 0.3623,
+      "num_input_tokens_seen": 2150400000,
+      "step": 21000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.3587830255621528,
+      "eval_average_loss_on_sentence_tokens": 0.3976552794947042,
+      "eval_average_shuffling_prob": 0.465,
+      "eval_loss": 0.3605078160762787,
+      "eval_non_padding_tokens_in_labels": 133.5387,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.38515,
+      "eval_padding_tokens_in_labels": 378.4613,
+      "eval_reconstruction_accuracy": 0.9260081703881333,
+      "eval_runtime": 154.3969,
+      "eval_samples_per_second": 32.384,
+      "eval_sentence_accuracy": 0.742862526244011,
+      "eval_steps_per_second": 0.084,
+      "eval_variance_shuffling_prob": 0.248775,
+      "num_input_tokens_seen": 2150400000,
+      "step": 21000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09637865488859665,
+      "learning_rate": 9.887878787878788e-05,
+      "loss": 0.366,
+      "num_input_tokens_seen": 2160640000,
+      "step": 21100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.0837396685633993,
+      "learning_rate": 9.886868686868688e-05,
+      "loss": 0.3635,
+      "num_input_tokens_seen": 2170880000,
+      "step": 21200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08748902918380133,
+      "learning_rate": 9.885858585858587e-05,
+      "loss": 0.3635,
+      "num_input_tokens_seen": 2181120000,
+      "step": 21300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08861983243508997,
+      "learning_rate": 9.884848484848486e-05,
+      "loss": 0.3654,
+      "num_input_tokens_seen": 2191360000,
+      "step": 21400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1095992454706993,
+      "learning_rate": 9.883838383838384e-05,
+      "loss": 0.3623,
+      "num_input_tokens_seen": 2201600000,
+      "step": 21500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1678159484473445,
+      "learning_rate": 9.882828282828283e-05,
+      "loss": 0.3645,
+      "num_input_tokens_seen": 2211840000,
+      "step": 21600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11648618534569517,
+      "learning_rate": 9.881818181818182e-05,
+      "loss": 0.3671,
+      "num_input_tokens_seen": 2222080000,
+      "step": 21700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10011749768739321,
+      "learning_rate": 9.880808080808081e-05,
+      "loss": 0.3678,
+      "num_input_tokens_seen": 2232320000,
+      "step": 21800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.13526673032373565,
+      "learning_rate": 9.87979797979798e-05,
+      "loss": 0.3643,
+      "num_input_tokens_seen": 2242560000,
+      "step": 21900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.13394247434184048,
+      "learning_rate": 9.87878787878788e-05,
+      "loss": 0.3634,
+      "num_input_tokens_seen": 2252800000,
+      "step": 22000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.3583377337473865,
+      "eval_average_loss_on_sentence_tokens": 0.40822070061419796,
+      "eval_average_shuffling_prob": 0.485,
+      "eval_loss": 0.36058592796325684,
+      "eval_non_padding_tokens_in_labels": 133.52525,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.376,
+      "eval_padding_tokens_in_labels": 378.47475,
+      "eval_reconstruction_accuracy": 0.9261606204151949,
+      "eval_runtime": 155.145,
+      "eval_samples_per_second": 32.228,
+      "eval_sentence_accuracy": 0.7347022089830783,
+      "eval_steps_per_second": 0.084,
+      "eval_variance_shuffling_prob": 0.2497749999999999,
+      "num_input_tokens_seen": 2252800000,
+      "step": 22000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10097576958217712,
+      "learning_rate": 9.877777777777778e-05,
+      "loss": 0.3616,
+      "num_input_tokens_seen": 2263040000,
+      "step": 22100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08778364505416811,
+      "learning_rate": 9.876767676767677e-05,
+      "loss": 0.3608,
+      "num_input_tokens_seen": 2273280000,
+      "step": 22200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11667769923972934,
+      "learning_rate": 9.875757575757576e-05,
+      "loss": 0.3631,
+      "num_input_tokens_seen": 2283520000,
+      "step": 22300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.17587799482940783,
+      "learning_rate": 9.874747474747475e-05,
+      "loss": 0.3641,
+      "num_input_tokens_seen": 2293760000,
+      "step": 22400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.18467841943373034,
+      "learning_rate": 9.873737373737374e-05,
+      "loss": 0.365,
+      "num_input_tokens_seen": 2304000000,
+      "step": 22500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.11327524004811648,
+      "learning_rate": 9.872727272727274e-05,
+      "loss": 0.3626,
+      "num_input_tokens_seen": 2314240000,
+      "step": 22600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.13776796365489793,
+      "learning_rate": 9.871717171717172e-05,
+      "loss": 0.3658,
+      "num_input_tokens_seen": 2324480000,
+      "step": 22700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1409646640902976,
+      "learning_rate": 9.870707070707072e-05,
+      "loss": 0.3613,
+      "num_input_tokens_seen": 2334720000,
+      "step": 22800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14753019525405045,
+      "learning_rate": 9.86969696969697e-05,
+      "loss": 0.3664,
+      "num_input_tokens_seen": 2344960000,
+      "step": 22900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1397460316378644,
+      "learning_rate": 9.868686868686869e-05,
+      "loss": 0.3637,
+      "num_input_tokens_seen": 2355200000,
+      "step": 23000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.3573655042544687,
+      "eval_average_loss_on_sentence_tokens": 0.42437406804439193,
+      "eval_average_shuffling_prob": 0.495,
+      "eval_loss": 0.3604394495487213,
+      "eval_non_padding_tokens_in_labels": 133.5656,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.385,
+      "eval_padding_tokens_in_labels": 378.4344,
+      "eval_reconstruction_accuracy": 0.9260999844059516,
+      "eval_runtime": 157.9229,
+      "eval_samples_per_second": 31.661,
+      "eval_sentence_accuracy": 0.7266361009923377,
+      "eval_steps_per_second": 0.082,
+      "eval_variance_shuffling_prob": 0.24997499999999995,
+      "num_input_tokens_seen": 2355200000,
+      "step": 23000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.0987903451138666,
+      "learning_rate": 9.867676767676768e-05,
+      "loss": 0.3622,
+      "num_input_tokens_seen": 2365440000,
+      "step": 23100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.16954454367678937,
+      "learning_rate": 9.866666666666668e-05,
+      "loss": 0.362,
+      "num_input_tokens_seen": 2375680000,
+      "step": 23200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09387896280569132,
+      "learning_rate": 9.865656565656565e-05,
+      "loss": 0.3634,
+      "num_input_tokens_seen": 2385920000,
+      "step": 23300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.07207999681262237,
+      "learning_rate": 9.864646464646466e-05,
+      "loss": 0.3626,
+      "num_input_tokens_seen": 2396160000,
+      "step": 23400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09396686653666976,
+      "learning_rate": 9.863636363636364e-05,
+      "loss": 0.3612,
+      "num_input_tokens_seen": 2406400000,
+      "step": 23500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.12398359266910461,
+      "learning_rate": 9.862626262626263e-05,
+      "loss": 0.3609,
+      "num_input_tokens_seen": 2416640000,
+      "step": 23600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14763177045615983,
+      "learning_rate": 9.861616161616162e-05,
+      "loss": 0.3643,
+      "num_input_tokens_seen": 2426880000,
+      "step": 23700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10030012588072933,
+      "learning_rate": 9.860606060606061e-05,
+      "loss": 0.3605,
+      "num_input_tokens_seen": 2437120000,
+      "step": 23800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.10084957691663816,
+      "learning_rate": 9.859595959595959e-05,
+      "loss": 0.3589,
+      "num_input_tokens_seen": 2447360000,
+      "step": 23900
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09548032136785647,
+      "learning_rate": 9.85858585858586e-05,
+      "loss": 0.3566,
+      "num_input_tokens_seen": 2457600000,
+      "step": 24000
+    },
+    {
+      "epoch": 0.02,
+      "eval_average_loss_on_non_sentence_tokens": 0.3553217848237189,
+      "eval_average_loss_on_sentence_tokens": 0.3849351337254851,
+      "eval_average_shuffling_prob": 0.46,
+      "eval_loss": 0.3566015660762787,
+      "eval_non_padding_tokens_in_labels": 133.4909,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.37935,
+      "eval_padding_tokens_in_labels": 378.5091,
+      "eval_reconstruction_accuracy": 0.9265581998113339,
+      "eval_runtime": 162.898,
+      "eval_samples_per_second": 30.694,
+      "eval_sentence_accuracy": 0.7520456870098875,
+      "eval_steps_per_second": 0.08,
+      "eval_variance_shuffling_prob": 0.24839999999999995,
+      "num_input_tokens_seen": 2457600000,
+      "step": 24000
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1188563096869887,
+      "learning_rate": 9.857575757575758e-05,
+      "loss": 0.3621,
+      "num_input_tokens_seen": 2467840000,
+      "step": 24100
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.14877812617022562,
+      "learning_rate": 9.856565656565657e-05,
+      "loss": 0.3604,
+      "num_input_tokens_seen": 2478080000,
+      "step": 24200
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08250986132874133,
+      "learning_rate": 9.855555555555556e-05,
+      "loss": 0.3609,
+      "num_input_tokens_seen": 2488320000,
+      "step": 24300
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09244225265273415,
+      "learning_rate": 9.854545454545455e-05,
+      "loss": 0.3588,
+      "num_input_tokens_seen": 2498560000,
+      "step": 24400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.1454294300819924,
+      "learning_rate": 9.853535353535353e-05,
+      "loss": 0.361,
+      "num_input_tokens_seen": 2508800000,
+      "step": 24500
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.20041363836805748,
+      "learning_rate": 9.852525252525254e-05,
+      "loss": 0.36,
+      "num_input_tokens_seen": 2519040000,
+      "step": 24600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.09722078986711696,
+      "learning_rate": 9.851515151515151e-05,
+      "loss": 0.3635,
+      "num_input_tokens_seen": 2529280000,
+      "step": 24700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08990032032163263,
+      "learning_rate": 9.85050505050505e-05,
+      "loss": 0.3631,
+      "num_input_tokens_seen": 2539520000,
+      "step": 24800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.08959055680739043,
+      "learning_rate": 9.84949494949495e-05,
+      "loss": 0.3609,
+      "num_input_tokens_seen": 2549760000,
+      "step": 24900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.11585233560862784,
+      "learning_rate": 9.848484848484849e-05,
+      "loss": 0.3623,
+      "num_input_tokens_seen": 2560000000,
+      "step": 25000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.3556082151245955,
+      "eval_average_loss_on_sentence_tokens": 0.43450918672873573,
+      "eval_average_shuffling_prob": 0.53,
+      "eval_loss": 0.3591601550579071,
+      "eval_non_padding_tokens_in_labels": 133.57675,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3766,
+      "eval_padding_tokens_in_labels": 378.42325,
+      "eval_reconstruction_accuracy": 0.9264658314428117,
+      "eval_runtime": 157.6612,
+      "eval_samples_per_second": 31.714,
+      "eval_sentence_accuracy": 0.7161609273781111,
+      "eval_steps_per_second": 0.082,
+      "eval_variance_shuffling_prob": 0.2490999999999999,
+      "num_input_tokens_seen": 2560000000,
+      "step": 25000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09943665461342704,
+      "learning_rate": 9.847474747474747e-05,
+      "loss": 0.3594,
+      "num_input_tokens_seen": 2570240000,
+      "step": 25100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.18746017986890912,
+      "learning_rate": 9.846464646464647e-05,
+      "loss": 0.3591,
+      "num_input_tokens_seen": 2580480000,
+      "step": 25200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08602902567018297,
+      "learning_rate": 9.845454545454545e-05,
+      "loss": 0.3577,
+      "num_input_tokens_seen": 2590720000,
+      "step": 25300
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08012464962844001,
+      "learning_rate": 9.844444444444444e-05,
+      "loss": 0.3609,
+      "num_input_tokens_seen": 2600960000,
+      "step": 25400
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.12572008831678216,
+      "learning_rate": 9.843434343434344e-05,
+      "loss": 0.364,
+      "num_input_tokens_seen": 2611200000,
+      "step": 25500
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.14940579397721981,
+      "learning_rate": 9.842424242424243e-05,
+      "loss": 0.3639,
+      "num_input_tokens_seen": 2621440000,
+      "step": 25600
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08412110210655141,
+      "learning_rate": 9.841414141414142e-05,
+      "loss": 0.3604,
+      "num_input_tokens_seen": 2631680000,
+      "step": 25700
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09362195877472666,
+      "learning_rate": 9.840404040404041e-05,
+      "loss": 0.3596,
+      "num_input_tokens_seen": 2641920000,
+      "step": 25800
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08569747148738543,
+      "learning_rate": 9.839393939393939e-05,
+      "loss": 0.3608,
+      "num_input_tokens_seen": 2652160000,
+      "step": 25900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09877899875115037,
+      "learning_rate": 9.838383838383838e-05,
+      "loss": 0.3591,
+      "num_input_tokens_seen": 2662400000,
+      "step": 26000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.3556134664693423,
+      "eval_average_loss_on_sentence_tokens": 0.42163200170538406,
+      "eval_average_shuffling_prob": 0.52,
+      "eval_loss": 0.3585839867591858,
+      "eval_non_padding_tokens_in_labels": 133.5339,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3887,
+      "eval_padding_tokens_in_labels": 378.4661,
+      "eval_reconstruction_accuracy": 0.9265349576723824,
+      "eval_runtime": 166.333,
+      "eval_samples_per_second": 30.06,
+      "eval_sentence_accuracy": 0.7202029536849283,
+      "eval_steps_per_second": 0.078,
+      "eval_variance_shuffling_prob": 0.2496,
+      "num_input_tokens_seen": 2662400000,
+      "step": 26000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08541063254934746,
+      "learning_rate": 9.837373737373737e-05,
+      "loss": 0.3583,
+      "num_input_tokens_seen": 2672640000,
+      "step": 26100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08526475341411524,
+      "learning_rate": 9.836363636363637e-05,
+      "loss": 0.3608,
+      "num_input_tokens_seen": 2682880000,
+      "step": 26200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.07892816724288247,
+      "learning_rate": 9.835353535353536e-05,
+      "loss": 0.3573,
+      "num_input_tokens_seen": 2693120000,
+      "step": 26300
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.1057304341337342,
+      "learning_rate": 9.834343434343435e-05,
+      "loss": 0.3595,
+      "num_input_tokens_seen": 2703360000,
+      "step": 26400
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08824599002870707,
+      "learning_rate": 9.833333333333333e-05,
+      "loss": 0.3603,
+      "num_input_tokens_seen": 2713600000,
+      "step": 26500
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.14259526724512908,
+      "learning_rate": 9.832323232323233e-05,
+      "loss": 0.3596,
+      "num_input_tokens_seen": 2723840000,
+      "step": 26600
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10958661741770682,
+      "learning_rate": 9.831313131313131e-05,
+      "loss": 0.3606,
+      "num_input_tokens_seen": 2734080000,
+      "step": 26700
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.1002029330532675,
+      "learning_rate": 9.83030303030303e-05,
+      "loss": 0.361,
+      "num_input_tokens_seen": 2744320000,
+      "step": 26800
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.11207220852080024,
+      "learning_rate": 9.82929292929293e-05,
+      "loss": 0.3592,
+      "num_input_tokens_seen": 2754560000,
+      "step": 26900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.15408787575922175,
+      "learning_rate": 9.828282828282829e-05,
+      "loss": 0.3591,
+      "num_input_tokens_seen": 2764800000,
+      "step": 27000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.35464164042436336,
+      "eval_average_loss_on_sentence_tokens": 0.45656515384165897,
+      "eval_average_shuffling_prob": 0.57,
+      "eval_loss": 0.3592187464237213,
+      "eval_non_padding_tokens_in_labels": 133.5122,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3798,
+      "eval_padding_tokens_in_labels": 378.4878,
+      "eval_reconstruction_accuracy": 0.9266176202210964,
+      "eval_runtime": 167.2988,
+      "eval_samples_per_second": 29.887,
+      "eval_sentence_accuracy": 0.6986873508353222,
+      "eval_steps_per_second": 0.078,
+      "eval_variance_shuffling_prob": 0.24509999999999996,
+      "num_input_tokens_seen": 2764800000,
+      "step": 27000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.19324864000629008,
+      "learning_rate": 9.827272727272728e-05,
+      "loss": 0.3613,
+      "num_input_tokens_seen": 2775040000,
+      "step": 27100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10080048909742759,
+      "learning_rate": 9.826262626262627e-05,
+      "loss": 0.3611,
+      "num_input_tokens_seen": 2785280000,
+      "step": 27200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10157552507215743,
+      "learning_rate": 9.825252525252527e-05,
+      "loss": 0.3582,
+      "num_input_tokens_seen": 2795520000,
+      "step": 27300
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.11988636067218172,
+      "learning_rate": 9.824242424242424e-05,
+      "loss": 0.3612,
+      "num_input_tokens_seen": 2805760000,
+      "step": 27400
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10519465177125238,
+      "learning_rate": 9.823232323232325e-05,
+      "loss": 0.3603,
+      "num_input_tokens_seen": 2816000000,
+      "step": 27500
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.13230602730567367,
+      "learning_rate": 9.822222222222223e-05,
+      "loss": 0.3581,
+      "num_input_tokens_seen": 2826240000,
+      "step": 27600
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08350293092452066,
+      "learning_rate": 9.821212121212122e-05,
+      "loss": 0.3624,
+      "num_input_tokens_seen": 2836480000,
+      "step": 27700
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10992582448277533,
+      "learning_rate": 9.820202020202021e-05,
+      "loss": 0.3573,
+      "num_input_tokens_seen": 2846720000,
+      "step": 27800
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.097079855298227,
+      "learning_rate": 9.81919191919192e-05,
+      "loss": 0.3569,
+      "num_input_tokens_seen": 2856960000,
+      "step": 27900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08641760728189113,
+      "learning_rate": 9.818181818181818e-05,
+      "loss": 0.3603,
+      "num_input_tokens_seen": 2867200000,
+      "step": 28000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.3541137710581413,
+      "eval_average_loss_on_sentence_tokens": 0.39053072234269565,
+      "eval_average_shuffling_prob": 0.48,
+      "eval_loss": 0.35579100251197815,
+      "eval_non_padding_tokens_in_labels": 133.52385,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3881,
+      "eval_padding_tokens_in_labels": 378.47615,
+      "eval_reconstruction_accuracy": 0.9268094160225488,
+      "eval_runtime": 171.34,
+      "eval_samples_per_second": 29.182,
+      "eval_sentence_accuracy": 0.7443564160999157,
+      "eval_steps_per_second": 0.076,
+      "eval_variance_shuffling_prob": 0.24960000000000004,
+      "num_input_tokens_seen": 2867200000,
+      "step": 28000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10167108746333979,
+      "learning_rate": 9.817171717171719e-05,
+      "loss": 0.3564,
+      "num_input_tokens_seen": 2877440000,
+      "step": 28100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08710525143703719,
+      "learning_rate": 9.816161616161617e-05,
+      "loss": 0.3572,
+      "num_input_tokens_seen": 2887680000,
+      "step": 28200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.0717770177483454,
+      "learning_rate": 9.815151515151516e-05,
+      "loss": 0.3582,
+      "num_input_tokens_seen": 2897920000,
+      "step": 28300
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09463541202404317,
+      "learning_rate": 9.814141414141415e-05,
+      "loss": 0.3568,
+      "num_input_tokens_seen": 2908160000,
+      "step": 28400
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.07740621416559264,
+      "learning_rate": 9.813131313131314e-05,
+      "loss": 0.3584,
+      "num_input_tokens_seen": 2918400000,
+      "step": 28500
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.18716247298945485,
+      "learning_rate": 9.812121212121212e-05,
+      "loss": 0.3592,
+      "num_input_tokens_seen": 2928640000,
+      "step": 28600
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.15144869547979808,
+      "learning_rate": 9.811111111111113e-05,
+      "loss": 0.3611,
+      "num_input_tokens_seen": 2938880000,
+      "step": 28700
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08190077384648202,
+      "learning_rate": 9.81010101010101e-05,
+      "loss": 0.3588,
+      "num_input_tokens_seen": 2949120000,
+      "step": 28800
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09443291148084239,
+      "learning_rate": 9.80909090909091e-05,
+      "loss": 0.3608,
+      "num_input_tokens_seen": 2959360000,
+      "step": 28900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.07502971721046336,
+      "learning_rate": 9.808080808080809e-05,
+      "loss": 0.356,
+      "num_input_tokens_seen": 2969600000,
+      "step": 29000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.35357406801707514,
+      "eval_average_loss_on_sentence_tokens": 0.3986508759030095,
+      "eval_average_shuffling_prob": 0.5,
+      "eval_loss": 0.35554686188697815,
+      "eval_non_padding_tokens_in_labels": 133.5861,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.40495,
+      "eval_padding_tokens_in_labels": 378.4139,
+      "eval_reconstruction_accuracy": 0.9268235948088003,
+      "eval_runtime": 261.4153,
+      "eval_samples_per_second": 19.127,
+      "eval_sentence_accuracy": 0.7389685430760673,
+      "eval_steps_per_second": 0.05,
+      "eval_variance_shuffling_prob": 0.25,
+      "num_input_tokens_seen": 2969600000,
+      "step": 29000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.12099518917998703,
+      "learning_rate": 9.807070707070708e-05,
+      "loss": 0.358,
+      "num_input_tokens_seen": 2979840000,
+      "step": 29100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.0940573888242316,
+      "learning_rate": 9.806060606060606e-05,
+      "loss": 0.3589,
+      "num_input_tokens_seen": 2990080000,
+      "step": 29200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.12086193137495242,
+      "learning_rate": 9.805050505050506e-05,
+      "loss": 0.3593,
+      "num_input_tokens_seen": 3000320000,
+      "step": 29300
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.0944554231200817,
+      "learning_rate": 9.804040404040404e-05,
+      "loss": 0.3602,
+      "num_input_tokens_seen": 3010560000,
+      "step": 29400
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.07325507209615767,
+      "learning_rate": 9.803030303030303e-05,
+      "loss": 0.3554,
+      "num_input_tokens_seen": 3020800000,
+      "step": 29500
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.11685171099918144,
+      "learning_rate": 9.802020202020203e-05,
+      "loss": 0.3562,
+      "num_input_tokens_seen": 3031040000,
+      "step": 29600
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09985223433551571,
+      "learning_rate": 9.801010101010102e-05,
+      "loss": 0.357,
+      "num_input_tokens_seen": 3041280000,
+      "step": 29700
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.17135952464793094,
+      "learning_rate": 9.8e-05,
+      "loss": 0.3586,
+      "num_input_tokens_seen": 3051520000,
+      "step": 29800
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10098197662404099,
+      "learning_rate": 9.7989898989899e-05,
+      "loss": 0.3567,
+      "num_input_tokens_seen": 3061760000,
+      "step": 29900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.15149886956103967,
+      "learning_rate": 9.797979797979798e-05,
+      "loss": 0.3591,
+      "num_input_tokens_seen": 3072000000,
+      "step": 30000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.35290258691268767,
+      "eval_average_loss_on_sentence_tokens": 0.39201884173473056,
+      "eval_average_shuffling_prob": 0.475,
+      "eval_loss": 0.3546484410762787,
+      "eval_non_padding_tokens_in_labels": 133.53105,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3804,
+      "eval_padding_tokens_in_labels": 378.46895,
+      "eval_reconstruction_accuracy": 0.9266994940320824,
+      "eval_runtime": 191.0999,
+      "eval_samples_per_second": 26.164,
+      "eval_sentence_accuracy": 0.7558723778419797,
+      "eval_steps_per_second": 0.068,
+      "eval_variance_shuffling_prob": 0.24937499999999996,
+      "num_input_tokens_seen": 3072000000,
+      "step": 30000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09305983056837887,
+      "learning_rate": 9.796969696969697e-05,
+      "loss": 0.3583,
+      "num_input_tokens_seen": 3082240000,
+      "step": 30100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08551824234236988,
+      "learning_rate": 9.795959595959596e-05,
+      "loss": 0.3592,
+      "num_input_tokens_seen": 3092480000,
+      "step": 30200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.0770125852850996,
+      "learning_rate": 9.794949494949496e-05,
+      "loss": 0.3567,
+      "num_input_tokens_seen": 3102720000,
+      "step": 30300
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09387334765692501,
+      "learning_rate": 9.793939393939394e-05,
+      "loss": 0.3587,
+      "num_input_tokens_seen": 3112960000,
+      "step": 30400
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08424991391139344,
+      "learning_rate": 9.792929292929294e-05,
+      "loss": 0.3588,
+      "num_input_tokens_seen": 3123200000,
+      "step": 30500
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.12246957334765836,
+      "learning_rate": 9.791919191919192e-05,
+      "loss": 0.3557,
+      "num_input_tokens_seen": 3133440000,
+      "step": 30600
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10614110629234996,
+      "learning_rate": 9.790909090909091e-05,
+      "loss": 0.3561,
+      "num_input_tokens_seen": 3143680000,
+      "step": 30700
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.13356568839284386,
+      "learning_rate": 9.78989898989899e-05,
+      "loss": 0.3574,
+      "num_input_tokens_seen": 3153920000,
+      "step": 30800
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.0895629230825024,
+      "learning_rate": 9.78888888888889e-05,
+      "loss": 0.3563,
+      "num_input_tokens_seen": 3164160000,
+      "step": 30900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.16700400204720783,
+      "learning_rate": 9.787878787878789e-05,
+      "loss": 0.3546,
+      "num_input_tokens_seen": 3174400000,
+      "step": 31000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.3518938701253859,
+      "eval_average_loss_on_sentence_tokens": 0.38815311963009613,
+      "eval_average_shuffling_prob": 0.48,
+      "eval_loss": 0.3535449206829071,
+      "eval_non_padding_tokens_in_labels": 133.55145,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3702,
+      "eval_padding_tokens_in_labels": 378.44855,
+      "eval_reconstruction_accuracy": 0.9271468005770392,
+      "eval_runtime": 197.7139,
+      "eval_samples_per_second": 25.289,
+      "eval_sentence_accuracy": 0.7477883252283454,
+      "eval_steps_per_second": 0.066,
+      "eval_variance_shuffling_prob": 0.2496,
+      "num_input_tokens_seen": 3174400000,
+      "step": 31000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.07575415582861136,
+      "learning_rate": 9.786868686868688e-05,
+      "loss": 0.3548,
+      "num_input_tokens_seen": 3184640000,
+      "step": 31100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.10933940810732458,
+      "learning_rate": 9.785858585858586e-05,
+      "loss": 0.3562,
+      "num_input_tokens_seen": 3194880000,
+      "step": 31200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.11331540242392588,
+      "learning_rate": 9.784848484848485e-05,
+      "loss": 0.3523,
+      "num_input_tokens_seen": 3205120000,
+      "step": 31300
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.11292417128466069,
+      "learning_rate": 9.783838383838384e-05,
+      "loss": 0.356,
+      "num_input_tokens_seen": 3215360000,
+      "step": 31400
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.08401367596771823,
+      "learning_rate": 9.782828282828283e-05,
+      "loss": 0.3545,
+      "num_input_tokens_seen": 3225600000,
+      "step": 31500
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09355949128705393,
+      "learning_rate": 9.781818181818183e-05,
+      "loss": 0.3555,
+      "num_input_tokens_seen": 3235840000,
+      "step": 31600
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.14463991996407455,
+      "learning_rate": 9.780808080808082e-05,
+      "loss": 0.3561,
+      "num_input_tokens_seen": 3246080000,
+      "step": 31700
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09184912325885994,
+      "learning_rate": 9.77979797979798e-05,
+      "loss": 0.3539,
+      "num_input_tokens_seen": 3256320000,
+      "step": 31800
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.0846744714701543,
+      "learning_rate": 9.77878787878788e-05,
+      "loss": 0.3576,
+      "num_input_tokens_seen": 3266560000,
+      "step": 31900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.09247798050269185,
+      "learning_rate": 9.777777777777778e-05,
+      "loss": 0.3542,
+      "num_input_tokens_seen": 3276800000,
+      "step": 32000
+    },
+    {
+      "epoch": 0.03,
+      "eval_average_loss_on_non_sentence_tokens": 0.35196479632527816,
+      "eval_average_loss_on_sentence_tokens": 0.3895615696675367,
+      "eval_average_shuffling_prob": 0.495,
+      "eval_loss": 0.35371094942092896,
+      "eval_non_padding_tokens_in_labels": 133.5149,
+      "eval_num_sentence_tokens": 23.1033,
+      "eval_num_sentinel_tokens_in_labels": 52.3711,
+      "eval_padding_tokens_in_labels": 378.4851,
+      "eval_reconstruction_accuracy": 0.9270111993418964,
+      "eval_runtime": 197.3653,
+      "eval_samples_per_second": 25.334,
+      "eval_sentence_accuracy": 0.7485016239883718,
+      "eval_steps_per_second": 0.066,
+      "eval_variance_shuffling_prob": 0.24997499999999995,
+      "num_input_tokens_seen": 3276800000,
+      "step": 32000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 1000000,
+  "num_input_tokens_seen": 3276800000,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 2000,
+  "total_flos": 4.382447189884928e+18,
+  "train_batch_size": 25,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d82b00925601fb06e8309afb6ae8395bc9272184c0df2dafa9f57b036de0f2da
+size 6584

checkpoints/pre_train/from_pretrained/depth/allenai_c4_en/lr_0_0001_linear_bsz_200_shuffle_p_0_5/2024-04-09_19-16/checkpoint-32000/zero_to_fp32.py

@@ -0,0 +1,592 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)