Outer Step 0. Inner Step 88. Batch Size 384

config.json

@@ -268,23 +268,23 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5131247,
-    5131254,
-    5131261,
-    5131269,
-    5131277,
-    5131285,
-    5131294,
-    5131306,
-    5131319,
-    5131328
+    5131362,
+    5131368,
+    5131375,
+    5131382,
+    5131389,
+    5131402,
+    5131409,
+    5131420,
+    5131426,
+    5131432
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 80,
+  "inner_step": 88,
   "inner_steps": 0,
   "last_allreduce_block": 5072405,
   "layer_norm_epsilon": 1e-05,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75ea758fe173aacee9cecea5b719495d83b549fc577e0ce4da6a3a82f3b0ce4f
+oid sha256:e5da09f92bd67224701923f3fcd1f56394aa44ef513b5195b251d04a446ef4fd
 size 8081781770

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d75adcbec29ce995dc58f74c9719705a130e71554bd08dcc3c023349853aaaf
+oid sha256:546d5af66c8d5e9c67ea238fcbe847e1f9b8774d71798dc8fc4a8f37e48817b7
 size 4040701744