Outer Step 1. Inner Step 2385. Batch Size 104

config.json

@@ -268,23 +268,23 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5184006,
-    5184010,
-    5184014,
-    5184019,
-    5184023,
-    5184028,
-    5184032,
-    5184037,
-    5184041,
-    5184045
+    5184079,
+    5184083,
+    5184088,
+    5184093,
+    5184098,
+    5184103,
+    5184108,
+    5184112,
+    5184117,
+    5184121
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 2375,
+  "inner_step": 2385,
   "inner_steps": 0,
   "last_allreduce_block": 4016775,
   "layer_norm_epsilon": 1e-05,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03b2bd3be474503293ec33d495c9ecde9df7563db2992cb62c4c33d0ac2f2253
+oid sha256:11803b97e28902285512a575ba6d2f5571429168f3c3f5760fdc9133a08a99cb
 size 8081781770

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2a6bf14a14d37effaea94bef3ffdf8869f882ab5cd7a530cf676393c1ed3474
+oid sha256:3d72984d131e8371305b3753c59285c814e253c7a056e3953292309b3edae2da
 size 4040701744