SparkBET-9M inference export at update 414181

bet_model.py

@@ -7,7 +7,7 @@ import torch.nn.functional as F
 from torch.utils.checkpoint import checkpoint
 
 SEQ_LEN = 1024
-MAX_LOOPS = 10
+MAX_LOOPS = 8
 PAD_ID, BOS_ID, EOS_ID = 256, 257, 258
 VOCAB_SIZE = 259
 EXPECTED_PARAM_COUNT = 9_353_876
@@ -451,7 +451,7 @@ def verify_architecture(model, rank=0):
         vocab_size=259, hidden_size=324, intermediate_size=864,
         prelude_layers=1, body_blocks=6, coda_layers=1,
         num_heads=6, num_kv_heads=2, head_dim=54,
-        lora_rank=16, hyper_lanes=2, max_seq_len=1024, max_loops=10,
+        lora_rank=16, hyper_lanes=2, max_seq_len=1024, max_loops=8,
         rope_theta=10_000.0, rms_eps=1e-6,
         ddl_beta_init=1.0, ddl_k_eps=1e-2, ddl_v_sigmoid_scale=4.0,
     )

config.json

@@ -3,7 +3,7 @@
   "architectures": [
     "BETForCausalLM"
   ],
-  "step": 364187,
+  "step": 414181,
   "vocab_size": 259,
   "hidden_size": 324,
   "intermediate_size": 864,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5910267d9f3129d8dd557232fcc7d5f0ea3f46af1aa64bdecad81cee4e63c110
+oid sha256:df04de2dd47fbccc32718553a085aec7db909389993f3c046e6c5ba064eb808a
 size 37429752