Update model.py

model.py

@@ -36,19 +36,19 @@ except ImportError:
 @dataclass
 class SLMConfig:
     # Vocab / embedding
-    vocab_size:        int   = 32000
-    hidden_size:       int   = 576
+    vocab_size:        int   = 4096
+    hidden_size:       int   = 256
     tie_embeddings:    bool  = True
 
     # Depth / width
-    num_layers:        int   = 12
-    intermediate_size: int   = 1536
+    num_layers:        int   = 11
+    intermediate_size: int   = 768
 
     # Attention
-    num_heads:         int   = 9
-    num_kv_heads:      int   = 3
-    max_seq_len:       int   = 2048
-    rope_theta:        float = 500000.0
+    num_heads:         int   = 4
+    num_kv_heads:      int   = 2
+    max_seq_len:       int   = 1024
+    rope_theta:        float = 10000.0
     qk_norm:           bool  = True
     use_bias:          bool  = False