Upload configuration_nexus_small.py with huggingface_hub

configuration_nexus_small.py

@@ -1,4 +1,6 @@
 from dataclasses import dataclass
+from transformers import PretrainedConfig
+
 
 @dataclass
 class NexusConfig:
@@ -32,4 +34,34 @@ class NexusConfig:
     
     seed: int = 42
 
-nexus_config = NexusConfig()
+nexus_config = NexusConfig()
+
+
+class NexusSmAllConfig(PretrainedConfig):
+    model_type = "nexus_small"
+
+    def __init__(
+        self,
+        vocab_size=50304,
+        max_seq_len=512,
+        dim=768,
+        num_layers=10,
+        num_heads=12,
+        num_kv_heads=4,
+        multiple_of=256,
+        ff_dim=2048,
+        norm_eps=1e-6,
+        rope_theta=500000.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.max_seq_len = max_seq_len
+        self.dim = dim
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.num_kv_heads = num_kv_heads
+        self.multiple_of = multiple_of
+        self.ff_dim = ff_dim
+        self.norm_eps = norm_eps
+        self.rope_theta = rope_theta