User01110 commited on
Commit
a2c9b48
·
verified ·
1 Parent(s): 9d8725c

Benchmark-selected step 0: Open SLM average 30.66%

Browse files
README.md ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ pipeline_tag: text-generation
4
+ tags:
5
+ - custom_code
6
+ - causal-lm
7
+ - cma
8
+ - small-language-model
9
+ - generalist
10
+ - 4k-tokenizer
11
+ datasets:
12
+ - HuggingFaceFW/fineweb_edu_100BT-shuffled
13
+ - HuggingFaceTB/smollm-corpus
14
+ - epfml/FineWeb-HQ
15
+ - HuggingFaceTB/finemath
16
+ ---
17
+
18
+ # CMA 8M
19
+
20
+ Benchmark-selected checkpoint from a 7.85M-parameter
21
+ Channel-Mixing Attention generalist language model using the unmodified native
22
+ GPT-S 4,096-token tokenizer. It has no arithmetic-specific token splitting,
23
+ place embeddings, role embeddings, or inference-time equation detection. It
24
+ was selected at step 0 for an Open SLM Leaderboard-style average of
25
+ 30.66%. WikiText normalized BPB is reported separately
26
+ and is not used for selection.
27
+
28
+ ## Loading
29
+
30
+ This is a custom Transformers architecture. `trust_remote_code=True` is
31
+ required because stock Hugging Face model classes do not implement CMA or this
32
+ model's exact rotary convention.
33
+
34
+ ```python
35
+ from transformers import AutoModelForCausalLM, AutoTokenizer
36
+
37
+ repo = "User01110/cma-8M"
38
+ tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
39
+ model = AutoModelForCausalLM.from_pretrained(
40
+ repo, trust_remote_code=True, dtype="auto"
41
+ )
42
+ ```
43
+
44
+ ## Architecture
45
+
46
+ - Parameters: 7,849,161, with tied input/output embeddings
47
+ - Weights: safetensors only (`model.safetensors`); no PyTorch `.bin` weights
48
+ - Runtime: PyTorch 2.5+ for native SDPA grouped-query attention
49
+ - Tokenizer: AxiomicLabs/GPT-S-5M at revision df47402
50
+ - Vocabulary: 4,096 native tokens
51
+ - Parameter allocation: 1,179,648 tied embedding parameters and
52
+ 6,669,513 non-embedding parameters
53
+ - Context: 1,024 tokens
54
+ - Width/layers: 288 / 9
55
+ - Token-attention heads: 6 query, 2 KV
56
+ - CMA: chunk=24, heads=3, expansion=2
57
+ - Contiguous-half RoPE without scaling
58
+ - No task-specific model features or tokenizer transformations
59
+
60
+ ## Training mixture
61
+
62
+ - FineWeb-Edu 100BT shuffled: 55.00% of trained tokens
63
+ - Cosmopedia v2: 25.00% of trained tokens
64
+ - FineWeb-HQ: 10.00% of trained tokens
65
+ - FineMath 4+: 10.00% of trained tokens
66
+
67
+ FineMath-4+ supplies high-quality mathematical explanations and reasoning as
68
+ ordinary causal-language-model text, with no task-specific model features or
69
+ tokenizer transformations. All four training sources are streamed natural-text
70
+ corpora.
71
+
72
+ ## Zero-shot evaluation at step 0
73
+
74
+ The four lm-eval tasks use normalized accuracy when supplied by lm-eval
75
+ 0.4.12, with float32 weights and softmax. ArithMark
76
+ uses float32 weights and its official raw continuation log-likelihood-sum rule.
77
+
78
+ | Benchmark | Accuracy |
79
+ |---|---:|
80
+ | HellaSwag | 26.31% |
81
+ | ARC-Easy | 25.55% |
82
+ | ARC-Challenge | 23.46% |
83
+ | PIQA | 49.56% |
84
+ | ArithMark-2 | 22.28% |
85
+ | ARC mean | 24.51% |
86
+ | Open SLM Leaderboard-style average | 30.66% |
87
+
88
+ The average is `(HellaSwag + mean(ARC-Easy, ARC-Challenge) + PIQA +
89
+ ArithMark-2) / 4`, matching the Open SLM Leaderboard formula.
90
+
91
+ WikiText-103 validation at this step: loss 8.3686, perplexity
92
+ 4309.76, normalized BPB 3.7838 over 358,911
93
+ scored tokens and 1,145,226 normalized UTF-8 bytes.
benchmark_results.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 0,
3
+ "parameters": 7849161,
4
+ "lm_eval_version": "0.4.12",
5
+ "num_fewshot": 0,
6
+ "leaderboard_accuracy": {
7
+ "arc_easy": 0.25547138047138046,
8
+ "arc_challenge": 0.23464163822525597,
9
+ "hellaswag": 0.26309500099581756,
10
+ "piqa": 0.4956474428726877
11
+ },
12
+ "lm_eval_metric_policy": "acc_norm when available, otherwise acc",
13
+ "selection_metric": "open_slm_leaderboard_average",
14
+ "arithmark_2.0": {
15
+ "acc": 0.2228,
16
+ "correct": 557,
17
+ "total": 2500,
18
+ "by_operator_count": {
19
+ "1": {
20
+ "acc": 0.2208,
21
+ "correct": 276,
22
+ "total": 1250
23
+ },
24
+ "2": {
25
+ "acc": 0.21866666666666668,
26
+ "correct": 164,
27
+ "total": 750
28
+ },
29
+ "3": {
30
+ "acc": 0.234,
31
+ "correct": 117,
32
+ "total": 500
33
+ }
34
+ }
35
+ },
36
+ "arc_average": 0.2450565093483182,
37
+ "open_slm_leaderboard_average": 0.30664973830420583,
38
+ "average_formula": "(hellaswag + mean(arc_easy, arc_challenge) + piqa + arithmark_2.0) / 4",
39
+ "validation": {
40
+ "loss": 8.368637380723772,
41
+ "perplexity": 4309.759498478308,
42
+ "normalized_bpb": 3.7837711245079073,
43
+ "tokens": 358911,
44
+ "normalized_utf8_bytes": 1145226
45
+ }
46
+ }
config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "CMAForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "modeling_cma.CMAConfig",
7
+ "AutoModelForCausalLM": "modeling_cma.CMAForCausalLM"
8
+ },
9
+ "model_type": "cma",
10
+ "vocab_size": 4096,
11
+ "seq_len": 1024,
12
+ "max_position_embeddings": 1024,
13
+ "n_positions": 1024,
14
+ "n_ctx": 1024,
15
+ "d_model": 288,
16
+ "n_layers": 9,
17
+ "n_heads": 6,
18
+ "n_kv_heads": 2,
19
+ "chunk": 24,
20
+ "cma_heads": 3,
21
+ "expand": 2,
22
+ "cma_identity_prob": 0.9,
23
+ "bos_token_id": 1,
24
+ "eos_token_id": 2,
25
+ "pad_token_id": 2,
26
+ "tie_word_embeddings": true,
27
+ "safe_serialization": true,
28
+ "minimum_torch_version": "2.5",
29
+ "torch_dtype": "float32",
30
+ "transformers_version": "5.13.0"
31
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "eos_token_id": 2,
3
+ "pad_token_id": 2,
4
+ "do_sample": false,
5
+ "repetition_penalty": 1.2
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:767c860fea9367ec00d2e2b3d271f29fa6d926be279518e1fe15fc59a0b37b0f
3
+ size 36330428
modeling_cma.py ADDED
@@ -0,0 +1,362 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import math
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+ # CMA is text-only. Disable Transformers' optional vision backend before it
8
+ # imports PreTrainedModel: some cloud images expose a system torchvision built
9
+ # against a different PyTorch, which otherwise crashes on torchvision::nms.
10
+ from transformers.utils import import_utils as _transformers_import_utils
11
+ _transformers_import_utils._torchvision_available = False
12
+ _transformers_import_utils.is_torchvision_available = lambda: False
13
+ try:
14
+ from transformers import GenerationMixin
15
+ except ImportError:
16
+ from transformers.generation import GenerationMixin
17
+ from transformers import PretrainedConfig, PreTrainedModel
18
+ from transformers.modeling_outputs import CausalLMOutputWithPast
19
+
20
+
21
+ class CMAConfig(PretrainedConfig):
22
+ model_type = "cma"
23
+
24
+ def __init__(
25
+ self,
26
+ vocab_size=4096,
27
+ seq_len=512,
28
+ d_model=288,
29
+ n_layers=9,
30
+ n_heads=6,
31
+ n_kv_heads=2,
32
+ chunk=24,
33
+ cma_heads=3,
34
+ expand=2,
35
+ cma_identity_prob=0.90,
36
+ max_position_embeddings=None,
37
+ n_positions=None,
38
+ n_ctx=None,
39
+ **kwargs,
40
+ ):
41
+ super().__init__(**kwargs)
42
+ self.vocab_size = vocab_size
43
+ self.seq_len = seq_len
44
+ self.max_position_embeddings = max_position_embeddings or seq_len
45
+ self.n_positions = n_positions or self.max_position_embeddings
46
+ self.n_ctx = n_ctx or self.max_position_embeddings
47
+ self.d_model = d_model
48
+ self.n_layers = n_layers
49
+ self.n_heads = n_heads
50
+ self.n_kv_heads = n_kv_heads
51
+ self.chunk = chunk
52
+ self.cma_heads = cma_heads
53
+ self.expand = expand
54
+ self.cma_identity_prob = cma_identity_prob
55
+
56
+
57
+ class RMSNorm(nn.Module):
58
+ def __init__(self, d):
59
+ super().__init__()
60
+ self.w = nn.Parameter(torch.ones(d))
61
+
62
+ def forward(self, x):
63
+ return F.rms_norm(
64
+ x, (x.shape[-1],), self.w.to(dtype=x.dtype), eps=1e-6
65
+ )
66
+
67
+
68
+ def rope_cache(seq_len, head_dim, device, base=10000.0):
69
+ inv = 1.0 / (
70
+ base ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim)
71
+ )
72
+ t = torch.arange(seq_len, device=device).float()
73
+ freqs = torch.outer(t, inv)
74
+ return torch.cos(freqs), torch.sin(freqs)
75
+
76
+
77
+ def apply_rope(x, cos, sin):
78
+ x1, x2 = x.chunk(2, dim=-1)
79
+ return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1)
80
+
81
+
82
+ class TokenAttention(nn.Module):
83
+ def __init__(self, d, n_heads, n_kv_heads):
84
+ super().__init__()
85
+ if d % n_heads or n_heads % n_kv_heads:
86
+ raise ValueError("d_model % n_heads and n_heads % n_kv_heads must be zero.")
87
+ self.h, self.kv_h, self.hd = n_heads, n_kv_heads, d // n_heads
88
+ self.q = nn.Linear(d, d, bias=False)
89
+ self.k = nn.Linear(d, n_kv_heads * self.hd, bias=False)
90
+ self.v = nn.Linear(d, n_kv_heads * self.hd, bias=False)
91
+ self.o = nn.Linear(d, d, bias=False)
92
+ self.qn, self.kn = RMSNorm(self.hd), RMSNorm(self.hd)
93
+
94
+ def forward(self, x, cos, sin):
95
+ B, T, d = x.shape
96
+ q = self.q(x).view(B, T, self.h, self.hd).transpose(1, 2)
97
+ k = self.k(x).view(B, T, self.kv_h, self.hd).transpose(1, 2)
98
+ v = self.v(x).view(B, T, self.kv_h, self.hd).transpose(1, 2)
99
+ q, k = self.qn(q), self.kn(k)
100
+ q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
101
+ # Native GQA keeps compact K/V heads and lets SDPA select a fused CUDA
102
+ # implementation without materializing repeated K/V activations.
103
+ y = F.scaled_dot_product_attention(
104
+ q,
105
+ k,
106
+ v,
107
+ is_causal=True,
108
+ enable_gqa=self.h != self.kv_h,
109
+ )
110
+ return self.o(y.transpose(1, 2).reshape(B, T, d))
111
+
112
+
113
+ class CMA(nn.Module):
114
+ def __init__(self, d, chunk=32, heads=4, expand=2, identity_prob=0.90):
115
+ super().__init__()
116
+ if min(d, chunk, heads, expand) <= 0:
117
+ raise ValueError("CMA dimensions and expansion must be positive.")
118
+ if d % chunk or chunk % heads:
119
+ raise ValueError("CMA requires d % chunk == 0 and chunk % heads == 0.")
120
+ if d // chunk < 2:
121
+ raise ValueError("CMA requires at least two channel chunks.")
122
+ self.n, self.c, self.h = d // chunk, chunk, heads
123
+ self.hd = chunk // heads
124
+ self.chunk_emb = nn.Parameter(torch.randn(self.n, chunk) * 0.02)
125
+ self.wqk = nn.Parameter(torch.randn(self.n, chunk, 2 * chunk) * 0.02)
126
+ self.global_proj = nn.Linear(d, chunk, bias=False)
127
+ self.wv = nn.Linear(d, d * expand, bias=False)
128
+ self.bias = nn.Parameter(torch.zeros(heads, self.n, self.n))
129
+ self.qn, self.kn = RMSNorm(self.hd), RMSNorm(self.hd)
130
+ self.logit_scale = nn.Parameter(torch.zeros(heads))
131
+ self.layer_gain = nn.Parameter(torch.zeros(heads))
132
+ self.route_gate_weight = nn.Parameter(
133
+ torch.randn(heads, self.hd) * 0.02
134
+ )
135
+ self.route_gate_bias = nn.Parameter(torch.zeros(heads))
136
+ self.gate = nn.Linear(d, d * expand, bias=False)
137
+ self.o = nn.Linear(d * expand, d, bias=False)
138
+ nn.init.zeros_(self.o.weight)
139
+ if not 0.0 < identity_prob < 1.0:
140
+ raise ValueError("cma_identity_prob must be between zero and one.")
141
+ diagonal_bias = math.log(
142
+ (self.n - 1) * identity_prob / (1.0 - identity_prob)
143
+ )
144
+ with torch.no_grad():
145
+ self.bias.add_(torch.eye(self.n) * diagonal_bias)
146
+ self.expand = expand
147
+ self.record_diagnostics = False
148
+ self.last_diagnostics = None
149
+
150
+ def forward(self, x):
151
+ B, T, d = x.shape
152
+ xc = x.view(B * T, self.n, self.c)
153
+ global_state = self.global_proj(x).reshape(B * T, 1, self.c)
154
+ qk_in = xc + self.chunk_emb + global_state
155
+ value_slots = self.wv(x).reshape(
156
+ B * T, self.n, self.h, self.hd, self.expand
157
+ )
158
+ # Key j summarizes value slot j, restoring the key/value contract while
159
+ # retaining the full dense value projection and exact parameter count.
160
+ key_in = value_slots.mean(dim=-1).reshape(B * T, self.n, self.c)
161
+ key_in = key_in + self.chunk_emb
162
+ q = torch.einsum("bnc,nco->bno", qk_in, self.wqk[..., : self.c])
163
+ k = torch.einsum("bnc,nco->bno", key_in, self.wqk[..., self.c :])
164
+ v = value_slots.reshape(
165
+ B * T, self.n, self.h, self.hd * self.expand
166
+ )
167
+ q = self.qn(q.view(B * T, self.n, self.h, self.hd)).transpose(1, 2)
168
+ k = self.kn(k.view(B * T, self.n, self.h, self.hd)).transpose(1, 2)
169
+ v = v.transpose(1, 2)
170
+ q = F.normalize(q.float(), dim=-1).to(v.dtype)
171
+ k = F.normalize(k.float(), dim=-1).to(v.dtype)
172
+ scale = self.logit_scale.clamp(max=math.log(100.0)).exp()
173
+ scale = scale.view(1, self.h, 1, 1)
174
+ routed = F.scaled_dot_product_attention(
175
+ q * scale.to(q.dtype),
176
+ k,
177
+ v,
178
+ attn_mask=self.bias.unsqueeze(0).to(q.dtype),
179
+ dropout_p=0.0,
180
+ is_causal=False,
181
+ scale=1.0,
182
+ )
183
+ route_signal = (
184
+ torch.einsum(
185
+ "bhnd,hd->bhn", q.float(), self.route_gate_weight.float()
186
+ )
187
+ + self.route_gate_bias.float().view(1, self.h, 1)
188
+ + self.layer_gain.float().view(1, self.h, 1)
189
+ )
190
+ route_coeff = torch.tanh(route_signal).to(v.dtype)
191
+ contribution = route_coeff.unsqueeze(-1) * (routed - v)
192
+ y = v + contribution
193
+ if self.record_diagnostics:
194
+ attn_logits = (
195
+ q.float() @ k.float().transpose(-2, -1)
196
+ ) * scale.float()
197
+ attn_logits = attn_logits + self.bias.float().unsqueeze(0)
198
+ attn = F.softmax(attn_logits, dim=-1, dtype=torch.float32)
199
+ probs = attn.float().clamp_min(1e-9)
200
+ entropy = -(probs * probs.log()).sum(dim=-1) / math.log(self.n)
201
+ diagonal_mass = probs.diagonal(dim1=-2, dim2=-1).mean()
202
+ if self.h > 1:
203
+ flattened = probs.transpose(0, 1).reshape(self.h, -1)
204
+ normalized = F.normalize(flattened, dim=-1)
205
+ similarity = normalized @ normalized.mT
206
+ off_diagonal = (
207
+ similarity.sum() - similarity.diagonal().sum()
208
+ ) / (self.h * (self.h - 1))
209
+ else:
210
+ off_diagonal = probs.new_zeros(())
211
+ self.last_diagnostics = {
212
+ "entropy": entropy.mean().item(),
213
+ "diagonal_mass": diagonal_mass.item(),
214
+ "head_similarity": off_diagonal.item(),
215
+ "gate_mean": route_coeff.float().mean().item(),
216
+ "gate_std": route_coeff.float().std(unbiased=False).item(),
217
+ "gate_abs_mean": route_coeff.float().abs().mean().item(),
218
+ "gate_saturation": (
219
+ route_coeff.float().abs() > 0.95
220
+ ).float().mean().item(),
221
+ "contribution_ratio": (
222
+ contribution.float().norm() / (v.float().norm() + 1e-9)
223
+ ).item(),
224
+ }
225
+ y = y.transpose(1, 2).reshape(B, T, d * self.expand)
226
+ return self.o(y * F.silu(self.gate(x)))
227
+
228
+
229
+ class Block(nn.Module):
230
+ def __init__(self, config):
231
+ super().__init__()
232
+ d = config.d_model
233
+ self.n1, self.n2 = RMSNorm(d), RMSNorm(d)
234
+ self.attn = TokenAttention(d, config.n_heads, config.n_kv_heads)
235
+ self.mix = CMA(
236
+ d,
237
+ config.chunk,
238
+ config.cma_heads,
239
+ config.expand,
240
+ config.cma_identity_prob,
241
+ )
242
+
243
+ def forward(self, x, cos, sin):
244
+ # Expose the full attention -> residual -> norm -> CMA handoff to
245
+ # Inductor as one graph so pointwise boundaries can be fused.
246
+ x = torch.add(x, self.attn(self.n1(x), cos, sin))
247
+ return torch.add(x, self.mix(self.n2(x)))
248
+
249
+
250
+ class CMAModel(nn.Module):
251
+ def __init__(self, config):
252
+ super().__init__()
253
+ d = config.d_model
254
+ self.config = config
255
+ self.emb = nn.Embedding(config.vocab_size, d)
256
+ self.blocks = nn.ModuleList(Block(config) for _ in range(config.n_layers))
257
+ self.norm = RMSNorm(d)
258
+ hd = d // config.n_heads
259
+ cos, sin = rope_cache(config.seq_len, hd, "cpu")
260
+ self.register_buffer("cos", cos)
261
+ self.register_buffer("sin", sin)
262
+
263
+ def forward_hidden(self, idx):
264
+ if idx.size(1) > self.config.seq_len:
265
+ idx = idx[:, -self.config.seq_len :]
266
+ x = self.emb(idx)
267
+ device_type = x.device.type
268
+ rope_dtype = (
269
+ torch.get_autocast_dtype(device_type)
270
+ if torch.is_autocast_enabled(device_type)
271
+ else x.dtype
272
+ )
273
+ cos = self.cos[: idx.size(1)].to(device=idx.device, dtype=rope_dtype)
274
+ sin = self.sin[: idx.size(1)].to(device=idx.device, dtype=rope_dtype)
275
+ for b in self.blocks:
276
+ x = b(x, cos, sin)
277
+ return self.norm(x)
278
+
279
+
280
+ class CMAForCausalLM(PreTrainedModel, GenerationMixin):
281
+ config_class = CMAConfig
282
+ base_model_prefix = "model"
283
+ _tied_weights_keys = {"head.weight": "model.emb.weight"}
284
+ all_tied_weights_keys = {"head.weight": "model.emb.weight"}
285
+
286
+ def __init__(self, config):
287
+ super().__init__(config)
288
+ self.model = CMAModel(config)
289
+ self.head = nn.Linear(config.d_model, config.vocab_size, bias=False)
290
+ # Modern Transformers creates loader metadata and performs configured
291
+ # tying in post_init(); omitting it leaves all_tied_weights_keys absent.
292
+ self.post_init()
293
+ self.head.weight = self.model.emb.weight
294
+
295
+ def get_input_embeddings(self):
296
+ return self.model.emb
297
+
298
+ def set_input_embeddings(self, value):
299
+ self.model.emb = value
300
+
301
+ def get_output_embeddings(self):
302
+ return self.head
303
+
304
+ def set_output_embeddings(self, new_embeddings):
305
+ self.head = new_embeddings
306
+
307
+ def raw_logits(self, idx):
308
+ return self.head(self.model.forward_hidden(idx))
309
+
310
+ def logits(self, idx):
311
+ return self.raw_logits(idx)
312
+
313
+ def _masked_logits(self, input_ids, attention_mask):
314
+ if attention_mask is None or bool(attention_mask.all()):
315
+ return self.logits(input_ids)
316
+
317
+ B, T = input_ids.shape
318
+ out = None
319
+ for i in range(B):
320
+ keep = attention_mask[i].bool().nonzero(as_tuple=False).flatten()
321
+ if keep.numel() == 0:
322
+ keep = torch.tensor([T - 1], device=input_ids.device)
323
+ trimmed = input_ids[i, keep].unsqueeze(0)
324
+ logits_i = self.logits(trimmed)
325
+ if out is None:
326
+ out = logits_i.new_zeros(B, T, logits_i.size(-1))
327
+ out[i, keep, :] = logits_i[0, -keep.numel() :, :]
328
+ return out
329
+
330
+ def forward(
331
+ self,
332
+ input_ids=None,
333
+ attention_mask=None,
334
+ labels=None,
335
+ use_cache=False,
336
+ past_key_values=None,
337
+ **kwargs,
338
+ ):
339
+ if input_ids.size(1) > self.config.seq_len:
340
+ input_ids = input_ids[:, -self.config.seq_len :]
341
+ if attention_mask is not None:
342
+ attention_mask = attention_mask[:, -self.config.seq_len :]
343
+ if labels is not None:
344
+ labels = labels[:, -self.config.seq_len :]
345
+ logits = self._masked_logits(input_ids, attention_mask)
346
+ loss = None
347
+ if labels is not None:
348
+ shift_logits = logits[:, :-1, :].contiguous()
349
+ shift_labels = labels[:, 1:].contiguous()
350
+ loss = F.cross_entropy(
351
+ shift_logits.view(-1, shift_logits.size(-1)).float(),
352
+ shift_labels.view(-1),
353
+ ignore_index=-100,
354
+ )
355
+ return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=None)
356
+
357
+ def prepare_inputs_for_generation(self, input_ids, **kwargs):
358
+ attention_mask = kwargs.get("attention_mask")
359
+ result = {"input_ids": input_ids[:, -self.config.seq_len :]}
360
+ if attention_mask is not None:
361
+ result["attention_mask"] = attention_mask[:, -self.config.seq_len :]
362
+ return result
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": true,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<bos>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<eos>",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1000000000,
10
+ "pad_token": "<eos>",
11
+ "tokenizer_class": "TokenizersBackend",
12
+ "unk_token": "<unk>",
13
+ "vocab_size": 4096
14
+ }
training_config.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "steps": 40000,
3
+ "batch_size": 256,
4
+ "grad_accum": 2,
5
+ "seq_len": 1024,
6
+ "d_model": 288,
7
+ "n_layers": 9,
8
+ "n_heads": 6,
9
+ "n_kv_heads": 2,
10
+ "chunk": 24,
11
+ "cma_heads": 3,
12
+ "expand": 2,
13
+ "cma_identity_prob": 0.9,
14
+ "lr": 0.0025,
15
+ "muon_lr": 0.03,
16
+ "muon_momentum": 0.95,
17
+ "muon_ns_steps": 5,
18
+ "muon_nesterov": true,
19
+ "warmup": 750,
20
+ "decay_start_frac": 0.85,
21
+ "min_lr_ratio": 0.0,
22
+ "wd": 0.01,
23
+ "grad_clip": 1.0,
24
+ "log_every": 5,
25
+ "early_eval_step": 1000,
26
+ "eval_every": 10000,
27
+ "val_batch_size": 32,
28
+ "val_batches": 0,
29
+ "val_stride": 512,
30
+ "infer_tokens": 512,
31
+ "infer_repeat_penalty": 1.2,
32
+ "infer_prompt": "The process of photosynthesis",
33
+ "lm_eval_tasks": "arc_easy,arc_challenge,hellaswag,piqa",
34
+ "lm_eval_batch_size": "auto",
35
+ "lm_eval_device": "cuda",
36
+ "lm_eval_dtype": "float32",
37
+ "lm_eval_softmax_dtype": "float32",
38
+ "lm_eval_expected_version": "0.4.12",
39
+ "lm_eval_retries": 3,
40
+ "lm_eval_export_dir": "cma_8m_lm_eval_hf",
41
+ "lm_eval_output_dir": "lm_eval_results_cma_8m",
42
+ "arithmark_batch_size": 16,
43
+ "arithmark_data_path": "benchmark_cache/arithmark_2.0.jsonl",
44
+ "arithmark_force_download": false,
45
+ "recipe_version": "cma_8m_finemath4plus_v5",
46
+ "hf_repo_id": "User01110/cma-8M",
47
+ "hf_repo_private": false,
48
+ "hub_upload_retries": 3,
49
+ "tokenizer_name": "AxiomicLabs/GPT-S-5M",
50
+ "tokenizer_revision": "df47402",
51
+ "data_seed": 1337,
52
+ "shuffle_buffer": 50000,
53
+ "tokenize_batch_size": 64,
54
+ "compile": true,
55
+ "compile_mode": "max-autotune",
56
+ "compile_fullgraph": true,
57
+ "compile_dynamic": false
58
+ }