ThomasHeim commited on
Commit
f174ef0
·
verified ·
1 Parent(s): 137b270

Delete sudoku-extreme/shrek-large

Browse files
Files changed (43) hide show
  1. sudoku-extreme/shrek-large/all_config.yaml +0 -36
  2. sudoku-extreme/shrek-large/hrm_act_v1.py +0 -408
  3. sudoku-extreme/shrek-large/losses.py +0 -130
  4. sudoku-extreme/shrek-large/step_10416 +0 -3
  5. sudoku-extreme/shrek-large/step_11718 +0 -3
  6. sudoku-extreme/shrek-large/step_1302 +0 -3
  7. sudoku-extreme/shrek-large/step_13020 +0 -3
  8. sudoku-extreme/shrek-large/step_14322 +0 -3
  9. sudoku-extreme/shrek-large/step_15624 +0 -3
  10. sudoku-extreme/shrek-large/step_16926 +0 -3
  11. sudoku-extreme/shrek-large/step_18228 +0 -3
  12. sudoku-extreme/shrek-large/step_19530 +0 -3
  13. sudoku-extreme/shrek-large/step_20832 +0 -3
  14. sudoku-extreme/shrek-large/step_22134 +0 -3
  15. sudoku-extreme/shrek-large/step_23436 +0 -3
  16. sudoku-extreme/shrek-large/step_24738 +0 -3
  17. sudoku-extreme/shrek-large/step_2604 +0 -3
  18. sudoku-extreme/shrek-large/step_26040 +0 -3
  19. sudoku-extreme/shrek-large/step_27342 +0 -3
  20. sudoku-extreme/shrek-large/step_28644 +0 -3
  21. sudoku-extreme/shrek-large/step_29946 +0 -3
  22. sudoku-extreme/shrek-large/step_31248 +0 -3
  23. sudoku-extreme/shrek-large/step_32550 +0 -3
  24. sudoku-extreme/shrek-large/step_33852 +0 -3
  25. sudoku-extreme/shrek-large/step_35154 +0 -3
  26. sudoku-extreme/shrek-large/step_36456 +0 -3
  27. sudoku-extreme/shrek-large/step_37758 +0 -3
  28. sudoku-extreme/shrek-large/step_3906 +0 -3
  29. sudoku-extreme/shrek-large/step_39060 +0 -3
  30. sudoku-extreme/shrek-large/step_40362 +0 -3
  31. sudoku-extreme/shrek-large/step_41664 +0 -3
  32. sudoku-extreme/shrek-large/step_42966 +0 -3
  33. sudoku-extreme/shrek-large/step_44268 +0 -3
  34. sudoku-extreme/shrek-large/step_45570 +0 -3
  35. sudoku-extreme/shrek-large/step_46872 +0 -3
  36. sudoku-extreme/shrek-large/step_48174 +0 -3
  37. sudoku-extreme/shrek-large/step_49476 +0 -3
  38. sudoku-extreme/shrek-large/step_50778 +0 -3
  39. sudoku-extreme/shrek-large/step_5208 +0 -3
  40. sudoku-extreme/shrek-large/step_52080 +0 -3
  41. sudoku-extreme/shrek-large/step_6510 +0 -3
  42. sudoku-extreme/shrek-large/step_7812 +0 -3
  43. sudoku-extreme/shrek-large/step_9114 +0 -3
sudoku-extreme/shrek-large/all_config.yaml DELETED
@@ -1,36 +0,0 @@
1
- arch:
2
- H_cycles: 2
3
- H_layers: 4
4
- L_cycles: 2
5
- L_layers: 4
6
- expansion: 4
7
- halt_exploration_prob: 0.1
8
- halt_max_steps: 16
9
- hidden_size: 512
10
- loss:
11
- loss_type: stablemax_cross_entropy
12
- name: losses@ACTLossHead
13
- name: hrm.hrm_act_v1@HierarchicalReasoningModel_ACTV1
14
- num_heads: 8
15
- pos_encodings: rope
16
- puzzle_emb_ndim: 512
17
- beta1: 0.9
18
- beta2: 0.95
19
- checkpoint_every_eval: true
20
- checkpoint_path: checkpoints/HRM_Sudoku_Comparison/SHREK_Large_Sudoku
21
- data_path: ../../dataset/data/sudoku-extreme-1k-aug-1000-hint
22
- ema: true
23
- ema_rate: 0.999
24
- epochs: 40000
25
- eval_interval: 1000
26
- eval_save_outputs: []
27
- global_batch_size: 768
28
- lr: 0.0001
29
- lr_min_ratio: 1.0
30
- lr_warmup_steps: 2000
31
- project_name: HRM_Sudoku_Comparison
32
- puzzle_emb_lr: 0.0001
33
- puzzle_emb_weight_decay: 1.0
34
- run_name: SHREK_Large_Sudoku
35
- seed: 0
36
- weight_decay: 1.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sudoku-extreme/shrek-large/hrm_act_v1.py DELETED
@@ -1,408 +0,0 @@
1
- from typing import Tuple, List, Dict, Optional
2
- from dataclasses import dataclass
3
- import math
4
-
5
- import torch
6
- import torch.nn.functional as F
7
- from torch import nn
8
- from pydantic import BaseModel
9
-
10
- from models.common import trunc_normal_init_
11
- from models.layers import rms_norm, SwiGLU, Attention, RotaryEmbedding, CosSin, CastedEmbedding, CastedLinear
12
- from models.sparse_embedding import CastedSparseEmbedding
13
- from models.hrm.error_singals import get_error_signal # SHREK: error signal module
14
-
15
-
16
- @dataclass
17
- class HierarchicalReasoningModel_ACTV1InnerCarry:
18
- z_H: torch.Tensor
19
- z_L: torch.Tensor
20
- # SHREK: prev_pred stores last step's argmax predictions for flip rate computation.
21
- # zeros = fresh start (first step after init or reset gives flip_rate ≈ 1.0)
22
- prev_pred: torch.Tensor # (B, seq_len) int32
23
- # SHREK: Q-values cached in carry — no longer used for Q-targets (see Bug 4),
24
- # but kept because removing them causes torch.compile regression.
25
- prev_q_halt: torch.Tensor # (B,) float32
26
- prev_q_continue: torch.Tensor # (B,) float32
27
-
28
-
29
- @dataclass
30
- class HierarchicalReasoningModel_ACTV1Carry:
31
- inner_carry: HierarchicalReasoningModel_ACTV1InnerCarry
32
-
33
- steps: torch.Tensor
34
- halted: torch.Tensor
35
-
36
- current_data: Dict[str, torch.Tensor]
37
-
38
-
39
- class HierarchicalReasoningModel_ACTV1Config(BaseModel):
40
- batch_size: int
41
- seq_len: int
42
- puzzle_emb_ndim: int = 0
43
- num_puzzle_identifiers: int
44
- vocab_size: int
45
-
46
- H_cycles: int
47
- L_cycles: int
48
-
49
- H_layers: int
50
- L_layers: int
51
-
52
- # Transformer config
53
- hidden_size: int
54
- expansion: float
55
- num_heads: int
56
- pos_encodings: str
57
-
58
- rms_norm_eps: float = 1e-5
59
- rope_theta: float = 10000.0
60
-
61
- # Halting Q-learning config
62
- halt_max_steps: int
63
- halt_exploration_prob: float
64
-
65
- # SHREK: error injection warmup — ramps alpha from 0 to alpha_max over warmup steps.
66
- # Prevents small models from collapsing before the error estimator is accurate.
67
- alpha_max: float = 0.01
68
- alpha_warmup_steps: int = 5000
69
-
70
- forward_dtype: str = "bfloat16"
71
-
72
-
73
- class HierarchicalReasoningModel_ACTV1Block(nn.Module):
74
- def __init__(self, config: HierarchicalReasoningModel_ACTV1Config) -> None:
75
- super().__init__()
76
-
77
- self.self_attn = Attention(
78
- hidden_size=config.hidden_size,
79
- head_dim=config.hidden_size // config.num_heads,
80
- num_heads=config.num_heads,
81
- num_key_value_heads=config.num_heads,
82
- causal=False
83
- )
84
- self.mlp = SwiGLU(
85
- hidden_size=config.hidden_size,
86
- expansion=config.expansion,
87
- )
88
- self.norm_eps = config.rms_norm_eps
89
-
90
- def forward(self, cos_sin: CosSin, hidden_states: torch.Tensor) -> torch.Tensor:
91
- # Post Norm
92
- # Self Attention
93
- hidden_states = rms_norm(hidden_states + self.self_attn(cos_sin=cos_sin, hidden_states=hidden_states), variance_epsilon=self.norm_eps)
94
- # Fully Connected
95
- hidden_states = rms_norm(hidden_states + self.mlp(hidden_states), variance_epsilon=self.norm_eps)
96
- return hidden_states
97
-
98
-
99
- class HierarchicalReasoningModel_ACTV1ReasoningModule(nn.Module):
100
- def __init__(self, layers: List[HierarchicalReasoningModel_ACTV1Block]):
101
- super().__init__()
102
-
103
- self.layers = torch.nn.ModuleList(layers)
104
-
105
- def forward(self, hidden_states: torch.Tensor, input_injection: torch.Tensor, **kwargs) -> torch.Tensor:
106
- # Input injection (add)
107
- hidden_states = hidden_states + input_injection
108
- # Layers
109
- for layer in self.layers:
110
- hidden_states = layer(hidden_states=hidden_states, **kwargs)
111
-
112
- return hidden_states
113
-
114
-
115
- class HierarchicalReasoningModel_ACTV1_Inner(nn.Module):
116
- def __init__(self, config: HierarchicalReasoningModel_ACTV1Config) -> None:
117
- super().__init__()
118
- self.config = config
119
- self.forward_dtype = getattr(torch, self.config.forward_dtype)
120
-
121
- # I/O
122
- self.embed_scale = math.sqrt(self.config.hidden_size)
123
- embed_init_std = 1.0 / self.embed_scale
124
-
125
- self.embed_tokens = CastedEmbedding(self.config.vocab_size, self.config.hidden_size, init_std=embed_init_std, cast_to=self.forward_dtype)
126
- self.lm_head = CastedLinear(self.config.hidden_size, self.config.vocab_size, bias=False)
127
- # Q-head: same as original HRM — reads CLS token (position 0) only
128
- self.q_head = CastedLinear(self.config.hidden_size, 2, bias=True)
129
-
130
- # SHREK: error_encoder maps the scalar error score -> hidden_size vector for injection into z_H
131
- # alpha follows a linear warmup schedule (0 → alpha_max over warmup steps).
132
- # This lets the error estimator train before its signal affects z_H.
133
- self.error_encoder = nn.Linear(1, self.config.hidden_size)
134
- # SHREK: step counter for alpha warmup (not a learned parameter)
135
- self.register_buffer('_alpha_step', torch.tensor(0, dtype=torch.long))
136
- # SHREK: error_estimator reads z_H and predicts how wrong the model is.
137
- # trained via auxiliary loss in pretrain.py using the real lm_loss as target.
138
- # catches "stuck but wrong" — a model confidently on the wrong answer.
139
- # flip rate catches oscillation; estimator catches confident-but-wrong.
140
- self.error_estimator = nn.Linear(self.config.hidden_size, 1)
141
-
142
- self.puzzle_emb_len = -(self.config.puzzle_emb_ndim // -self.config.hidden_size) # ceil div
143
- if self.config.puzzle_emb_ndim > 0:
144
- # Zero init puzzle embeddings
145
- self.puzzle_emb = CastedSparseEmbedding(self.config.num_puzzle_identifiers, self.config.puzzle_emb_ndim,
146
- batch_size=self.config.batch_size, init_std=0, cast_to=self.forward_dtype)
147
-
148
- # LM Blocks
149
- if self.config.pos_encodings == "rope":
150
- self.rotary_emb = RotaryEmbedding(dim=self.config.hidden_size // self.config.num_heads,
151
- max_position_embeddings=self.config.seq_len + self.puzzle_emb_len,
152
- base=self.config.rope_theta)
153
- elif self.config.pos_encodings == "learned":
154
- self.embed_pos = CastedEmbedding(self.config.seq_len + self.puzzle_emb_len, self.config.hidden_size, init_std=embed_init_std, cast_to=self.forward_dtype)
155
- else:
156
- raise NotImplementedError()
157
-
158
- # Reasoning Layers
159
- self.H_level = HierarchicalReasoningModel_ACTV1ReasoningModule(layers=[HierarchicalReasoningModel_ACTV1Block(self.config) for _i in range(self.config.H_layers)])
160
- self.L_level = HierarchicalReasoningModel_ACTV1ReasoningModule(layers=[HierarchicalReasoningModel_ACTV1Block(self.config) for _i in range(self.config.L_layers)])
161
-
162
- # Initial states
163
- self.H_init = nn.Buffer(trunc_normal_init_(torch.empty(self.config.hidden_size, dtype=self.forward_dtype), std=1), persistent=True)
164
- self.L_init = nn.Buffer(trunc_normal_init_(torch.empty(self.config.hidden_size, dtype=self.forward_dtype), std=1), persistent=True)
165
-
166
- # Q head special init
167
- # Init Q to (almost) zero for faster learning during bootstrapping
168
- with torch.no_grad():
169
- self.q_head.weight.zero_()
170
- self.q_head.bias.fill_(-5) # type: ignore
171
-
172
- def _input_embeddings(self, input: torch.Tensor, puzzle_identifiers: torch.Tensor):
173
- # Token embedding
174
- embedding = self.embed_tokens(input.to(torch.int32))
175
-
176
- # Puzzle embeddings
177
- if self.config.puzzle_emb_ndim > 0:
178
- puzzle_embedding = self.puzzle_emb(puzzle_identifiers)
179
-
180
- pad_count = self.puzzle_emb_len * self.config.hidden_size - puzzle_embedding.shape[-1]
181
- if pad_count > 0:
182
- puzzle_embedding = F.pad(puzzle_embedding, (0, pad_count))
183
-
184
- embedding = torch.cat((puzzle_embedding.view(-1, self.puzzle_emb_len, self.config.hidden_size), embedding), dim=-2)
185
-
186
- # Position embeddings
187
- if self.config.pos_encodings == "learned":
188
- # scale by 1/sqrt(2) to maintain forward variance
189
- embedding = 0.707106781 * (embedding + self.embed_pos.embedding_weight.to(self.forward_dtype))
190
-
191
- # Scale
192
- return self.embed_scale * embedding
193
-
194
- def empty_carry(self, batch_size: int):
195
- return HierarchicalReasoningModel_ACTV1InnerCarry(
196
- z_H=torch.empty(batch_size, self.config.seq_len + self.puzzle_emb_len, self.config.hidden_size, dtype=self.forward_dtype),
197
- z_L=torch.empty(batch_size, self.config.seq_len + self.puzzle_emb_len, self.config.hidden_size, dtype=self.forward_dtype),
198
- # SHREK: zeros = no previous prediction — first step gives flip_rate ≈ 1.0
199
- # device=H_init.device ensures prev_pred is on CUDA, matching z_H and logits
200
- prev_pred=torch.zeros(batch_size, self.config.seq_len, dtype=torch.int32, device=self.H_init.device),
201
- prev_q_halt=torch.full((batch_size,), -5.0, device=self.H_init.device),
202
- prev_q_continue=torch.full((batch_size,), -5.0, device=self.H_init.device),
203
- )
204
-
205
- def reset_carry(self, reset_flag: torch.Tensor, carry: HierarchicalReasoningModel_ACTV1InnerCarry):
206
- # SHREK: zero out prev_pred for reset sequences so they start fresh.
207
- # a reset sequence is one that just halted — it will solve a new puzzle next.
208
- # zeroing prev_pred means first step gives flip_rate ≈ 1.0 (maximum uncertainty).
209
- new_prev_pred = torch.where(
210
- reset_flag.view(-1, 1),
211
- torch.zeros_like(carry.prev_pred),
212
- carry.prev_pred
213
- )
214
- new_prev_q_halt = torch.where(reset_flag, torch.full_like(carry.prev_q_halt, -5.0), carry.prev_q_halt)
215
- new_prev_q_continue = torch.where(reset_flag, torch.full_like(carry.prev_q_continue, -5.0), carry.prev_q_continue)
216
-
217
- return HierarchicalReasoningModel_ACTV1InnerCarry(
218
- z_H=torch.where(reset_flag.view(-1, 1, 1), self.H_init, carry.z_H),
219
- z_L=torch.where(reset_flag.view(-1, 1, 1), self.L_init, carry.z_L),
220
- prev_pred=new_prev_pred,
221
- prev_q_halt=new_prev_q_halt,
222
- prev_q_continue=new_prev_q_continue,
223
- )
224
-
225
-
226
- # SHREK: removed task_type parameter — error signal is now universal (no task rules needed)
227
- def forward(self, carry: HierarchicalReasoningModel_ACTV1InnerCarry, batch: Dict[str, torch.Tensor], require_trace=False):
228
- # -> Tuple[HierarchicalReasoningModel_ACTV1InnerCarry, torch.Tensor, Tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
229
- seq_info = dict(
230
- cos_sin=self.rotary_emb() if hasattr(self, "rotary_emb") else None,
231
- )
232
-
233
- # Input encoding
234
- input_embeddings = self._input_embeddings(batch["inputs"], batch["puzzle_identifiers"])
235
-
236
- z_H_trace = []
237
-
238
- # Forward iterations
239
- with torch.no_grad():
240
- z_H, z_L = carry.z_H, carry.z_L
241
-
242
- for _H_step in range(self.config.H_cycles):
243
- for _L_step in range(self.config.L_cycles):
244
- if not ((_H_step == self.config.H_cycles - 1) and (_L_step == self.config.L_cycles - 1)):
245
- z_L = self.L_level(z_L, z_H + input_embeddings, **seq_info)
246
-
247
- if not (_H_step == self.config.H_cycles - 1):
248
- z_H = self.H_level(z_H, z_L, **seq_info)
249
- if require_trace:
250
- z_H_trace.append(z_H.detach().cpu().clone())
251
-
252
- assert not z_H.requires_grad and not z_L.requires_grad
253
-
254
- # 1-step grad
255
- z_L = self.L_level(z_L, z_H + input_embeddings, **seq_info)
256
- z_H = self.H_level(z_H, z_L, **seq_info)
257
-
258
- if require_trace:
259
- z_H_trace.append(z_H.detach().cpu().clone())
260
-
261
- # LM Outputs — decode z_H into token predictions
262
- output = self.lm_head(z_H)[:, self.puzzle_emb_len:] # (B, seq_len, vocab_size)
263
-
264
- # SHREK Component 1: Combined Error Signal
265
- # Signal A — flip rate: what fraction of tokens changed from last step?
266
- # catches oscillation (model changing its mind between wrong options)
267
- flip_err, current_pred = get_error_signal(output, carry.prev_pred) # (B,), (B, seq_len)
268
-
269
- # Signal B — learned estimator: reads z_H and predicts how wrong the model is.
270
- # catches stuck-but-wrong (model confidently on wrong answer without oscillating)
271
- # average over content positions (skip puzzle embedding prefix positions)
272
- # CRITICAL: detach z_H_mean so aux_loss only trains error_estimator weights,
273
- # NOT the main reasoning layers. Without detach, aux_loss sends parasitic
274
- # gradients through z_H → H_level/L_level that conflict with lm_loss.
275
- z_H_mean = z_H[:, self.puzzle_emb_len:].mean(dim=1).detach() # (B, hidden_size)
276
- learned_err = torch.sigmoid(self.error_estimator(z_H_mean.float())) # (B, 1)
277
- learned_err = learned_err.squeeze(-1) # (B,)
278
-
279
- # SHREK: combined error = 50/50 blend of both signals
280
- # flip_err works immediately from step 1 (no learning needed)
281
- # learned_err becomes accurate over training and takes over as the stronger signal
282
- error = 0.5 * flip_err + 0.5 * learned_err # (B,)
283
-
284
- # Q head: same as original HRM — reads CLS token (position 0) before error injection.
285
- # This keeps Q-learning stable by matching HRM's gradient structure exactly.
286
- q_logits = self.q_head(z_H[:, 0].to(torch.float32)).to(torch.float32) # (B, 2)
287
-
288
- # SHREK: inject combined error into z_H (AFTER Q-head, only affects carry for next step)
289
- # error_encoder maps scalar -> hidden_size vector
290
- # alpha follows linear warmup: 0 → alpha_max over warmup steps
291
- # scaled by 1/sqrt(hidden_size) so injection is proportional to model size
292
- error_emb = self.error_encoder(error.unsqueeze(-1)) # (B, hidden_size)
293
- # SHREK: compute alpha from warmup schedule (not learned)
294
- # During warmup, alpha ramps linearly from 0 to alpha_max.
295
- # After warmup, alpha stays at alpha_max.
296
- # Uses torch.clamp instead of Python min() to stay compatible with torch.compile.
297
- with torch.no_grad():
298
- if self.training:
299
- self._alpha_step += 1
300
- alpha = self.config.alpha_max * torch.clamp(self._alpha_step / self.config.alpha_warmup_steps, max=1.0)
301
- scale = math.sqrt(self.config.hidden_size)
302
- z_H = z_H + (alpha * error_emb.unsqueeze(1) / scale).to(z_H.dtype) # (B, seq_len, hidden_size)
303
-
304
- # New carry: store error-injected z_H so next ACT step starts from it
305
- new_carry = HierarchicalReasoningModel_ACTV1InnerCarry(
306
- z_H=z_H.detach(),
307
- z_L=z_L.detach(),
308
- # SHREK: store current predictions — next step compares against these for flip rate
309
- prev_pred=current_pred.detach(),
310
- # SHREK: Q-values written to carry for torch.compile compatibility (not read for Q-targets)
311
- prev_q_halt=q_logits[..., 0].detach(),
312
- prev_q_continue=q_logits[..., 1].detach(),
313
- )
314
-
315
- # SHREK: also return learned_err so pretrain.py can compute auxiliary loss
316
- # auxiliary loss trains the estimator: predicted error should match real lm_loss
317
- if require_trace:
318
- return z_H_trace, new_carry, output, (q_logits[..., 0], q_logits[..., 1]), learned_err
319
- else:
320
- return new_carry, output, (q_logits[..., 0], q_logits[..., 1]), learned_err
321
-
322
-
323
- class HierarchicalReasoningModel_ACTV1(nn.Module):
324
- """ACT wrapper."""
325
-
326
- def __init__(self, config_dict: dict):
327
- super().__init__()
328
- self.config = HierarchicalReasoningModel_ACTV1Config(**config_dict)
329
- self.inner = HierarchicalReasoningModel_ACTV1_Inner(self.config)
330
-
331
- @property
332
- def puzzle_emb(self):
333
- return self.inner.puzzle_emb
334
-
335
- def initial_carry(self, batch: Dict[str, torch.Tensor]):
336
- batch_size = batch["inputs"].shape[0]
337
-
338
- return HierarchicalReasoningModel_ACTV1Carry(
339
- inner_carry=self.inner.empty_carry(batch_size), # Empty is expected, it will be reseted in first pass as all sequences are halted.
340
-
341
- steps=torch.zeros((batch_size, ), dtype=torch.int32),
342
- halted=torch.ones((batch_size, ), dtype=torch.bool), # Default to halted
343
-
344
- current_data={k: torch.empty_like(v) for k, v in batch.items()}
345
- )
346
-
347
- # SHREK: removed task_type parameter — error signal is universal, no task rules needed
348
- def forward(self, carry: HierarchicalReasoningModel_ACTV1Carry, batch: Dict[str, torch.Tensor], require_trace=False):
349
- # -> Tuple[HierarchicalReasoningModel_ACTV1Carry, Dict[str, torch.Tensor], torch.Tensor]:
350
- # Update data, carry (removing halted sequences)
351
- new_inner_carry = self.inner.reset_carry(carry.halted, carry.inner_carry)
352
-
353
- new_steps = torch.where(carry.halted, 0, carry.steps)
354
-
355
- new_current_data = {k: torch.where(carry.halted.view((-1, ) + (1, ) * (batch[k].ndim - 1)), batch[k], v) for k, v in carry.current_data.items()}
356
-
357
- # SHREK: run inner forward — unpack learned_err for auxiliary loss in pretrain.py
358
- if require_trace:
359
- z_H_trace, new_inner_carry, logits, (q_halt_logits, q_continue_logits), learned_err = self.inner(new_inner_carry, new_current_data, require_trace=require_trace)
360
- else:
361
- new_inner_carry, logits, (q_halt_logits, q_continue_logits), learned_err = self.inner(new_inner_carry, new_current_data)
362
-
363
- outputs = {
364
- "logits": logits,
365
- "q_halt_logits": q_halt_logits,
366
- "q_continue_logits": q_continue_logits,
367
- "learned_err": learned_err, # SHREK: for auxiliary loss in pretrain.py
368
- }
369
-
370
- with torch.no_grad():
371
- # Step
372
- new_steps = new_steps + 1
373
- is_last_step = new_steps >= self.config.halt_max_steps
374
-
375
- halted = is_last_step
376
-
377
- # if training, and ACT is enabled
378
- if self.training and (self.config.halt_max_steps > 1):
379
- # Halt signal
380
- # NOTE: During evaluation, always use max steps, this is to guarantee the same halting steps inside a batch for batching purposes
381
- halted = halted | (q_halt_logits > q_continue_logits)
382
-
383
- # Exploration
384
- min_halt_steps = (torch.rand_like(q_halt_logits) < self.config.halt_exploration_prob) * torch.randint_like(new_steps, low=2, high=self.config.halt_max_steps + 1)
385
-
386
- halted = halted & (new_steps >= min_halt_steps)
387
-
388
- # SHREK: Q-target via double forward pass (same as original HRM).
389
- # Run inner() a second time to get NEXT step's Q-values (step T+1).
390
- # Save/restore _alpha_step so the second call doesn't double-count
391
- # the warmup — without this, alpha reaches full strength at step ~2500
392
- # instead of 5000, destabilizing early training.
393
- # NOTE: prev_q fields in carry are not read here — they must stay
394
- # in the dataclass for torch.compile compatibility.
395
- saved_alpha_step = self.inner._alpha_step.clone()
396
- next_q_halt_logits, next_q_continue_logits = self.inner(new_inner_carry, new_current_data)[-2]
397
- self.inner._alpha_step.copy_(saved_alpha_step)
398
-
399
- outputs["target_q_continue"] = torch.sigmoid(
400
- torch.where(is_last_step,
401
- next_q_halt_logits,
402
- torch.maximum(next_q_halt_logits, next_q_continue_logits))
403
- )
404
-
405
- if require_trace:
406
- return HierarchicalReasoningModel_ACTV1Carry(new_inner_carry, new_steps, halted, new_current_data), outputs, new_steps, (q_halt_logits > q_continue_logits), z_H_trace
407
- else:
408
- return HierarchicalReasoningModel_ACTV1Carry(new_inner_carry, new_steps, halted, new_current_data), outputs, new_steps, (q_halt_logits > q_continue_logits)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sudoku-extreme/shrek-large/losses.py DELETED
@@ -1,130 +0,0 @@
1
- from typing import Any, Tuple, Dict, Sequence, Optional
2
-
3
- import torch
4
- import torch.nn.functional as F
5
- from torch import nn
6
-
7
-
8
- IGNORE_LABEL_ID = -100
9
-
10
-
11
- def s(x, epsilon=1e-30):
12
- return torch.where(
13
- x<0,
14
- 1/(1-x+ epsilon),
15
- x + 1
16
- )
17
-
18
-
19
- def log_stablemax(x, dim=-1):
20
- s_x = s(x)
21
- return torch.log(s_x/torch.sum(s_x, dim=dim, keepdim=True))
22
-
23
-
24
- def stablemax_cross_entropy(logits, labels, ignore_index: int = -100):
25
- logprobs = log_stablemax(logits.to(torch.float64), dim=-1)
26
-
27
- valid_mask = labels != ignore_index
28
- transformed_labels = torch.where(valid_mask, labels, 0)
29
- prediction_logprobs = torch.gather(logprobs, index=transformed_labels.to(torch.long).unsqueeze(-1), dim=-1).squeeze(-1)
30
-
31
- return -torch.where(valid_mask, prediction_logprobs, 0)
32
-
33
-
34
- def softmax_cross_entropy(logits, labels, ignore_index: int = -100):
35
- # Cast logits to f32
36
- # Flatten logits
37
- return F.cross_entropy(logits.to(torch.float32).view(-1, logits.shape[-1]), labels.to(torch.long).view(-1), ignore_index=ignore_index, reduction="none").view(labels.shape)
38
-
39
-
40
- class ACTLossHead(nn.Module):
41
- def __init__(self, model: nn.Module, loss_type: str):
42
- super().__init__()
43
- self.model = model
44
- self.loss_fn = globals()[loss_type]
45
-
46
- def initial_carry(self, *args, **kwargs):
47
- return self.model.initial_carry(*args, **kwargs) # type: ignore
48
-
49
- def forward(
50
- self,
51
- return_keys: Sequence[str],
52
- require_trace=False,
53
- # Model args
54
- **model_kwargs,
55
- ) -> Tuple[Any, torch.Tensor, Dict[str, torch.Tensor], Optional[Dict[str, torch.Tensor]], torch.Tensor]:
56
- # Model logits
57
- # B x SeqLen x D
58
- if require_trace:
59
- new_carry, outputs, steps, act_halt, z_H_trace = self.model(**model_kwargs, require_trace=require_trace)
60
- else:
61
- new_carry, outputs, steps, act_halt = self.model(**model_kwargs)
62
- labels = new_carry.current_data["labels"]
63
- alpha = 1.0
64
- ds_mask = (alpha ** (16-steps.detach())).unsqueeze(dim=1)
65
- # print(ds_mask.shape)
66
-
67
- # Correctness
68
- with torch.no_grad():
69
- mask = labels != IGNORE_LABEL_ID
70
- loss_counts = mask.sum(-1)
71
- loss_divisor = loss_counts.clamp_min(1).unsqueeze(-1) # Avoid NaNs in division
72
-
73
- is_correct = mask & (torch.argmax(outputs["logits"], dim=-1) == labels)
74
- seq_is_correct = is_correct.sum(-1) == loss_counts
75
-
76
- # # Metrics (halted)
77
- # valid_metrics = new_carry.halted & (loss_counts > 0)
78
- valid_metrics = (loss_counts > 0)
79
-
80
- metrics = {
81
- "count": valid_metrics.sum(),
82
-
83
- "accuracy": torch.where(valid_metrics, (is_correct.to(torch.float32) / loss_divisor).sum(-1), 0).sum(),
84
- "exact_accuracy": (valid_metrics & seq_is_correct).sum(),
85
-
86
- "q_halt_accuracy": (valid_metrics & ((outputs["q_halt_logits"] >= 0) == seq_is_correct)).sum(),
87
- "steps": torch.where(valid_metrics, new_carry.steps, 0).sum(),
88
- }
89
-
90
- # Losses
91
- # FIXME: Assuming the batch is always full
92
- loss = self.loss_fn(outputs["logits"], labels, ignore_index=IGNORE_LABEL_ID)
93
- # print(loss.shape)
94
- lm_loss = (ds_mask * loss / loss_divisor).sum()
95
- q_halt_loss = F.binary_cross_entropy_with_logits(outputs["q_halt_logits"], seq_is_correct.to(outputs["q_halt_logits"].dtype), reduction="sum")
96
-
97
- metrics.update({
98
- "lm_loss": lm_loss.detach(),
99
- "q_halt_loss": q_halt_loss.detach(),
100
- })
101
-
102
- # Q continue (bootstrapping target loss)
103
- q_continue_loss = 0
104
- if "target_q_continue" in outputs:
105
- q_continue_loss = F.binary_cross_entropy_with_logits(outputs["q_continue_logits"], outputs["target_q_continue"], reduction="sum")
106
-
107
- metrics["q_continue_loss"] = q_continue_loss.detach()
108
-
109
- # SHREK: auxiliary loss — trains error_estimator to predict per-sample lm_loss.
110
- # Computed here (not in pretrain.py) so learned_err still has gradients.
111
- # per_sample_lm_loss is detached so only error_estimator weights get updated by aux_loss.
112
- aux_loss = 0
113
- if "learned_err" in outputs:
114
- per_sample_lm_loss = (ds_mask * loss / loss_divisor).sum(dim=1).detach() # (B,)
115
- # SHREK: min-max normalize target to 0-1 so it matches sigmoid output range.
116
- # When all losses are equal (early training), target = 0 (no distinguishable error).
117
- # When losses vary, target spreads across 0-1 giving the estimator a useful signal.
118
- lm_min = per_sample_lm_loss.min()
119
- lm_max = per_sample_lm_loss.max()
120
- target = (per_sample_lm_loss - lm_min) / (lm_max - lm_min + 1e-6) # (B,) in [0, 1]
121
- aux_loss = F.mse_loss(outputs["learned_err"], target, reduction="sum")
122
- metrics["aux_loss"] = aux_loss.detach()
123
-
124
- # Filter outputs for return
125
- detached_outputs = {k: outputs[k].detach() for k in return_keys if k in outputs}
126
-
127
- if require_trace:
128
- return z_H_trace, new_carry, lm_loss + 0.5 * (q_halt_loss + q_continue_loss) + 0.1 * aux_loss, metrics, detached_outputs, new_carry.halted.all(), new_carry.halted & act_halt
129
- else:
130
- return new_carry, lm_loss + 0.5 * (q_halt_loss + q_continue_loss) + 0.1 * aux_loss, metrics, detached_outputs, new_carry.halted.all(), new_carry.halted & act_halt
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sudoku-extreme/shrek-large/step_10416 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:74b36a6f4b570bbaded31bbfa85a751d4cfced3d674f7f6aad41f0c3e7281e06
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_11718 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:55825da0748b97e990279bae33398574939972c0062db8470f8989c69cf04b8d
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_1302 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:aac22b0422bf84888516502f9b48b73c57a4bd97ab33f606fc61f32387985798
3
- size 109132227
 
 
 
 
sudoku-extreme/shrek-large/step_13020 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5dab3287dafeb6fb2d6fbdd67f73b8d43f3f66427cc7e749974d19256ebeead1
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_14322 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:68774c4bce52069704dd2188c41a7f57bf7b67c167905932e191876be6db61f4
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_15624 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f12694ccb5e0840853c140b4c941c2d260be6dc95b637ac6fc054fe62177e3ef
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_16926 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:358dcaf758b9bf551c2f18b3931b43f07b9666fdce67cc57b5d208ca4f3a1dc4
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_18228 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8d709311a7e713e917d298464b3fafb94e679160b9dbafa61159f6a659448a74
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_19530 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d55f41e0866a5469f153677bdececc388cf59400e2055bf20ccabdcae8382536
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_20832 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:fdbd44a333a254b49459873037306014a88853b1e52e21fa2b16ebb29b6e17ff
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_22134 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:2e58c4b292cdf38b641ce5d78a24e8d0d1f048e5e675a9804f61e784a3a22ab7
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_23436 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f560eddacb2d743af120eae9ec09287cbb15482c58dd5f01a0d426c7146a2553
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_24738 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:416000213407950cc69e44306aa845568a94b37a95efba0e5303d54fb2bb9b97
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_2604 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f99d4f85a0d6f0a312bfc388b8b437fe3bf14e7d2f743596aa012e78b340916a
3
- size 109132227
 
 
 
 
sudoku-extreme/shrek-large/step_26040 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:47b0c85965ba2d960c9b76ba380851fe9ae0410c887075f06ae4fd1a268fbc81
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_27342 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:371992db5eb4e18e3b1fef732bc36e65ed7b7ee6315462fbb00938fa7f58dcd2
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_28644 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:299e5f41f22e538d7bfa61f24bef9bdaa86b9f5800f1f3817cd32932c9d93ef7
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_29946 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0f9f3a45e01784ff932e76b96c180d25fb1abfa561f8d829a2967064eb9388b3
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_31248 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1cf47014e8db4cb90e6e04c150f72983bb4cc7691e9c223c14ca825e79429c28
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_32550 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6d908f372029254996bc0a75c356ba3b2d00dc2eaae950c6be705917cad87d50
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_33852 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:32642c7583f090841aee80256befd5b6461e016a8697ca39c8c575d10db7e8e4
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_35154 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3f941bc59925dd5579e1c47f58d1985054e22990ea00ee1b289074fc1b568da8
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_36456 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d3aec656acec7c5b2e7648600534e8c9c0e106b34512cffd814a27a23050f8b2
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_37758 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d612abea7607f2d54a985d5ab58f90036b42a5578f038be75e1a1ed9ad52c20a
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_3906 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4b1053e271e46cf3617a3844e9136bcd62cca0b3ab78f148f1cf9e32f2147211
3
- size 109132227
 
 
 
 
sudoku-extreme/shrek-large/step_39060 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8c783bf56cc85a49c686b5d41e573b70b8096f2d9466d8f04cdd72939780d2d8
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_40362 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7d683608de2df52cea12ce4c3288c79135a4bd2fcba74d6f23c8bb8c1e8c302e
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_41664 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:cb74a1ecd4d13a3bf28df8019c2d785eeae1f0d612b6cb15f1466b927fc06389
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_42966 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:89351f04250e2ee624298ba9db46ef9413b9d408422ba95a473169f3ea4b72c8
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_44268 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e3d796bf6154c97978fc5cbe013c7ac7e8ef343411a538435571ba9aa9ed33c3
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_45570 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0ee7be2bbc997ccca722300bb09db40c99730ce833f3fa2f3b407ff9fe54bf19
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_46872 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e1b7268bd8ae1a97f3e8d3bdb8a846da6dfcecaae988ea1e1c5c14bce1e2b22d
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_48174 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:eadff3e342ff8c5f72088baf946ee5304f59af517621e73ba28e590890cea1f0
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_49476 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:310706de78eff0a63bdd1a361963c4b1d584c04647f30b8c6fb63e7dbae47c08
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_50778 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6b6b8c5192e6850662502b9175e8572fd926b88996a67df608e55c6d863ec6d0
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_5208 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9404c2300474a16496639cf7b73eb2e0f852f90c00473bf3be45cd6e4a31e0a9
3
- size 109132227
 
 
 
 
sudoku-extreme/shrek-large/step_52080 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c0f83e31f1a8f2574214fabdc91db7ab160b30623fc2916d7c76e2cdc4888f87
3
- size 109132277
 
 
 
 
sudoku-extreme/shrek-large/step_6510 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ab5f7e6aa9744cac59925e9f992568f5acfed44d448d7b7b4b31d5e842423ac1
3
- size 109132227
 
 
 
 
sudoku-extreme/shrek-large/step_7812 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:64ba5b1260184728779e3ec8ba6ce05f918a805a2087dd1526f60f058922e0b7
3
- size 109132227
 
 
 
 
sudoku-extreme/shrek-large/step_9114 DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c93a8ccf74ea81f88156c681f03aadee082fd3c3e7b227316a17c7c8b0413511
3
- size 109132227