Ayaka157 commited on
Commit
315f76d
·
verified ·
1 Parent(s): 7aeed23

Sync modeling_retnet.py with the audited mainline (20261024_Submit_pre-audit, verify.py PASS). API surface unchanged: RetNetHFConfig / RetNetForCausalLM both present.

Browse files
Files changed (1) hide show
  1. modeling_retnet.py +128 -38
modeling_retnet.py CHANGED
@@ -20,11 +20,28 @@ import torch.nn.functional as F
20
  from transformers import PreTrainedModel, PretrainedConfig
21
 
22
  #: Opt-in batched-chunk chunkwise retention (see ``retention_fast.py``).
23
- #: Mathematically identical to the reference per-chunk loop but replaces the
24
- #: Python loop's per-chunk slicing + small matmuls with 3 batched matmuls and a
25
- #: tiny state scan. Verified equivalent by ``scripts/test_fast_chunkwise.py``.
26
  FAST_CHUNKWISE = os.environ.get("RETNET_FAST_CHUNKWISE", "0") == "1"
27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  # NOTE(2026-10-10): 试过给 ``MultiScaleRetention._rotations`` 加一层
29
  # ``(length, offset, device, dtype)`` 键的 sin/cos 表缓存,理由是 profile 里
30
  # ``triton_poi_fused__to_copy*`` 合计 42.9 ms(占一个优化步 5.9%),而这 20 层
@@ -41,7 +58,8 @@ class RetNetHFConfig(PretrainedConfig):
41
  value_dim=None, ffn_dim=None, dropout=0.1, norm_eps=1e-5,
42
  tie_weights=True, embed_scale=None, prompt_loss_weight=0.1,
43
  use_gla=False, use_swiglu=False, use_growth_ln=False,
44
- growth_old_dim=None, checkpoint_layers=-1, **kwargs):
 
45
  super().__init__(**kwargs)
46
  self.vocab_size = vocab_size
47
  self.d_model = d_model
@@ -54,6 +72,9 @@ class RetNetHFConfig(PretrainedConfig):
54
  self.tie_weights = tie_weights
55
  self.embed_scale = embed_scale
56
  self.prompt_loss_weight = prompt_loss_weight
 
 
 
57
  self.use_gla = use_gla
58
  self.use_swiglu = use_swiglu
59
  self.use_growth_ln = use_growth_ln
@@ -174,14 +195,18 @@ class MultiScaleRetention(nn.Module):
174
  exp = delta.masked_fill(~causal, 0)
175
  mask = self.gamma.to(device=device, dtype=torch.float32)[:, None, None] ** exp
176
  mask = mask.masked_fill(~causal, 0.0)
177
- mask = mask / mask.sum(dim=-1, keepdim=True).sqrt().clamp_min(1e-8)
 
 
 
178
  return mask.to(dtype=dtype)
179
 
180
  def _parallel_retention(self, q, k, v):
181
  bsz, _, tgt_len, _ = q.shape
182
  mask = self._decay_mask(tgt_len, q.device, q.dtype)
183
  scores = torch.matmul(q, k.transpose(-1, -2)) * mask
184
- scores = scores / scores.detach().abs().sum(dim=-1, keepdim=True).clamp_min(1.0)
 
185
  return torch.matmul(scores, v)
186
 
187
  def _parallel_final_state(self, k, v, state):
@@ -200,8 +225,15 @@ class MultiScaleRetention(nn.Module):
200
  if state is None:
201
  state = torch.zeros(bsz, self.num_heads, key_dim, value_head_dim,
202
  device=q.device, dtype=q.dtype)
203
- outputs = []
204
  gamma = self.gamma.to(device=q.device, dtype=q.dtype)
 
 
 
 
 
 
 
 
205
  for t in range(tgt_len):
206
  state = gamma.view(1, -1, 1, 1) * state + \
207
  k[:, :, t, :].unsqueeze(-1) * v[:, :, t, :].unsqueeze(-2)
@@ -359,6 +391,59 @@ class RetNetBlock(nn.Module):
359
  return self._block_forward(x, mode, state, offset, return_state, chunk_size)
360
 
361
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
362
  class RetNetForCausalLM(PreTrainedModel):
363
  """RetNet language model with a from-scratch token embedding.
364
 
@@ -432,7 +517,15 @@ class RetNetForCausalLM(PreTrainedModel):
432
  return n
433
 
434
  def _forward_hidden(self, input_ids, mode="parallel", states=None, offset=0,
435
- return_state=False, chunk_size=64, return_hidden=False):
 
 
 
 
 
 
 
 
436
  if input_ids.ndim != 2:
437
  raise ValueError("input_ids must be [batch, seq]")
438
  x = self.embedding(input_ids) * self.embed_scale
@@ -441,9 +534,12 @@ class RetNetForCausalLM(PreTrainedModel):
441
  states = [None] * len(self.blocks)
442
  if len(states) != len(self.blocks):
443
  raise ValueError("states must contain one entry per block")
 
 
 
444
  new_states: List[Optional[torch.Tensor]] = []
445
  for block, block_state in zip(self.blocks, states):
446
- if return_state or (mode == "recurrent" and block_state is not None):
447
  x, ns = block(x, mode=mode, state=block_state, offset=offset,
448
  return_state=True, chunk_size=chunk_size)
449
  new_states.append(ns)
@@ -455,11 +551,11 @@ class RetNetForCausalLM(PreTrainedModel):
455
  if return_hidden:
456
  # 融合交叉熵需要的是 LM head 的**输入**(hidden),不是输出。
457
  # 这里提前返回可以完全不物化 [B,L,V]。
458
- if return_state or mode == "recurrent":
459
  return x, new_states
460
  return x
461
  logits = self.lm_head(x)
462
- if return_state or mode == "recurrent":
463
  return logits, new_states
464
  return logits
465
 
@@ -486,17 +582,24 @@ class RetNetForCausalLM(PreTrainedModel):
486
  if mode is None:
487
  mode = getattr(self.config, "mode", "parallel")
488
  _fused = (os.environ.get("RETNET_FUSED_CE") == "1"
489
- and labels is not None and wcodes is not None)
 
490
  logits = None
491
  # `loss` 必须在所有分支之前初始化:不传 labels 时(普通推理、prefill、
492
  # 以及 HF 的 generate 路径)上面几个分支都不会赋值,line 536 的
493
  # CausalLMOutput(loss=loss, ...) 会直接 UnboundLocalError。
494
  loss = None
495
  if _fused:
496
- # 融合路径:LM head 的输入直接进 fused CE,完全不物化 [B,L,V]
497
- hidden, new_states = self._forward_hidden(
 
 
498
  input_ids, mode=mode, states=states, offset=offset,
499
- return_state=True, chunk_size=chunk_size, return_hidden=True)
 
 
 
 
500
  B_, L_, D_ = hidden.shape
501
  w = torch.zeros_like(wcodes, dtype=torch.float32)
502
  w = torch.where(wcodes == 1, torch.ones_like(w), w)
@@ -515,29 +618,16 @@ class RetNetForCausalLM(PreTrainedModel):
515
  logits = self._forward_hidden(input_ids, mode=mode, states=states, offset=offset)
516
  new_states = None
517
 
518
- if not _fused:
519
- # 非融合路径:沿用原来的显式 fp32 上采样。
520
- # NOTE(2026-10-10): 试过去掉 ``logits.float()``(让 ``F.cross_entropy``
521
- # 直接吃 bf16)。loss 逐位一致但**步时 0% 变化**——profile 把这两个
522
- # 融合核记成 3.9%,是归因假象。故保留原状。
523
- if labels is not None and wcodes is not None:
524
- logits_f = logits.float()
525
- token_loss = F.cross_entropy(
526
- logits_f.reshape(-1, logits_f.shape[-1]), labels.reshape(-1),
527
- reduction="none")
528
- w = torch.zeros_like(wcodes, dtype=torch.float32)
529
- w = torch.where(wcodes == 1, torch.ones_like(w), w)
530
- w = torch.where(wcodes == 2,
531
- torch.full_like(w, self.config.prompt_loss_weight), w)
532
- w = torch.where(wcodes == 3, torch.ones_like(w), w)
533
- loss = (token_loss * w.reshape(-1)).sum() / w.sum().clamp_min(1.0)
534
- elif labels is not None:
535
- logits_f = logits.float()
536
- loss = F.cross_entropy(logits_f.reshape(-1, logits_f.shape[-1]),
537
- labels.reshape(-1))
538
-
539
- from transformers.modeling_outputs import CausalLMOutput
540
- return CausalLMOutput(loss=loss, logits=logits)
541
 
542
  @torch.inference_mode()
543
  def generate(self, input_ids: torch.Tensor, max_new_tokens: int = 32,
 
20
  from transformers import PreTrainedModel, PretrainedConfig
21
 
22
  #: Opt-in batched-chunk chunkwise retention (see ``retention_fast.py``).
23
+ #: Mathematically the same function as the reference per-chunk loop (5 batched
24
+ #: matmuls instead of a Python loop of small ones), equal up to floating-point
25
+ #: rounding **not bitwise**. Verified by ``tests/test_fast_chunkwise.py``.
26
  FAST_CHUNKWISE = os.environ.get("RETNET_FAST_CHUNKWISE", "0") == "1"
27
 
28
+ #: 每个交叉熵分块允许物化的 fp32 logits 切片字节数(默认 64 MB)。
29
+ #: 旧实现把 chunk 写成一个固定的 **token 数**(262144),于是分块大小与词表
30
+ #: 大小无关、且大于生产形状的 token 总数 —— B=16/L=2048/V=20000 时只有一块,
31
+ #: 仍然是 32768×20000×4 ≈ 2.6 GB 的整段拷贝,E22 声称消除的那一份原封不动。
32
+ #: 现在按字节预算反推 token 数:V 越大每 token 越贵,块就越小。
33
+ DEFAULT_CE_CHUNK_BYTES = 64 << 20
34
+
35
+
36
+ def ce_chunk_tokens(vocab_size: int, chunk_bytes: int = DEFAULT_CE_CHUNK_BYTES) -> int:
37
+ """按字节预算反推每个 CE 分块能放多少 token(至少 1)。
38
+
39
+ 单独成函数是为了让测试能直接断言“生产形状下必须切出多块”,而不是只
40
+ 验证数值等价 —— 旧测试传 ``chunk_tokens=7``,即使参数被无视也会通过。
41
+ """
42
+ per_token = 4 * max(1, int(vocab_size)) # fp32 logits,每 token V 个 float
43
+ return max(1, int(chunk_bytes) // per_token)
44
+
45
  # NOTE(2026-10-10): 试过给 ``MultiScaleRetention._rotations`` 加一层
46
  # ``(length, offset, device, dtype)`` 键的 sin/cos 表缓存,理由是 profile 里
47
  # ``triton_poi_fused__to_copy*`` 合计 42.9 ms(占一个优化步 5.9%),而这 20 层
 
58
  value_dim=None, ffn_dim=None, dropout=0.1, norm_eps=1e-5,
59
  tie_weights=True, embed_scale=None, prompt_loss_weight=0.1,
60
  use_gla=False, use_swiglu=False, use_growth_ln=False,
61
+ growth_old_dim=None, checkpoint_layers=-1,
62
+ ce_chunk_bytes=DEFAULT_CE_CHUNK_BYTES, **kwargs):
63
  super().__init__(**kwargs)
64
  self.vocab_size = vocab_size
65
  self.d_model = d_model
 
72
  self.tie_weights = tie_weights
73
  self.embed_scale = embed_scale
74
  self.prompt_loss_weight = prompt_loss_weight
75
+ #: 交叉熵分块的字节预算(E22)。按 V 反推 token 数,随 config 落盘,
76
+ #: 换词表/换卡时不必改代码。<=0 表示退回“整段不分块”(仅用于对照实验)。
77
+ self.ce_chunk_bytes = int(ce_chunk_bytes)
78
  self.use_gla = use_gla
79
  self.use_swiglu = use_swiglu
80
  self.use_growth_ln = use_growth_ln
 
195
  exp = delta.masked_fill(~causal, 0)
196
  mask = self.gamma.to(device=device, dtype=torch.float32)[:, None, None] ** exp
197
  mask = mask.masked_fill(~causal, 0.0)
198
+ # 2026-10-20: 删除了原来的 `mask / mask.sum(...).sqrt()`。
199
+ # 那一步只存在于 parallel 路径,chunkwise/recurrent 都没有,导致
200
+ # 训练(chunkwise)与生成 prefill(parallel)算的不是同一个函数。
201
+ # 见 docs/00 E16。
202
  return mask.to(dtype=dtype)
203
 
204
  def _parallel_retention(self, q, k, v):
205
  bsz, _, tgt_len, _ = q.shape
206
  mask = self._decay_mask(tgt_len, q.device, q.dtype)
207
  scores = torch.matmul(q, k.transpose(-1, -2)) * mask
208
+ # 2026-10-20: 删除了原来的 `scores / scores.detach().abs().sum(...)`。
209
+ # 同一理由:这是 parallel 独有的第二次归一化,见 docs/00 E16。
210
  return torch.matmul(scores, v)
211
 
212
  def _parallel_final_state(self, k, v, state):
 
225
  if state is None:
226
  state = torch.zeros(bsz, self.num_heads, key_dim, value_head_dim,
227
  device=q.device, dtype=q.dtype)
 
228
  gamma = self.gamma.to(device=q.device, dtype=q.dtype)
229
+ # docs/00 E21:解码时 tgt_len 恒为 1,走快路径,省掉 Python 循环与
230
+ # list/stack 的物化。数学与下面的循环逐位相同。
231
+ if tgt_len == 1:
232
+ state = gamma.view(1, -1, 1, 1) * state + \
233
+ k[:, :, 0, :].unsqueeze(-1) * v[:, :, 0, :].unsqueeze(-2)
234
+ out = torch.sum(q[:, :, 0, :].unsqueeze(-1) * state, dim=-2).unsqueeze(2)
235
+ return out, state
236
+ outputs = []
237
  for t in range(tgt_len):
238
  state = gamma.view(1, -1, 1, 1) * state + \
239
  k[:, :, t, :].unsqueeze(-1) * v[:, :, t, :].unsqueeze(-2)
 
391
  return self._block_forward(x, mode, state, offset, return_state, chunk_size)
392
 
393
 
394
+ def _chunked_weighted_ce(logits, labels, wcodes=None, prompt_loss_weight=0.1,
395
+ chunk_tokens: int | None = None,
396
+ chunk_bytes: int = DEFAULT_CE_CHUNK_BYTES):
397
+ """分块计算加权交叉熵,避免把整个 [B,L,V] 转成 fp32(docs/00 E22)。
398
+
399
+ 旧实现是 ``logits_f = logits.float()`` 之后一次性 reshape 求 CE。
400
+ 对本项目(B=16, L=2048, V=20000)那一步会物化 16×2048×20000×4 ≈ 2.6 GB
401
+ 的 fp32 副本,是显存峰值的主要来源;而 logits 本身还是 bf16,已经占 1.3 GB。
402
+
403
+ 分块大小**由字节预算反推**(``ce_chunk_tokens``),不是固定 token 数:
404
+ 每块只把 [chunk, V] 转 fp32,峰值约为 ``chunk_bytes`` 的量级。默认 64 MB
405
+ 在 V=20000 时是 838 token/块,于是生产形状 B=16/L=2048 会切出 40 块,
406
+ 而不是旧实现的一块 2.6 GB。显式传 ``chunk_tokens`` 仍可覆盖(测试用)。
407
+
408
+ 权重码语义与旧实现完全一致:pad=0 不计、reply/plain=1、prompt=prompt_loss_weight,
409
+ 最后除以权重和;没有 wcodes 时退化为普通均值。
410
+ """
411
+ V = logits.shape[-1]
412
+ flat_logits = logits.reshape(-1, V)
413
+ flat_labels = labels.reshape(-1)
414
+ n = flat_logits.shape[0]
415
+ if chunk_tokens is None:
416
+ chunk_tokens = ce_chunk_tokens(V, chunk_bytes)
417
+ chunk_tokens = max(1, int(chunk_tokens))
418
+
419
+ w = None
420
+ if wcodes is not None:
421
+ w = torch.zeros_like(wcodes, dtype=torch.float32)
422
+ w = torch.where(wcodes == 1, torch.ones_like(w), w)
423
+ w = torch.where(wcodes == 2, torch.full_like(w, float(prompt_loss_weight)), w)
424
+ w = torch.where(wcodes == 3, torch.ones_like(w), w)
425
+ w = w.reshape(-1)
426
+
427
+ total = torch.zeros((), device=flat_logits.device, dtype=torch.float32)
428
+ denom = torch.zeros((), device=flat_logits.device, dtype=torch.float32)
429
+ for s in range(0, n, chunk_tokens):
430
+ e = min(s + chunk_tokens, n)
431
+ ce = F.cross_entropy(flat_logits[s:e].float(), flat_labels[s:e], reduction="none")
432
+ if w is None:
433
+ total = total + ce.sum()
434
+ denom = denom + float(e - s)
435
+ else:
436
+ total = total + (ce * w[s:e]).sum()
437
+ denom = denom + w[s:e].sum()
438
+ return total / denom.clamp_min(1.0)
439
+
440
+
441
+ def build_causal_lm_output(loss, logits):
442
+ """构造 HF 的 CausalLMOutput;单独成函数是为了让 forward 里只有一处返回。"""
443
+ from transformers.modeling_outputs import CausalLMOutput
444
+ return CausalLMOutput(loss=loss, logits=logits)
445
+
446
+
447
  class RetNetForCausalLM(PreTrainedModel):
448
  """RetNet language model with a from-scratch token embedding.
449
 
 
517
  return n
518
 
519
  def _forward_hidden(self, input_ids, mode="parallel", states=None, offset=0,
520
+ return_state=False, chunk_size=64, return_hidden=False,
521
+ need_state=None):
522
+ """跑完所有 block,返回 logits(或 ``return_hidden`` 时的 final_norm 输出)。
523
+
524
+ ``need_state`` 与 ``return_state`` 分开:融合交叉熵只想要 hidden,完全
525
+ 不需要每层的递归状态,但旧签名里 ``return_state=True`` 会让每个 block 都
526
+ 算一次 ``_parallel_final_state``(对 ``[B,H,L,kd,vd]`` 做 einsum)再把结果
527
+ 丢掉。显式传 ``need_state=False`` 就能跳掉这部分(E22/P2 性能项)。
528
+ """
529
  if input_ids.ndim != 2:
530
  raise ValueError("input_ids must be [batch, seq]")
531
  x = self.embedding(input_ids) * self.embed_scale
 
534
  states = [None] * len(self.blocks)
535
  if len(states) != len(self.blocks):
536
  raise ValueError("states must contain one entry per block")
537
+ # 递归模式必须返回状态(解码靠它推进);其余路径由调用方决定。
538
+ want_state = (return_state if need_state is None else bool(need_state)) \
539
+ or mode == "recurrent"
540
  new_states: List[Optional[torch.Tensor]] = []
541
  for block, block_state in zip(self.blocks, states):
542
+ if want_state:
543
  x, ns = block(x, mode=mode, state=block_state, offset=offset,
544
  return_state=True, chunk_size=chunk_size)
545
  new_states.append(ns)
 
551
  if return_hidden:
552
  # 融合交叉熵需要的是 LM head 的**输入**(hidden),不是输出。
553
  # 这里提前返回可以完全不物化 [B,L,V]。
554
+ if want_state:
555
  return x, new_states
556
  return x
557
  logits = self.lm_head(x)
558
+ if want_state:
559
  return logits, new_states
560
  return logits
561
 
 
582
  if mode is None:
583
  mode = getattr(self.config, "mode", "parallel")
584
  _fused = (os.environ.get("RETNET_FUSED_CE") == "1"
585
+ and labels is not None and wcodes is not None
586
+ and not return_state and mode != "recurrent")
587
  logits = None
588
  # `loss` 必须在所有分支之前初始化:不传 labels 时(普通推理、prefill、
589
  # 以及 HF 的 generate 路径)上面几个分支都不会赋值,line 536 的
590
  # CausalLMOutput(loss=loss, ...) 会直接 UnboundLocalError。
591
  loss = None
592
  if _fused:
593
+ # 融合路径:LM head 的输入直接进 fused CE,完全不物化 [B,L,V]。
594
+ # need_state=False:这段只训练用,不需要每层的递归状态,
595
+ # 旧写法传 return_state=True 会让每个 block 白算一次最终状态。
596
+ hidden = self._forward_hidden(
597
  input_ids, mode=mode, states=states, offset=offset,
598
+ return_state=False, chunk_size=chunk_size, return_hidden=True,
599
+ need_state=False)
600
+ if isinstance(hidden, tuple): # recurrent 强制要状态
601
+ hidden = hidden[0]
602
+ new_states = None
603
  B_, L_, D_ = hidden.shape
604
  w = torch.zeros_like(wcodes, dtype=torch.float32)
605
  w = torch.where(wcodes == 1, torch.ones_like(w), w)
 
618
  logits = self._forward_hidden(input_ids, mode=mode, states=states, offset=offset)
619
  new_states = None
620
 
621
+ if not _fused and labels is not None:
622
+ # docs/00 E22:不再对整段 logits 做 .float()(B=16,L=2048,V=20000 时
623
+ # 那是约 2.6 GB 的 fp32 副本),改为按**字节预算**分块求 CE。
624
+ # 数值与旧实现一致:同样的权重码、同样的“除以权重和”。
625
+ loss = _chunked_weighted_ce(
626
+ logits, labels, wcodes, self.config.prompt_loss_weight,
627
+ chunk_bytes=int(getattr(self.config, "ce_chunk_bytes",
628
+ DEFAULT_CE_CHUNK_BYTES) or DEFAULT_CE_CHUNK_BYTES))
629
+
630
+ return build_causal_lm_output(loss, logits)
 
 
 
 
 
 
 
 
 
 
 
 
 
631
 
632
  @torch.inference_mode()
633
  def generate(self, input_ids: torch.Tensor, max_new_tokens: int = 32,