Text Generation
Transformers
Safetensors
Chinese
English
retnet_hf
retnet
retention
from-scratch
custom_code
Instructions to use Ayaka157/RetNet-0.5B-ep2 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Ayaka157/RetNet-0.5B-ep2 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="Ayaka157/RetNet-0.5B-ep2", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("Ayaka157/RetNet-0.5B-ep2", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Ayaka157/RetNet-0.5B-ep2 with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Ayaka157/RetNet-0.5B-ep2" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Ayaka157/RetNet-0.5B-ep2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/Ayaka157/RetNet-0.5B-ep2
- SGLang
How to use Ayaka157/RetNet-0.5B-ep2 with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Ayaka157/RetNet-0.5B-ep2" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Ayaka157/RetNet-0.5B-ep2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Ayaka157/RetNet-0.5B-ep2" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Ayaka157/RetNet-0.5B-ep2", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use Ayaka157/RetNet-0.5B-ep2 with Docker Model Runner:
docker model run hf.co/Ayaka157/RetNet-0.5B-ep2
Sync modeling_retnet.py with the audited mainline (20261024_Submit_pre-audit, verify.py PASS). API surface unchanged: RetNetHFConfig / RetNetForCausalLM both present.
Browse files- modeling_retnet.py +128 -38
modeling_retnet.py
CHANGED
|
@@ -20,11 +20,28 @@ import torch.nn.functional as F
|
|
| 20 |
from transformers import PreTrainedModel, PretrainedConfig
|
| 21 |
|
| 22 |
#: Opt-in batched-chunk chunkwise retention (see ``retention_fast.py``).
|
| 23 |
-
#: Mathematically
|
| 24 |
-
#:
|
| 25 |
-
#:
|
| 26 |
FAST_CHUNKWISE = os.environ.get("RETNET_FAST_CHUNKWISE", "0") == "1"
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
# NOTE(2026-10-10): 试过给 ``MultiScaleRetention._rotations`` 加一层
|
| 29 |
# ``(length, offset, device, dtype)`` 键的 sin/cos 表缓存,理由是 profile 里
|
| 30 |
# ``triton_poi_fused__to_copy*`` 合计 42.9 ms(占一个优化步 5.9%),而这 20 层
|
|
@@ -41,7 +58,8 @@ class RetNetHFConfig(PretrainedConfig):
|
|
| 41 |
value_dim=None, ffn_dim=None, dropout=0.1, norm_eps=1e-5,
|
| 42 |
tie_weights=True, embed_scale=None, prompt_loss_weight=0.1,
|
| 43 |
use_gla=False, use_swiglu=False, use_growth_ln=False,
|
| 44 |
-
growth_old_dim=None, checkpoint_layers=-1,
|
|
|
|
| 45 |
super().__init__(**kwargs)
|
| 46 |
self.vocab_size = vocab_size
|
| 47 |
self.d_model = d_model
|
|
@@ -54,6 +72,9 @@ class RetNetHFConfig(PretrainedConfig):
|
|
| 54 |
self.tie_weights = tie_weights
|
| 55 |
self.embed_scale = embed_scale
|
| 56 |
self.prompt_loss_weight = prompt_loss_weight
|
|
|
|
|
|
|
|
|
|
| 57 |
self.use_gla = use_gla
|
| 58 |
self.use_swiglu = use_swiglu
|
| 59 |
self.use_growth_ln = use_growth_ln
|
|
@@ -174,14 +195,18 @@ class MultiScaleRetention(nn.Module):
|
|
| 174 |
exp = delta.masked_fill(~causal, 0)
|
| 175 |
mask = self.gamma.to(device=device, dtype=torch.float32)[:, None, None] ** exp
|
| 176 |
mask = mask.masked_fill(~causal, 0.0)
|
| 177 |
-
|
|
|
|
|
|
|
|
|
|
| 178 |
return mask.to(dtype=dtype)
|
| 179 |
|
| 180 |
def _parallel_retention(self, q, k, v):
|
| 181 |
bsz, _, tgt_len, _ = q.shape
|
| 182 |
mask = self._decay_mask(tgt_len, q.device, q.dtype)
|
| 183 |
scores = torch.matmul(q, k.transpose(-1, -2)) * mask
|
| 184 |
-
|
|
|
|
| 185 |
return torch.matmul(scores, v)
|
| 186 |
|
| 187 |
def _parallel_final_state(self, k, v, state):
|
|
@@ -200,8 +225,15 @@ class MultiScaleRetention(nn.Module):
|
|
| 200 |
if state is None:
|
| 201 |
state = torch.zeros(bsz, self.num_heads, key_dim, value_head_dim,
|
| 202 |
device=q.device, dtype=q.dtype)
|
| 203 |
-
outputs = []
|
| 204 |
gamma = self.gamma.to(device=q.device, dtype=q.dtype)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
for t in range(tgt_len):
|
| 206 |
state = gamma.view(1, -1, 1, 1) * state + \
|
| 207 |
k[:, :, t, :].unsqueeze(-1) * v[:, :, t, :].unsqueeze(-2)
|
|
@@ -359,6 +391,59 @@ class RetNetBlock(nn.Module):
|
|
| 359 |
return self._block_forward(x, mode, state, offset, return_state, chunk_size)
|
| 360 |
|
| 361 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 362 |
class RetNetForCausalLM(PreTrainedModel):
|
| 363 |
"""RetNet language model with a from-scratch token embedding.
|
| 364 |
|
|
@@ -432,7 +517,15 @@ class RetNetForCausalLM(PreTrainedModel):
|
|
| 432 |
return n
|
| 433 |
|
| 434 |
def _forward_hidden(self, input_ids, mode="parallel", states=None, offset=0,
|
| 435 |
-
return_state=False, chunk_size=64, return_hidden=False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
if input_ids.ndim != 2:
|
| 437 |
raise ValueError("input_ids must be [batch, seq]")
|
| 438 |
x = self.embedding(input_ids) * self.embed_scale
|
|
@@ -441,9 +534,12 @@ class RetNetForCausalLM(PreTrainedModel):
|
|
| 441 |
states = [None] * len(self.blocks)
|
| 442 |
if len(states) != len(self.blocks):
|
| 443 |
raise ValueError("states must contain one entry per block")
|
|
|
|
|
|
|
|
|
|
| 444 |
new_states: List[Optional[torch.Tensor]] = []
|
| 445 |
for block, block_state in zip(self.blocks, states):
|
| 446 |
-
if
|
| 447 |
x, ns = block(x, mode=mode, state=block_state, offset=offset,
|
| 448 |
return_state=True, chunk_size=chunk_size)
|
| 449 |
new_states.append(ns)
|
|
@@ -455,11 +551,11 @@ class RetNetForCausalLM(PreTrainedModel):
|
|
| 455 |
if return_hidden:
|
| 456 |
# 融合交叉熵需要的是 LM head 的**输入**(hidden),不是输出。
|
| 457 |
# 这里提前返回可以完全不物化 [B,L,V]。
|
| 458 |
-
if
|
| 459 |
return x, new_states
|
| 460 |
return x
|
| 461 |
logits = self.lm_head(x)
|
| 462 |
-
if
|
| 463 |
return logits, new_states
|
| 464 |
return logits
|
| 465 |
|
|
@@ -486,17 +582,24 @@ class RetNetForCausalLM(PreTrainedModel):
|
|
| 486 |
if mode is None:
|
| 487 |
mode = getattr(self.config, "mode", "parallel")
|
| 488 |
_fused = (os.environ.get("RETNET_FUSED_CE") == "1"
|
| 489 |
-
and labels is not None and wcodes is not None
|
|
|
|
| 490 |
logits = None
|
| 491 |
# `loss` 必须在所有分支之前初始化:不传 labels 时(普通推理、prefill、
|
| 492 |
# 以及 HF 的 generate 路径)上面几个分支都不会赋值,line 536 的
|
| 493 |
# CausalLMOutput(loss=loss, ...) 会直接 UnboundLocalError。
|
| 494 |
loss = None
|
| 495 |
if _fused:
|
| 496 |
-
# 融合路径:LM head 的输入直接进 fused CE,完全不物化 [B,L,V]
|
| 497 |
-
|
|
|
|
|
|
|
| 498 |
input_ids, mode=mode, states=states, offset=offset,
|
| 499 |
-
return_state=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 500 |
B_, L_, D_ = hidden.shape
|
| 501 |
w = torch.zeros_like(wcodes, dtype=torch.float32)
|
| 502 |
w = torch.where(wcodes == 1, torch.ones_like(w), w)
|
|
@@ -515,29 +618,16 @@ class RetNetForCausalLM(PreTrainedModel):
|
|
| 515 |
logits = self._forward_hidden(input_ids, mode=mode, states=states, offset=offset)
|
| 516 |
new_states = None
|
| 517 |
|
| 518 |
-
if not _fused:
|
| 519 |
-
#
|
| 520 |
-
#
|
| 521 |
-
#
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
w = torch.zeros_like(wcodes, dtype=torch.float32)
|
| 529 |
-
w = torch.where(wcodes == 1, torch.ones_like(w), w)
|
| 530 |
-
w = torch.where(wcodes == 2,
|
| 531 |
-
torch.full_like(w, self.config.prompt_loss_weight), w)
|
| 532 |
-
w = torch.where(wcodes == 3, torch.ones_like(w), w)
|
| 533 |
-
loss = (token_loss * w.reshape(-1)).sum() / w.sum().clamp_min(1.0)
|
| 534 |
-
elif labels is not None:
|
| 535 |
-
logits_f = logits.float()
|
| 536 |
-
loss = F.cross_entropy(logits_f.reshape(-1, logits_f.shape[-1]),
|
| 537 |
-
labels.reshape(-1))
|
| 538 |
-
|
| 539 |
-
from transformers.modeling_outputs import CausalLMOutput
|
| 540 |
-
return CausalLMOutput(loss=loss, logits=logits)
|
| 541 |
|
| 542 |
@torch.inference_mode()
|
| 543 |
def generate(self, input_ids: torch.Tensor, max_new_tokens: int = 32,
|
|
|
|
| 20 |
from transformers import PreTrainedModel, PretrainedConfig
|
| 21 |
|
| 22 |
#: Opt-in batched-chunk chunkwise retention (see ``retention_fast.py``).
|
| 23 |
+
#: Mathematically the same function as the reference per-chunk loop (5 batched
|
| 24 |
+
#: matmuls instead of a Python loop of small ones), equal up to floating-point
|
| 25 |
+
#: rounding **not bitwise**. Verified by ``tests/test_fast_chunkwise.py``.
|
| 26 |
FAST_CHUNKWISE = os.environ.get("RETNET_FAST_CHUNKWISE", "0") == "1"
|
| 27 |
|
| 28 |
+
#: 每个交叉熵分块允许物化的 fp32 logits 切片字节数(默认 64 MB)。
|
| 29 |
+
#: 旧实现把 chunk 写成一个固定的 **token 数**(262144),于是分块大小与词表
|
| 30 |
+
#: 大小无关、且大于生产形状的 token 总数 —— B=16/L=2048/V=20000 时只有一块,
|
| 31 |
+
#: 仍然是 32768×20000×4 ≈ 2.6 GB 的整段拷贝,E22 声称消除的那一份原封不动。
|
| 32 |
+
#: 现在按字节预算反推 token 数:V 越大每 token 越贵,块就越小。
|
| 33 |
+
DEFAULT_CE_CHUNK_BYTES = 64 << 20
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def ce_chunk_tokens(vocab_size: int, chunk_bytes: int = DEFAULT_CE_CHUNK_BYTES) -> int:
|
| 37 |
+
"""按字节预算反推每个 CE 分块能放多少 token(至少 1)。
|
| 38 |
+
|
| 39 |
+
单独成函数是为了让测试能直接断言“生产形状下必须切出多块”,而不是只
|
| 40 |
+
验证数值等价 —— 旧测试传 ``chunk_tokens=7``,即使参数被无视也会通过。
|
| 41 |
+
"""
|
| 42 |
+
per_token = 4 * max(1, int(vocab_size)) # fp32 logits,每 token V 个 float
|
| 43 |
+
return max(1, int(chunk_bytes) // per_token)
|
| 44 |
+
|
| 45 |
# NOTE(2026-10-10): 试过给 ``MultiScaleRetention._rotations`` 加一层
|
| 46 |
# ``(length, offset, device, dtype)`` 键的 sin/cos 表缓存,理由是 profile 里
|
| 47 |
# ``triton_poi_fused__to_copy*`` 合计 42.9 ms(占一个优化步 5.9%),而这 20 层
|
|
|
|
| 58 |
value_dim=None, ffn_dim=None, dropout=0.1, norm_eps=1e-5,
|
| 59 |
tie_weights=True, embed_scale=None, prompt_loss_weight=0.1,
|
| 60 |
use_gla=False, use_swiglu=False, use_growth_ln=False,
|
| 61 |
+
growth_old_dim=None, checkpoint_layers=-1,
|
| 62 |
+
ce_chunk_bytes=DEFAULT_CE_CHUNK_BYTES, **kwargs):
|
| 63 |
super().__init__(**kwargs)
|
| 64 |
self.vocab_size = vocab_size
|
| 65 |
self.d_model = d_model
|
|
|
|
| 72 |
self.tie_weights = tie_weights
|
| 73 |
self.embed_scale = embed_scale
|
| 74 |
self.prompt_loss_weight = prompt_loss_weight
|
| 75 |
+
#: 交叉熵分块的字节预算(E22)。按 V 反推 token 数,随 config 落盘,
|
| 76 |
+
#: 换词表/换卡时不必改代码。<=0 表示退回“整段不分块”(仅用于对照实验)。
|
| 77 |
+
self.ce_chunk_bytes = int(ce_chunk_bytes)
|
| 78 |
self.use_gla = use_gla
|
| 79 |
self.use_swiglu = use_swiglu
|
| 80 |
self.use_growth_ln = use_growth_ln
|
|
|
|
| 195 |
exp = delta.masked_fill(~causal, 0)
|
| 196 |
mask = self.gamma.to(device=device, dtype=torch.float32)[:, None, None] ** exp
|
| 197 |
mask = mask.masked_fill(~causal, 0.0)
|
| 198 |
+
# 2026-10-20: 删除了原来的 `mask / mask.sum(...).sqrt()`。
|
| 199 |
+
# 那一步只存在于 parallel 路径,chunkwise/recurrent 都没有,导致
|
| 200 |
+
# 训练(chunkwise)与生成 prefill(parallel)算的不是同一个函数。
|
| 201 |
+
# 见 docs/00 E16。
|
| 202 |
return mask.to(dtype=dtype)
|
| 203 |
|
| 204 |
def _parallel_retention(self, q, k, v):
|
| 205 |
bsz, _, tgt_len, _ = q.shape
|
| 206 |
mask = self._decay_mask(tgt_len, q.device, q.dtype)
|
| 207 |
scores = torch.matmul(q, k.transpose(-1, -2)) * mask
|
| 208 |
+
# 2026-10-20: 删除了原来的 `scores / scores.detach().abs().sum(...)`。
|
| 209 |
+
# 同一理由:这是 parallel 独有的第二次归一化,见 docs/00 E16。
|
| 210 |
return torch.matmul(scores, v)
|
| 211 |
|
| 212 |
def _parallel_final_state(self, k, v, state):
|
|
|
|
| 225 |
if state is None:
|
| 226 |
state = torch.zeros(bsz, self.num_heads, key_dim, value_head_dim,
|
| 227 |
device=q.device, dtype=q.dtype)
|
|
|
|
| 228 |
gamma = self.gamma.to(device=q.device, dtype=q.dtype)
|
| 229 |
+
# docs/00 E21:解码时 tgt_len 恒为 1,走快路径,省掉 Python 循环与
|
| 230 |
+
# list/stack 的物化。数学与下面的循环逐位相同。
|
| 231 |
+
if tgt_len == 1:
|
| 232 |
+
state = gamma.view(1, -1, 1, 1) * state + \
|
| 233 |
+
k[:, :, 0, :].unsqueeze(-1) * v[:, :, 0, :].unsqueeze(-2)
|
| 234 |
+
out = torch.sum(q[:, :, 0, :].unsqueeze(-1) * state, dim=-2).unsqueeze(2)
|
| 235 |
+
return out, state
|
| 236 |
+
outputs = []
|
| 237 |
for t in range(tgt_len):
|
| 238 |
state = gamma.view(1, -1, 1, 1) * state + \
|
| 239 |
k[:, :, t, :].unsqueeze(-1) * v[:, :, t, :].unsqueeze(-2)
|
|
|
|
| 391 |
return self._block_forward(x, mode, state, offset, return_state, chunk_size)
|
| 392 |
|
| 393 |
|
| 394 |
+
def _chunked_weighted_ce(logits, labels, wcodes=None, prompt_loss_weight=0.1,
|
| 395 |
+
chunk_tokens: int | None = None,
|
| 396 |
+
chunk_bytes: int = DEFAULT_CE_CHUNK_BYTES):
|
| 397 |
+
"""分块计算加权交叉熵,避免把整个 [B,L,V] 转成 fp32(docs/00 E22)。
|
| 398 |
+
|
| 399 |
+
旧实现是 ``logits_f = logits.float()`` 之后一次性 reshape 求 CE。
|
| 400 |
+
对本项目(B=16, L=2048, V=20000)那一步会物化 16×2048×20000×4 ≈ 2.6 GB
|
| 401 |
+
的 fp32 副本,是显存峰值的主要来源;而 logits 本身还是 bf16,已经占 1.3 GB。
|
| 402 |
+
|
| 403 |
+
分块大小**由字节预算反推**(``ce_chunk_tokens``),不是固定 token 数:
|
| 404 |
+
每块只把 [chunk, V] 转 fp32,峰值约为 ``chunk_bytes`` 的量级。默认 64 MB
|
| 405 |
+
在 V=20000 时是 838 token/块,于是生产形状 B=16/L=2048 会切出 40 块,
|
| 406 |
+
而不是旧实现的一块 2.6 GB。显式传 ``chunk_tokens`` 仍可覆盖(测试用)。
|
| 407 |
+
|
| 408 |
+
权重码语义与旧实现完全一致:pad=0 不计、reply/plain=1、prompt=prompt_loss_weight,
|
| 409 |
+
最后除以权重和;没有 wcodes 时退化为普通均值。
|
| 410 |
+
"""
|
| 411 |
+
V = logits.shape[-1]
|
| 412 |
+
flat_logits = logits.reshape(-1, V)
|
| 413 |
+
flat_labels = labels.reshape(-1)
|
| 414 |
+
n = flat_logits.shape[0]
|
| 415 |
+
if chunk_tokens is None:
|
| 416 |
+
chunk_tokens = ce_chunk_tokens(V, chunk_bytes)
|
| 417 |
+
chunk_tokens = max(1, int(chunk_tokens))
|
| 418 |
+
|
| 419 |
+
w = None
|
| 420 |
+
if wcodes is not None:
|
| 421 |
+
w = torch.zeros_like(wcodes, dtype=torch.float32)
|
| 422 |
+
w = torch.where(wcodes == 1, torch.ones_like(w), w)
|
| 423 |
+
w = torch.where(wcodes == 2, torch.full_like(w, float(prompt_loss_weight)), w)
|
| 424 |
+
w = torch.where(wcodes == 3, torch.ones_like(w), w)
|
| 425 |
+
w = w.reshape(-1)
|
| 426 |
+
|
| 427 |
+
total = torch.zeros((), device=flat_logits.device, dtype=torch.float32)
|
| 428 |
+
denom = torch.zeros((), device=flat_logits.device, dtype=torch.float32)
|
| 429 |
+
for s in range(0, n, chunk_tokens):
|
| 430 |
+
e = min(s + chunk_tokens, n)
|
| 431 |
+
ce = F.cross_entropy(flat_logits[s:e].float(), flat_labels[s:e], reduction="none")
|
| 432 |
+
if w is None:
|
| 433 |
+
total = total + ce.sum()
|
| 434 |
+
denom = denom + float(e - s)
|
| 435 |
+
else:
|
| 436 |
+
total = total + (ce * w[s:e]).sum()
|
| 437 |
+
denom = denom + w[s:e].sum()
|
| 438 |
+
return total / denom.clamp_min(1.0)
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
def build_causal_lm_output(loss, logits):
|
| 442 |
+
"""构造 HF 的 CausalLMOutput;单独成函数是为了让 forward 里只有一处返回。"""
|
| 443 |
+
from transformers.modeling_outputs import CausalLMOutput
|
| 444 |
+
return CausalLMOutput(loss=loss, logits=logits)
|
| 445 |
+
|
| 446 |
+
|
| 447 |
class RetNetForCausalLM(PreTrainedModel):
|
| 448 |
"""RetNet language model with a from-scratch token embedding.
|
| 449 |
|
|
|
|
| 517 |
return n
|
| 518 |
|
| 519 |
def _forward_hidden(self, input_ids, mode="parallel", states=None, offset=0,
|
| 520 |
+
return_state=False, chunk_size=64, return_hidden=False,
|
| 521 |
+
need_state=None):
|
| 522 |
+
"""跑完所有 block,返回 logits(或 ``return_hidden`` 时的 final_norm 输出)。
|
| 523 |
+
|
| 524 |
+
``need_state`` 与 ``return_state`` 分开:融合交叉熵只想要 hidden,完全
|
| 525 |
+
不需要每层的递归状态,但旧签名里 ``return_state=True`` 会让每个 block 都
|
| 526 |
+
算一次 ``_parallel_final_state``(对 ``[B,H,L,kd,vd]`` 做 einsum)再把结果
|
| 527 |
+
丢掉。显式传 ``need_state=False`` 就能跳掉这部分(E22/P2 性能项)。
|
| 528 |
+
"""
|
| 529 |
if input_ids.ndim != 2:
|
| 530 |
raise ValueError("input_ids must be [batch, seq]")
|
| 531 |
x = self.embedding(input_ids) * self.embed_scale
|
|
|
|
| 534 |
states = [None] * len(self.blocks)
|
| 535 |
if len(states) != len(self.blocks):
|
| 536 |
raise ValueError("states must contain one entry per block")
|
| 537 |
+
# 递归模式必须返回状态(解码靠它推进);其余路径由调用方决定。
|
| 538 |
+
want_state = (return_state if need_state is None else bool(need_state)) \
|
| 539 |
+
or mode == "recurrent"
|
| 540 |
new_states: List[Optional[torch.Tensor]] = []
|
| 541 |
for block, block_state in zip(self.blocks, states):
|
| 542 |
+
if want_state:
|
| 543 |
x, ns = block(x, mode=mode, state=block_state, offset=offset,
|
| 544 |
return_state=True, chunk_size=chunk_size)
|
| 545 |
new_states.append(ns)
|
|
|
|
| 551 |
if return_hidden:
|
| 552 |
# 融合交叉熵需要的是 LM head 的**输入**(hidden),不是输出。
|
| 553 |
# 这里提前返回可以完全不物化 [B,L,V]。
|
| 554 |
+
if want_state:
|
| 555 |
return x, new_states
|
| 556 |
return x
|
| 557 |
logits = self.lm_head(x)
|
| 558 |
+
if want_state:
|
| 559 |
return logits, new_states
|
| 560 |
return logits
|
| 561 |
|
|
|
|
| 582 |
if mode is None:
|
| 583 |
mode = getattr(self.config, "mode", "parallel")
|
| 584 |
_fused = (os.environ.get("RETNET_FUSED_CE") == "1"
|
| 585 |
+
and labels is not None and wcodes is not None
|
| 586 |
+
and not return_state and mode != "recurrent")
|
| 587 |
logits = None
|
| 588 |
# `loss` 必须在所有分支之前初始化:不传 labels 时(普通推理、prefill、
|
| 589 |
# 以及 HF 的 generate 路径)上面几个分支都不会赋值,line 536 的
|
| 590 |
# CausalLMOutput(loss=loss, ...) 会直接 UnboundLocalError。
|
| 591 |
loss = None
|
| 592 |
if _fused:
|
| 593 |
+
# 融合路径:LM head 的输入直接进 fused CE,完全不物化 [B,L,V]。
|
| 594 |
+
# need_state=False:这段只训练用,不需要每层的递归状态,
|
| 595 |
+
# 旧写法传 return_state=True 会让每个 block 白算一次最终状态。
|
| 596 |
+
hidden = self._forward_hidden(
|
| 597 |
input_ids, mode=mode, states=states, offset=offset,
|
| 598 |
+
return_state=False, chunk_size=chunk_size, return_hidden=True,
|
| 599 |
+
need_state=False)
|
| 600 |
+
if isinstance(hidden, tuple): # recurrent 强制要状态
|
| 601 |
+
hidden = hidden[0]
|
| 602 |
+
new_states = None
|
| 603 |
B_, L_, D_ = hidden.shape
|
| 604 |
w = torch.zeros_like(wcodes, dtype=torch.float32)
|
| 605 |
w = torch.where(wcodes == 1, torch.ones_like(w), w)
|
|
|
|
| 618 |
logits = self._forward_hidden(input_ids, mode=mode, states=states, offset=offset)
|
| 619 |
new_states = None
|
| 620 |
|
| 621 |
+
if not _fused and labels is not None:
|
| 622 |
+
# docs/00 E22:不再对整段 logits 做 .float()(B=16,L=2048,V=20000 时
|
| 623 |
+
# 那是约 2.6 GB 的 fp32 副本),改为按**字节预算**分块求 CE。
|
| 624 |
+
# 数值与旧实现一致:同样的权重码、同样的“除以权重和”。
|
| 625 |
+
loss = _chunked_weighted_ce(
|
| 626 |
+
logits, labels, wcodes, self.config.prompt_loss_weight,
|
| 627 |
+
chunk_bytes=int(getattr(self.config, "ce_chunk_bytes",
|
| 628 |
+
DEFAULT_CE_CHUNK_BYTES) or DEFAULT_CE_CHUNK_BYTES))
|
| 629 |
+
|
| 630 |
+
return build_causal_lm_output(loss, logits)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 631 |
|
| 632 |
@torch.inference_mode()
|
| 633 |
def generate(self, input_ids: torch.Tensor, max_new_tokens: int = 32,
|