Text Generation
Transformers
Safetensors
kestrel
titans
titans-mag
looped-transformer
small-language-model
conversational
custom_code
Instructions to use lillycham/Kestrel-34M-instruct with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use lillycham/Kestrel-34M-instruct with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="lillycham/Kestrel-34M-instruct", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("lillycham/Kestrel-34M-instruct", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use lillycham/Kestrel-34M-instruct with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "lillycham/Kestrel-34M-instruct" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "lillycham/Kestrel-34M-instruct", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/lillycham/Kestrel-34M-instruct
- SGLang
How to use lillycham/Kestrel-34M-instruct with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "lillycham/Kestrel-34M-instruct" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "lillycham/Kestrel-34M-instruct", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "lillycham/Kestrel-34M-instruct" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "lillycham/Kestrel-34M-instruct", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use lillycham/Kestrel-34M-instruct with Docker Model Runner:
docker model run hf.co/lillycham/Kestrel-34M-instruct
Download modeling_kestrel.py from lillycham/Kestrel-34M-instruct: direct link, hf CLI and curl.
- Browser
- Download file 21.2 kB
-
https://huggingface.co/lillycham/Kestrel-34M-instruct/resolve/main/modeling_kestrel.py
- Command line
-
hf download hf://lillycham/Kestrel-34M-instruct/modeling_kestrel.py
-
curl -L -o modeling_kestrel.py https://huggingface.co/lillycham/Kestrel-34M-instruct/resolve/main/modeling_kestrel.py
21.2 kB
| """Hugging Face modeling code for Kestrel (Looped Titans-MAG).""" | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from transformers import PreTrainedModel | |
| from transformers.modeling_outputs import CausalLMOutput | |
| from transformers.models.qwen3.modeling_qwen3 import ( | |
| Qwen3MLP, | |
| Qwen3RMSNorm, | |
| Qwen3RotaryEmbedding, | |
| apply_rotary_pos_emb, | |
| ) | |
| from .configuration_kestrel import KestrelConfig | |
| def _silu_and_grad(x): | |
| sig = torch.sigmoid(x) | |
| silu = x * sig | |
| grad = sig * (1.0 + x * (1.0 - sig)) | |
| return silu, grad | |
| class NeuralMemory(nn.Module): | |
| """Two-layer fast-weight MLP memory updated per chunk via closed-form MSE gradients.""" | |
| def __init__(self, hidden_size, memory_hidden_size, chunk_size, heads=1, forget_bias=-2.0, min_forget=0.0): | |
| super().__init__() | |
| self.hidden_size = hidden_size | |
| self.memory_hidden_size = memory_hidden_size | |
| self.chunk_size = chunk_size | |
| self.heads = heads | |
| self.head_dim = hidden_size // heads | |
| self.mem_head_dim = memory_hidden_size // heads | |
| self.forget_bias = forget_bias | |
| self.min_forget = min_forget | |
| self.q_proj = nn.Linear(hidden_size, hidden_size, bias=False) | |
| self.k_proj = nn.Linear(hidden_size, hidden_size, bias=False) | |
| self.v_proj = nn.Linear(hidden_size, hidden_size, bias=False) | |
| self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False) | |
| # Per-head, per-chunk scalar gates: alpha (forget), eta (momentum), theta (inner lr). | |
| self.gate_proj = nn.Linear(hidden_size, 3 * heads, bias=False) | |
| init_shape_w1 = (memory_hidden_size, hidden_size) if heads == 1 else (heads, self.mem_head_dim, self.head_dim) | |
| init_shape_w2 = (hidden_size, memory_hidden_size) if heads == 1 else (heads, self.head_dim, self.mem_head_dim) | |
| self.w1_init = nn.Parameter(torch.empty(*init_shape_w1)) | |
| self.w2_init = nn.Parameter(torch.empty(*init_shape_w2)) | |
| nn.init.normal_(self.w1_init, std=self.head_dim ** -0.5) | |
| nn.init.normal_(self.w2_init, std=self.mem_head_dim ** -0.5) | |
| def _project_qkv(self, proj, x, bsz, num_chunks): | |
| h = proj(x).view(bsz, num_chunks, self.chunk_size, self.heads, self.head_dim) | |
| h = F.normalize(h, dim=-1).permute(0, 3, 1, 2, 4).contiguous() | |
| return h.view(bsz * self.heads, num_chunks, self.chunk_size, self.head_dim).float() | |
| def _gates(self, chunk_ctx): | |
| """Per-head forget (alpha), momentum (eta) and inner-lr (theta) gates, shaped (B * H, n, 1, 1).""" | |
| bsz, num_chunks, _ = chunk_ctx.shape | |
| gates = self.gate_proj(chunk_ctx).view(bsz, num_chunks, self.heads, 3).permute(0, 2, 1, 3).contiguous() | |
| gates = gates.view(bsz * self.heads, num_chunks, 3).float() | |
| alpha = (self.min_forget + (1.0 - self.min_forget) * torch.sigmoid(gates[..., 0] + self.forget_bias))[..., None, None] | |
| eta = torch.sigmoid(gates[..., 1])[..., None, None] | |
| theta = 0.05 * torch.sigmoid(gates[..., 2])[..., None, None] | |
| return alpha, eta, theta | |
| def _initial_state(self, bsz): | |
| w1_base = self.w1_init.unsqueeze(0) if self.heads == 1 else self.w1_init | |
| w2_base = self.w2_init.unsqueeze(0) if self.heads == 1 else self.w2_init | |
| w1 = w1_base.float().unsqueeze(0).expand(bsz, -1, -1, -1).reshape(bsz * self.heads, self.mem_head_dim, self.head_dim).clone() | |
| w2 = w2_base.float().unsqueeze(0).expand(bsz, -1, -1, -1).reshape(bsz * self.heads, self.head_dim, self.mem_head_dim).clone() | |
| return w1, w2, torch.zeros_like(w1), torch.zeros_like(w2) | |
| def _retrieve(qc, w1, w2): | |
| return torch.bmm(F.silu(torch.bmm(qc, w1.transpose(1, 2))), w2.transpose(1, 2)) | |
| def _update(self, state, kc, vc, a_i, e_i, t_i): | |
| """One momentum step on the closed-form gradient of 0.5 / C * ||M(kc) - vc||^2.""" | |
| w1, w2, s1, s2 = state | |
| hk = torch.bmm(kc, w1.transpose(1, 2)) # (B * H, C, m_h) | |
| ak, dsilu = _silu_and_grad(hk) | |
| pred = torch.bmm(ak, w2.transpose(1, 2)) | |
| err = (pred - vc) * (1.0 / self.chunk_size) # (B * H, C, d_h) | |
| gw2 = torch.bmm(err.transpose(1, 2), ak) # (B * H, d_h, m_h) | |
| dak = torch.bmm(err, w2) * dsilu # (B * H, C, m_h) | |
| gw1 = torch.bmm(dak.transpose(1, 2), kc) # (B * H, m_h, d_h) | |
| s1 = e_i * s1 - t_i * gw1 | |
| s2 = e_i * s2 - t_i * gw2 | |
| return (1.0 - a_i) * w1 + s1, (1.0 - a_i) * w2 + s2, s1, s2 | |
| def forward(self, x, cache=None): | |
| if cache is not None: | |
| return self._forward_cached(x, cache) | |
| bsz, seq_len, dim = x.shape | |
| c = self.chunk_size | |
| pad = (c - seq_len % c) % c | |
| if pad: | |
| x = F.pad(x, (0, 0, 0, pad)) | |
| padded_len = x.shape[1] | |
| num_chunks = padded_len // c | |
| q = self._project_qkv(self.q_proj, x, bsz, num_chunks) | |
| k = self._project_qkv(self.k_proj, x, bsz, num_chunks) | |
| v = self._project_qkv(self.v_proj, x, bsz, num_chunks) | |
| # Average chunk hidden state drives scalar forget/momentum/lr gates per head. | |
| alpha, eta, theta = self._gates(x.view(bsz, num_chunks, c, dim).mean(dim=2)) | |
| state = self._initial_state(bsz) | |
| outputs = [] | |
| for idx in range(num_chunks): | |
| # Retrieve from current memory state (before updating on chunk idx -> causal). | |
| outputs.append(self._retrieve(q[:, idx], state[0], state[1])) | |
| if idx + 1 < num_chunks: | |
| state = self._update(state, k[:, idx], v[:, idx], alpha[:, idx], eta[:, idx], theta[:, idx]) | |
| out = torch.cat(outputs, dim=1).view(bsz, self.heads, padded_len, self.head_dim) | |
| out = out.permute(0, 2, 1, 3).contiguous().view(bsz, padded_len, dim)[:, :seq_len].to(x.dtype) | |
| return self.out_proj(out) | |
| def _forward_cached(self, x, cache): | |
| """Streaming forward for decoding. Gives the same outputs as forward() on the whole sequence. | |
| The cache holds the memory state after the last complete chunk, plus the | |
| hidden states, keys and values of the current incomplete chunk. The memory | |
| updates only when a chunk is complete, as in the parallel path. | |
| """ | |
| bsz, seq_len, dim = x.shape | |
| c = self.chunk_size | |
| if "state" not in cache: | |
| cache["state"] = self._initial_state(bsz) | |
| cache["x"] = x.new_zeros(bsz, 0, dim) | |
| cache["k"] = x.new_zeros(bsz * self.heads, 0, self.head_dim, dtype=torch.float32) | |
| cache["v"] = x.new_zeros(bsz * self.heads, 0, self.head_dim, dtype=torch.float32) | |
| def project(proj): | |
| h = F.normalize(proj(x).view(bsz, seq_len, self.heads, self.head_dim), dim=-1) | |
| return h.transpose(1, 2).reshape(bsz * self.heads, seq_len, self.head_dim).float() | |
| q, k, v = project(self.q_proj), project(self.k_proj), project(self.v_proj) | |
| outputs = [] | |
| start = 0 | |
| while start < seq_len: | |
| n = min(c - cache["x"].shape[1], seq_len - start) | |
| span = slice(start, start + n) | |
| w1, w2 = cache["state"][:2] | |
| outputs.append(self._retrieve(q[:, span], w1, w2)) | |
| cache["x"] = torch.cat([cache["x"], x[:, span]], dim=1) | |
| cache["k"] = torch.cat([cache["k"], k[:, span]], dim=1) | |
| cache["v"] = torch.cat([cache["v"], v[:, span]], dim=1) | |
| if cache["x"].shape[1] == c: | |
| alpha, eta, theta = self._gates(cache["x"].mean(dim=1, keepdim=True)) | |
| cache["state"] = self._update(cache["state"], cache["k"], cache["v"], alpha[:, 0], eta[:, 0], theta[:, 0]) | |
| cache["x"], cache["k"], cache["v"] = cache["x"][:, :0], cache["k"][:, :0], cache["v"][:, :0] | |
| start += n | |
| out = torch.cat(outputs, dim=1).view(bsz, self.heads, seq_len, self.head_dim) | |
| out = out.transpose(1, 2).reshape(bsz, seq_len, dim).to(x.dtype) | |
| return self.out_proj(out) | |
| class SlidingWindowAttention(nn.Module): | |
| """GQA with QK-norm, RoPE, and a local sliding causal window.""" | |
| def __init__(self, config): | |
| super().__init__() | |
| self.num_heads = config.num_attention_heads | |
| self.num_kv_heads = config.num_key_value_heads | |
| self.num_kv_groups = self.num_heads // self.num_kv_heads | |
| self.head_dim = config.head_dim | |
| self.window = config.attention_window | |
| self.q_proj = nn.Linear(config.hidden_size, self.num_heads * self.head_dim, bias=False) | |
| self.k_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=False) | |
| self.v_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=False) | |
| self.o_proj = nn.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=False) | |
| self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) | |
| self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) | |
| def forward(self, x, cos, sin, cache=None): | |
| bsz, seq_len, _ = x.shape | |
| q = self.q_norm(self.q_proj(x).view(bsz, seq_len, self.num_heads, self.head_dim)).transpose(1, 2) | |
| k = self.k_norm(self.k_proj(x).view(bsz, seq_len, self.num_kv_heads, self.head_dim)).transpose(1, 2) | |
| v = self.v_proj(x).view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2) | |
| q, k = apply_rotary_pos_emb(q, k, cos, sin) | |
| if cache is not None: | |
| # Keys and values are stored after QK-norm and RoPE. RoPE is relative, | |
| # so the cache only needs the last window - 1 positions. | |
| if "k" in cache: | |
| k = torch.cat([cache["k"], k], dim=2) | |
| v = torch.cat([cache["v"], v], dim=2) | |
| keep = self.window - 1 | |
| cache["k"] = k[:, :, max(0, k.shape[2] - keep):] | |
| cache["v"] = v[:, :, max(0, v.shape[2] - keep):] | |
| key_len = k.shape[2] | |
| past = key_len - seq_len | |
| if self.num_kv_groups > 1: | |
| k = k.repeat_interleave(self.num_kv_groups, dim=1) | |
| v = v.repeat_interleave(self.num_kv_groups, dim=1) | |
| if past == 0 and self.window >= seq_len: | |
| attn = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| elif seq_len == 1 and key_len <= self.window: | |
| attn = F.scaled_dot_product_attention(q, k, v) | |
| else: | |
| pos = torch.arange(key_len, device=x.device) | |
| diff = pos[past:, None] - pos[None, :] | |
| mask = (diff >= 0) & (diff < self.window) | |
| attn = F.scaled_dot_product_attention(q, k, v, attn_mask=mask) | |
| return self.o_proj(attn.transpose(1, 2).contiguous().view(bsz, seq_len, -1)) | |
| class TitansMAGBlock(nn.Module): | |
| """Parallel sliding-window attention + neural memory combined via a learned gate.""" | |
| def __init__(self, config): | |
| super().__init__() | |
| self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.self_attn = SlidingWindowAttention(config) | |
| self.attn_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) if getattr(config, "norm_attn_branch", False) else nn.Identity() | |
| self.memory = NeuralMemory( | |
| config.hidden_size, config.memory_hidden_size, config.memory_chunk_size, | |
| heads=getattr(config, "memory_heads", 1), | |
| forget_bias=getattr(config, "memory_forget_bias", -2.0), | |
| min_forget=getattr(config, "memory_min_forget", 0.0), | |
| ) | |
| self.mem_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.mag_gate = nn.Linear(config.hidden_size, config.hidden_size, bias=False) | |
| self.post_attention_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.mlp = Qwen3MLP(config) | |
| def forward(self, x, cos, sin, cache=None): | |
| h = self.input_layernorm(x) | |
| attn_cache, mem_cache = (None, None) if cache is None else (cache.setdefault("attn", {}), cache.setdefault("mem", {})) | |
| attn_out = self.attn_norm(self.self_attn(h, cos, sin, attn_cache)) | |
| mem_out = self.mem_norm(self.memory(h, mem_cache)) | |
| gate = torch.sigmoid(self.mag_gate(h)) | |
| x = x + (1.0 - gate) * attn_out + gate * mem_out | |
| x = x + self.mlp(self.post_attention_layernorm(x)) | |
| return x | |
| class KestrelForCausalLM(PreTrainedModel): | |
| """Titans-MAG causal LM with optional prelude/coda and looped recurrent core.""" | |
| config_class = KestrelConfig | |
| _tied_weights_keys = {"lm_head.weight": "embed_tokens.weight"} | |
| def __init__(self, config): | |
| super().__init__(config) | |
| self.config = config | |
| self.num_loops = getattr(config, "num_loops", 1) | |
| prelude_layers = getattr(config, "prelude_layers", 0) | |
| coda_layers = getattr(config, "coda_layers", 0) | |
| self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=getattr(config, "pad_token_id", None)) | |
| if prelude_layers: | |
| self.prelude = nn.ModuleList([TitansMAGBlock(config) for _ in range(prelude_layers)]) | |
| self.layers = nn.ModuleList([TitansMAGBlock(config) for _ in range(config.num_hidden_layers)]) | |
| if self.num_loops > 1: | |
| self.loop_x0_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.loop_inject = nn.Linear(config.hidden_size, config.hidden_size, bias=False) | |
| # 3D step embedding stays in AdamW (no_decay) rather than Muon. | |
| self.loop_embed = nn.Parameter(torch.empty(self.num_loops, 1, config.hidden_size)) | |
| if coda_layers: | |
| self.coda = nn.ModuleList([TitansMAGBlock(config) for _ in range(coda_layers)]) | |
| self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.rotary_emb = Qwen3RotaryEmbedding(config=config) | |
| self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) | |
| self.post_init() | |
| def get_input_embeddings(self): | |
| return self.embed_tokens | |
| def get_output_embeddings(self): | |
| return self.lm_head | |
| def forward(self, input_ids, cache=None, logits_to_keep=0, **kwargs): | |
| """Run the model. With a cache dict, continue from the tokens that it holds. | |
| The cache is a plain dict that this method fills on the first call. Each | |
| block call (prelude, every loop of the core, coda) gets its own entry. | |
| """ | |
| x = self.embed_tokens(input_ids) | |
| offset = 0 if cache is None else cache.get("pos", 0) | |
| pos_ids = torch.arange(offset, offset + input_ids.shape[1], device=input_ids.device).unsqueeze(0) | |
| cos, sin = self.rotary_emb(x, pos_ids) | |
| layer_caches = None if cache is None else iter(cache.setdefault("layers", [{} for _ in range(self._block_calls())])) | |
| def run(layer, x): | |
| return layer(x, cos, sin, None if layer_caches is None else next(layer_caches)) | |
| if hasattr(self, "prelude"): | |
| for layer in self.prelude: | |
| x = run(layer, x) | |
| if self.num_loops > 1: | |
| x0_normed = self.loop_x0_norm(x) | |
| for loop_idx in range(self.num_loops): | |
| x = x + self.loop_inject(x0_normed) + self.loop_embed[loop_idx] | |
| for layer in self.layers: | |
| x = run(layer, x) | |
| else: | |
| for layer in self.layers: | |
| x = run(layer, x) | |
| if hasattr(self, "coda"): | |
| for layer in self.coda: | |
| x = run(layer, x) | |
| if cache is not None: | |
| cache["pos"] = offset + input_ids.shape[1] | |
| if logits_to_keep: | |
| x = x[:, -logits_to_keep:] | |
| logits = self.lm_head(self.norm(x)) | |
| return CausalLMOutput(logits=logits) | |
| def _block_calls(self): | |
| prelude = len(self.prelude) if hasattr(self, "prelude") else 0 | |
| coda = len(self.coda) if hasattr(self, "coda") else 0 | |
| return prelude + self.num_loops * len(self.layers) + coda | |
| def generate(self, input_ids, max_new_tokens=64, do_sample=True, temperature=1.0, | |
| top_k=0, top_p=1.0, repetition_penalty=1.0, eos_token_id=None, bad_words_ids=None, | |
| use_cache=True, num_return_sequences=1, return_logprobs=False, pad_token_id=None, **kwargs): | |
| """Sample new tokens. | |
| With use_cache, the prompt runs once and each step then feeds only the | |
| newest token. This gives the same logits as the uncached path while the | |
| sequence fits in max_position_embeddings. After that, the cached path | |
| keeps streaming (attention is local and RoPE is relative), but the | |
| uncached path drops the oldest tokens and recomputes the memory. | |
| num_return_sequences samples a group of continuations of one prompt | |
| (input_ids must have batch size 1). The prompt runs once and its cache | |
| is copied for each row. All rows stay at the same position, so their | |
| memory chunks complete on the same step. A row stops at a stop token; | |
| after that it gets pad_token_id (or the first stop token) until all | |
| rows stop. | |
| With return_logprobs, the result is (tokens, logprobs). logprobs has one | |
| value per new token: the log-probability of that token under the raw | |
| model distribution (temperature 1, no top-k, top-p or penalty). It is | |
| 0 for padding after a row stops. | |
| """ | |
| stop_list = list(eos_token_id) if isinstance(eos_token_id, (list, tuple, set)) else ([eos_token_id] if eos_token_id is not None else []) | |
| bad_ids = [item[0] for item in bad_words_ids] if bad_words_ids else [] | |
| group = num_return_sequences | |
| if group > 1 and input_ids.shape[0] != 1: | |
| raise ValueError("num_return_sequences > 1 needs a batch of one prompt") | |
| fill = pad_token_id if pad_token_id is not None else (stop_list[0] if stop_list else 0) | |
| stop_ids = torch.tensor(stop_list, device=input_ids.device, dtype=input_ids.dtype) | |
| max_ctx = self.config.max_position_embeddings | |
| cache = {} if use_cache else None | |
| tokens = input_ids | |
| if cache is not None: | |
| logits = self(tokens[:, -max_ctx:], cache=cache, logits_to_keep=1).logits[:, -1, :] | |
| if group > 1: | |
| cache = _repeat_batch(cache, group) | |
| logits = logits.repeat(group, 1) | |
| tokens = tokens.repeat(group, 1) | |
| finished = torch.zeros(tokens.shape[0], dtype=torch.bool, device=tokens.device) | |
| logprobs = [] | |
| for step in range(max_new_tokens): | |
| if cache is None: | |
| logits = self(tokens[:, -max_ctx:]).logits[:, -1, :] | |
| elif step > 0: | |
| logits = self(tokens[:, -1:], cache=cache, logits_to_keep=1).logits[:, -1, :] | |
| raw = logits.float().log_softmax(dim=-1) if return_logprobs else None | |
| if repetition_penalty != 1.0: | |
| score = torch.gather(logits, 1, tokens) | |
| score = torch.where(score < 0, score * repetition_penalty, score / repetition_penalty) | |
| logits = logits.scatter(1, tokens, score) | |
| if bad_ids: | |
| logits[:, bad_ids] = float("-inf") | |
| if not do_sample: | |
| next_tok = logits.argmax(dim=-1, keepdim=True) | |
| else: | |
| logits = logits / temperature | |
| if top_k > 0: | |
| v, _ = torch.topk(logits, min(top_k, logits.size(-1))) | |
| logits = logits.masked_fill(logits < v[:, [-1]], float("-inf")) | |
| if top_p < 1.0: | |
| sorted_logits, sorted_idx = torch.sort(logits, descending=True) | |
| probs = F.softmax(sorted_logits, dim=-1) | |
| remove = torch.cumsum(probs, dim=-1) - probs > top_p | |
| sorted_logits = sorted_logits.masked_fill(remove, float("-inf")) | |
| logits = logits.scatter(1, sorted_idx, sorted_logits) | |
| next_tok = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1) | |
| next_tok = next_tok.masked_fill(finished[:, None], fill) | |
| if return_logprobs: | |
| logprobs.append(raw.gather(1, next_tok).squeeze(1).masked_fill(finished, 0.0)) | |
| tokens = torch.cat([tokens, next_tok], dim=-1) | |
| finished = finished | torch.isin(next_tok[:, 0], stop_ids) | |
| if bool(finished.all()): | |
| break | |
| if return_logprobs: | |
| empty = tokens.new_zeros(tokens.shape[0], 0, dtype=torch.float32) | |
| return tokens, torch.stack(logprobs, dim=1) if logprobs else empty | |
| return tokens | |
| def _repeat_batch(cache, n): | |
| """Copy a batch-1 decode cache n times along the batch dimension. | |
| Memory tensors use a (batch * heads) leading dimension in batch-major order, | |
| so tiling a batch-1 cache with repeat gives the right layout for both kinds. | |
| """ | |
| if isinstance(cache, torch.Tensor): | |
| return cache.repeat(n, *([1] * (cache.ndim - 1))) | |
| if isinstance(cache, dict): | |
| return {key: _repeat_batch(value, n) for key, value in cache.items()} | |
| if isinstance(cache, (list, tuple)): | |
| return type(cache)(_repeat_batch(value, n) for value in cache) | |
| return cache | |