File size: 22,872 Bytes
90d354b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
"""PyTorch twin of the MLX decision path, for cloud GPUs. Same prompt, same decision position, same float32 candidate head."""
from pathlib import Path
import json
import torch
from vision_decision.scoring import MAX_READOUT_CODES, DEFAULT_PROMPT_LAYOUT, check_prompt_layout, check_readout_codes, readout_codes, verified_label_ids

TARGETS=['q_proj','k_proj','v_proj','o_proj','gate_proj','up_proj','down_proj','in_proj_qkv','in_proj_z','out_proj']

class _LazyMultimodalProcessor:
 """Text requests need only the tokenizer; load optional vision/video processors on demand."""
 def __init__(self,path):
  from transformers import AutoTokenizer
  self.path=path;self.tokenizer=AutoTokenizer.from_pretrained(path,local_files_only=True);self._multimodal=None
 def _vision(self):
  if self._multimodal is None:
   from transformers import AutoProcessor
   self._multimodal=AutoProcessor.from_pretrained(self.path,local_files_only=True)
   self._multimodal.tokenizer=self.tokenizer  # preserve padding-side changes made by collate()
  return self._multimodal
 def apply_chat_template(self,messages,**kwargs):
  if any(item.get('type') in ('image','video') for message in messages for item in message.get('content',[]) if isinstance(item,dict)):
   return self._vision().apply_chat_template(messages,**kwargs)
  return self.tokenizer.apply_chat_template(messages,**kwargs)
 def __call__(self,text,images=None,**kwargs):
  if images is not None:return self._vision()(text=text,images=images,**kwargs)
  kwargs.setdefault('add_special_tokens',False)
  return self.tokenizer(text,**kwargs)
 def __getattr__(self,name):
  if name.startswith('_'):raise AttributeError(name)
  return getattr(self._vision(),name)

class TorchDecision:
 def __init__(self,path,device,dtype=torch.bfloat16,max_length=4096,pad_multiple=0):
  from transformers import Qwen3_5ForConditionalGeneration
  self.processor=_LazyMultimodalProcessor(path);self.device=device;self.max_length=max_length;self.pad_multiple=pad_multiple  # pad batches to a multiple (fewer distinct kernel shapes)
  self.model=Qwen3_5ForConditionalGeneration.from_pretrained(path,local_files_only=True,dtype=dtype).to(device).eval()
  self.readout=None;self._codebook=None
  self.codes=MAX_READOUT_CODES  # readout size: 255 shipped; 256 with the extended readout (enable_readout(codes=256))
  self.prompt_layout=DEFAULT_PROMPT_LAYOUT  # set from the adapter's decision_readout.json, or by the trainer
 def add_lora(self,rank=16,alpha=32,targets=None):
  from peft import LoraConfig,get_peft_model
  # Language layers only, as in the MLX run; the vision tower stays frozen. `targets` narrows the module list (e.g. attention only).
  self.model=get_peft_model(self.model,LoraConfig(r=rank,lora_alpha=alpha,lora_dropout=0.0,target_modules=r'.*language_model.*\.('+'|'.join(targets or TARGETS)+')'))
  return self.model
 def _ensure_codebook(self,n_images=0):
  if self._codebook is None or len(self._codebook)!=self.codes:self._codebook=readout_codes(self.processor.tokenizer,self.render('',n_images),self.codes,limit=self.codes)
  return self._codebook
 def labels(self,count,n_images=0):
  codebook=self._ensure_codebook(n_images)
  if count>len(codebook):raise ValueError(f'{count} candidates exceed the {len(codebook)}-code readout (at most {len(codebook)-1} options + unknown)')
  return [x[0] for x in codebook[:count]]
 @property
 def max_options(self):return self.codes-1
 def enable_readout(self,adapter=None,trainable=True,codes=None):
  """Install the decision readout Linear[codes, hidden], initialized from the matching LM-head rows.



  codes: 255 (shipped) or 256 (extended); None = the adapter's own row count (255 without an adapter).

  A 255-row trained readout loaded with codes=256 gets row 256 appended from its LM-head row, exactly as

  the original rows were initialized, so questions with <= 254 options use unchanged rows.

  The adapter's prompt layout (decision_readout.json "prompt_layout", absent = standard) is adopted.

  Older adapters omit decision_readout.safetensors and continue through vocabulary rows.

  """
  from safetensors.torch import load_file
  base=self.model.get_base_model() if hasattr(self.model,'get_base_model') else self.model
  trained=manifest=None
  if adapter is not None:
   path=Path(adapter)/'decision_readout.safetensors'
   if not path.exists():return False
   trained=load_file(str(path),device=str(self.device))['weight'].float()
   manifest_path=Path(adapter)/'decision_readout.json'
   if not manifest_path.exists():raise ValueError('Trained readout is missing decision_readout.json tokenizer binding')
   manifest=json.loads(manifest_path.read_text())
  rows=trained.shape[0] if trained is not None and trained.ndim==2 else None
  wanted=check_readout_codes(codes if codes is not None else rows if rows in (255,256) else MAX_READOUT_CODES)
  if trained is not None and rows not in (255,256):raise ValueError('Decision readout must be finite with shape [255 or 256, hidden_size]')
  if rows is not None and rows>wanted:raise ValueError(f'This adapter has a {rows}-code readout; load it with codes={rows}')
  self.codes=wanted;self._codebook=None;codebook=self._ensure_codebook(0)
  ids=[x[1] for x in codebook]
  weight=base.lm_head.weight[torch.tensor(ids,device=self.device)].detach().float().clone()
  if trained is not None:
   actual=[{'code':c,'token_id':i} for c,i in codebook]
   bound=manifest.get('codes')
   if manifest.get('version')!=1 or not isinstance(bound,list) or len(bound)!=rows or bound!=actual[:rows]:raise ValueError('Decision readout code/token binding does not match this tokenizer')
   weight=torch.cat([trained,weight[rows:]]) if rows<wanted else trained  # appended rows: LM-head rows, as at initialization
   self.prompt_layout=check_prompt_layout(manifest.get('prompt_layout',DEFAULT_PROMPT_LAYOUT))
  if tuple(weight.shape)!=(self.codes,base.lm_head.weight.shape[1]) or not bool(torch.isfinite(weight).all()):raise ValueError(f'Decision readout must be finite with shape [{self.codes}, hidden_size]')
  self.readout=torch.nn.Linear(weight.shape[1],self.codes,bias=False,device=self.device,dtype=torch.float32)
  self.readout.weight.data.copy_(weight);self.readout.weight.requires_grad_(trainable)
  return True
 def save_readout(self,directory):
  if self.readout is None:raise ValueError('Decision readout is not enabled')
  from safetensors.torch import save_file
  save_file({'weight':self.readout.weight.detach().cpu().contiguous()},str(Path(directory)/'decision_readout.safetensors'))
  manifest={'version':1,'codes':[{'code':c,'token_id':i} for c,i in self._codebook]}
  if self.prompt_layout!=DEFAULT_PROMPT_LAYOUT:manifest['prompt_layout']=check_prompt_layout(self.prompt_layout)  # absent = standard, so shipped adapters stay byte-identical
  (Path(directory)/'decision_readout.json').write_text(json.dumps(manifest,indent=2)+'\n')
 def render(self,prompt,n_images):
  messages=[dict(role='user',content=[dict(type='image')]*n_images+[dict(type='text',text=prompt)])]
  rendered=self.processor.apply_chat_template(messages,add_generation_prompt=True,tokenize=False,enable_thinking=False)
  if not rendered.endswith('<think>\n\n</think>\n\n'):raise ValueError('Unexpected Qwen non-thinking template boundary')
  return rendered
 def prepare(self,images,prompt,labels):
  images=[] if images is None else images if isinstance(images,list) else [images];rendered=self.render(prompt,len(images))
  token_ids=verified_label_ids(self.processor.tokenizer,rendered,labels)
  inputs=self.processor(text=[rendered],images=images or None,return_tensors='pt')
  if inputs['input_ids'].shape[-1]>self.max_length:raise ValueError(f'Processed request exceeds the {self.max_length}-token limit')
  suffix=self.processor.tokenizer.encode('</think>\n\n',add_special_tokens=False)
  if inputs['input_ids'][0,-len(suffix):].tolist()!=suffix:raise ValueError('Processed decision-position suffix mismatch')
  return rendered,inputs,token_ids
 def candidate_logits(self,inputs,token_ids):
  inputs={k:v.to(self.device) for k,v in inputs.items()};base=self.model.get_base_model() if hasattr(self.model,'get_base_model') else self.model
  hidden=base.model(**inputs).last_hidden_state[0,-1]
  indices=self._readout_indices(token_ids)
  return self.readout(hidden.float()) [indices] if self.readout is not None and indices is not None else hidden.float()@base.lm_head.weight[torch.tensor(token_ids,device=self.device)].float().T
 def _readout_indices(self,token_ids):
  if self._codebook is None:return None
  lookup={token:i for i,(_,token) in enumerate(self._codebook)}
  return [lookup[x] for x in token_ids] if all(x in lookup for x in token_ids) else None

 # ---- batched path (keeps large GPUs busy): left padding puts every decision position at index -1 ----
 def render_example(self,images,prompt,labels):
  """CPU-only half of prepare(): safe to run in DataLoader workers."""
  images=[] if images is None else images if isinstance(images,list) else [images];rendered=self.render(prompt,len(images))
  return rendered,images,verified_label_ids(self.processor.tokenizer,rendered,labels)
 def collate(self,examples):
  """examples: (rendered, images, token_ids, target). Returns model inputs plus per-example label ids and targets."""
  self.processor.tokenizer.padding_side='left'
  image_groups=[e[1] for e in examples]
  images=None if all(not group for group in image_groups) else image_groups
  inputs=self.processor(text=[e[0] for e in examples],images=images,return_tensors='pt',padding=True,**({'pad_to_multiple_of':self.pad_multiple} if self.pad_multiple else {}))
  if inputs['input_ids'].shape[-1]>self.max_length:raise ValueError(f'Processed request exceeds the {self.max_length}-token limit')
  suffix=self.processor.tokenizer.encode('</think>\n\n',add_special_tokens=False)
  if not all(row[-len(suffix):].tolist()==suffix for row in inputs['input_ids']):raise ValueError('Left padding did not align the decision positions')
  return inputs,[e[2] for e in examples],[e[3] for e in examples]
 def candidate_logits_batch(self,inputs,token_ids):
  inputs={k:v.to(self.device) for k,v in inputs.items()};base=self.model.get_base_model() if hasattr(self.model,'get_base_model') else self.model
  hidden=base.model(**inputs).last_hidden_state[:,-1].float();head=base.lm_head.weight
  result=[]
  for i,ids in enumerate(token_ids):
   indices=self._readout_indices(ids)
   result.append(self.readout(hidden[i])[indices] if self.readout is not None and indices is not None else hidden[i]@head[torch.tensor(ids,device=self.device)].float().T)
  return result
 def candidate_logits_with_rationale(self,inputs,token_ids,position,labels):
  """--rationale-weight path: candidate logits read at `position` (the decision position, unchanged by the

  appended rationale block under causal attention) plus the per-example rationale LM loss [B]."""
  from decision_recipe import rationale_lm_loss
  inputs={k:v.to(self.device) for k,v in inputs.items()};base=self.model.get_base_model() if hasattr(self.model,'get_base_model') else self.model
  states=base.model(**inputs).last_hidden_state;hidden=states[:,position].float();head=base.lm_head.weight
  result=[]
  for i,ids in enumerate(token_ids):
   indices=self._readout_indices(ids)
   result.append(self.readout(hidden[i])[indices] if self.readout is not None and indices is not None else hidden[i]@head[torch.tensor(ids,device=self.device)].float().T)
  return result,rationale_lm_loss(states,head,labels,position+1)

 # ---- serving fast path (server --fast): same decision function, less work per request ----
 def merge_adapter(self,adapter,dtype=torch.bfloat16):
  """Fold a PEFT LoRA into the weights so every forward is a plain base-model forward. Call on a float32 model: the

  low-rank sum is formed in float32 and rounded once to `dtype`. The decision readout is separate and stays float32."""
  from peft import PeftModel
  self.model=PeftModel.from_pretrained(self.model,str(adapter)).merge_and_unload().to(dtype).eval()
 DECISION_TAIL='</think>\n\n'
 def label_ids(self,rendered,labels):
  """= verified_label_ids(tokenizer, rendered, labels), with one tokenization per label for the life of the process.

  The rendered prompt ends with the special token '</think>' then '\\n\\n'; the tokenizer splits at special tokens, so

  whether a label is one token at the decision position depends only on that tail (test_torch_fast_path checks this)."""
  if not rendered.endswith(self.DECISION_TAIL):return verified_label_ids(self.processor.tokenizer,rendered,labels)
  cache=self.__dict__.setdefault('_label_cache',{})
  for label in labels:
   if label not in cache:cache[label]=verified_label_ids(self.processor.tokenizer,self.DECISION_TAIL,[label])[0]
  ids=[cache[label] for label in labels]
  if len(set(ids))!=len(ids):raise ValueError('Choice labels do not have distinct token IDs')
  return ids
 def prepare_fast(self,images,prompt,labels):
  """prepare() with one tokenization: text-only requests skip the multimodal processor (same ids, checked by the test)."""
  images=[] if images is None else images if isinstance(images,list) else [images];rendered=self.render(prompt,len(images))
  token_ids=self.label_ids(rendered,labels)
  # Images: the processor resizes and patchifies as always but leaves the pixels as uint8; normalize_patches finishes the
  # identical float32 arithmetic on the GPU (8x fewer bytes to copy, no float work on the CPU).
  inputs=dict(self.processor(text=[rendered],images=images,return_tensors='pt',do_rescale=False,do_normalize=False)) if images else {'input_ids':self.processor.tokenizer(rendered,add_special_tokens=False,return_tensors='pt')['input_ids']}
  if inputs['input_ids'].shape[-1]>self.max_length:raise ValueError(f'Processed request exceeds the {self.max_length}-token limit')
  suffix=self.__dict__.get('_suffix_ids') or self.__dict__.setdefault('_suffix_ids',self.processor.tokenizer.encode(self.DECISION_TAIL,add_special_tokens=False))
  if inputs['input_ids'][0,-len(suffix):].tolist()!=suffix:raise ValueError('Processed decision-position suffix mismatch')
  return rendered,inputs,token_ids
 def normalize_patches(self,patches):
  """The image processor's rescale_and_normalize on already patchified uint8 pixels: one fused mean and std per channel,

  then x.float() - mean, / std, exactly the torchvision normalize the processor runs before patchify. Elementwise, so

  patchify first changes nothing; each patch row is channel-major (channel x temporal x patch x patch)."""
  ip=self.processor.image_processor
  mean,std,_=ip._fuse_mean_std_and_rescale_factor(do_normalize=True,image_mean=ip.image_mean,image_std=ip.image_std,do_rescale=True,rescale_factor=ip.rescale_factor,device=patches.device)
  per=ip.temporal_patch_size*ip.patch_size*ip.patch_size
  mean=torch.as_tensor(mean,dtype=torch.float32,device=patches.device).repeat_interleave(per);std=torch.as_tensor(std,dtype=torch.float32,device=patches.device).repeat_interleave(per)
  return patches.to(torch.float32).sub_(mean).div_(std)
 def _base(self):
  """The Qwen3.5 model under a PEFT wrapper (its LoRA layers stay in place) or the merged model itself."""
  return self.model.get_base_model() if hasattr(self.model,'get_base_model') else self.model
 def _embeds_positions(self,inputs):
  """The multimodal model's forward up to the language model: token embeddings with the image features scattered in, and the

  M-RoPE position ids (3 x 1 x n). Text-only positions are 0..n-1 on every axis, as the model's own default."""
  m=self._base().model;ids=inputs['input_ids'];embeds=m.get_input_embeddings()(ids)
  if 'pixel_values' not in inputs:return embeds,torch.arange(ids.shape[1],device=ids.device).view(1,1,-1).expand(3,1,-1)
  features=torch.cat(m.get_image_features(inputs['pixel_values'],inputs['image_grid_thw'],return_dict=True).pooler_output,dim=0).to(embeds.device,embeds.dtype)
  mask,_=m.get_placeholder_mask(ids,inputs_embeds=embeds,image_features=features);embeds=embeds.masked_scatter(mask,features)
  positions=m.compute_3d_position_ids(input_ids=ids,inputs_embeds=embeds,image_grid_thw=inputs['image_grid_thw'],attention_mask=inputs.get('attention_mask'),mm_token_type_ids=inputs.get('mm_token_type_ids'))
  return embeds,positions
 def candidate_logits_fast(self,inputs,token_ids):
  """candidate_logits(), through a recorded CUDA graph when one fits the prompt length."""
  inputs={k:v.to(self.device) for k,v in inputs.items()};base=self._base()
  if inputs.get('pixel_values') is not None and inputs['pixel_values'].dtype==torch.uint8:inputs['pixel_values']=self.normalize_patches(inputs['pixel_values'])
  embeds,positions=self._embeds_positions(inputs);graphs=self.__dict__.get('graphs')
  hidden=graphs.run(embeds,positions) if graphs is not None and graphs.fits(embeds.shape[1]) else base.model.language_model(inputs_embeds=embeds,position_ids=positions,use_cache=False).last_hidden_state[0,-1]
  indices=self._readout_indices(token_ids)
  return self.readout(hidden.float())[indices] if self.readout is not None and indices is not None else hidden.float()@base.lm_head.weight[torch.tensor(token_ids,device=self.device)].float().T
 def capture_graphs(self,lengths):
  base=self._base()
  self.graphs=DecoderGraphs(base.model.language_model,base.config.text_config.hidden_size,lengths,self.device,base.dtype)
  return self.graphs

 # ---- thinking (think-if-unsure): a greedy thought in Qwen3.5's thinking template, the decision read right after it ----
 THINK_TAIL='<think>\n'
 def render_thinking(self,prompt,n_images):
  messages=[dict(role='user',content=[dict(type='image')]*n_images+[dict(type='text',text=prompt)])]
  rendered=self.processor.apply_chat_template(messages,add_generation_prompt=True,tokenize=False,enable_thinking=True)
  if not rendered.endswith(self.THINK_TAIL):raise ValueError('Unexpected Qwen thinking template boundary')
  return rendered
 def _thinking_inputs(self,images,prompt):
  images=[] if images is None else images if isinstance(images,list) else [images]
  inputs=dict(self.processor(text=[self.render_thinking(prompt,len(images))],images=images or None,return_tensors='pt'))
  if inputs['input_ids'].shape[-1]>self.max_length:raise ValueError(f'Processed request exceeds the {self.max_length}-token limit')
  return inputs
 def think_end_id(self):
  ids=self.processor.tokenizer.encode('</think>',add_special_tokens=False)
  if len(ids)!=1:raise ValueError("'</think>' is not a single token")
  return ids[0]
 @torch.inference_mode()
 def generate_thought(self,images,prompt,max_tokens):
  """Greedy thought token ids (without '</think>') and whether the model closed the thought itself."""
  tok=self.processor.tokenizer;end=self.think_end_id();pad=tok.pad_token_id if tok.pad_token_id is not None else tok.eos_token_id
  inputs={k:v.to(self.device) for k,v in self._thinking_inputs(images,prompt).items()}
  out=self.model.generate(**inputs,max_new_tokens=int(max_tokens),do_sample=False,eos_token_id=[end,tok.eos_token_id],pad_token_id=pad)
  tokens=[]
  for token in out[0,inputs['input_ids'].shape[-1]:].tolist():
   if token==end:return tokens,True
   if token in (tok.eos_token_id,pad):break
   tokens.append(token)
  return tokens,False
 def inputs_after_thought(self,images,prompt,labels,thought,closed):
  """Model inputs ending '</think>\\n\\n' after `thought` (token ids; a cut thought is closed with '\\n</think>\\n\\n'),

  and the labels' token ids at that decision position. Same image, prompt and decision position as the single pass."""
  images=[] if images is None else images if isinstance(images,list) else [images]
  inputs=self._thinking_inputs(images,prompt);tok=self.processor.tokenizer
  tail=tok.encode('</think>\n\n' if closed else '\n</think>\n\n',add_special_tokens=False)
  extra=torch.tensor([list(thought)+tail],dtype=inputs['input_ids'].dtype)
  inputs['input_ids']=torch.cat([inputs['input_ids'],extra],dim=1)
  if 'attention_mask' in inputs:inputs['attention_mask']=torch.cat([inputs['attention_mask'],torch.ones_like(extra)],dim=1)
  if 'mm_token_type_ids' in inputs:inputs['mm_token_type_ids']=torch.cat([inputs['mm_token_type_ids'],torch.zeros_like(extra,dtype=inputs['mm_token_type_ids'].dtype)],dim=1)
  if inputs['input_ids'].shape[-1]>self.max_length:raise ValueError(f'Processed request exceeds the {self.max_length}-token limit')
  suffix=tok.encode(self.DECISION_TAIL,add_special_tokens=False)
  if inputs['input_ids'][0,-len(suffix):].tolist()!=suffix:raise ValueError('Decision-position suffix mismatch after the thought')
  return inputs,self.label_ids(self.DECISION_TAIL,labels)

GRAPH_LENGTHS=list(range(64,1025,64))+list(range(1152,2049,128))+list(range(2304,4097,256))

class DecoderGraphs:
 """CUDA graphs of the language model at fixed padded lengths, recorded once at load (the design of JevK5's runtime,

 Apache-2.0, also used by FlyMy's Decision 4B). A prompt is right-padded to the next recorded length: every layer is

 causal (full attention, gated DeltaNet, its short convolution), so padding after the decision position cannot change it."""
 def __init__(self,lm,hidden,lengths,device,dtype):
  self.graphs={};pool=None
  with torch.inference_mode():
   for n in sorted(set(lengths),reverse=True):  # longest first: the shorter graphs reuse its memory pool
    embeds=torch.zeros(1,n,hidden,device=device,dtype=dtype);positions=torch.arange(n,device=device).view(1,1,n).expand(3,1,n).contiguous()
    last=torch.zeros(1,dtype=torch.long,device=device)
    def step(embeds=embeds,positions=positions,last=last):
     return lm(inputs_embeds=embeds,position_ids=positions,use_cache=False).last_hidden_state[0].index_select(0,last)[0]
    side=torch.cuda.Stream();side.wait_stream(torch.cuda.current_stream())
    with torch.cuda.stream(side):
     for _ in range(3):step()  # warm-up: compiles the Triton kernels outside the capture
    torch.cuda.current_stream().wait_stream(side)
    graph=torch.cuda.CUDAGraph()
    with torch.cuda.graph(graph,pool=pool):out=step()
    pool=graph.pool();self.graphs[n]=(graph,embeds,positions,last,out)
  self.lengths=sorted(self.graphs)
 def fits(self,n):return bool(self.lengths) and n<=self.lengths[-1]
 def run(self,embeds,positions):
  n=embeds.shape[1];size=next(x for x in self.lengths if x>=n);graph,static_embeds,static_positions,last,out=self.graphs[size]
  static_embeds.zero_();static_embeds[:,:n].copy_(embeds)
  static_positions[:,:,:n].copy_(positions);static_positions[:,:,n:].copy_(positions[:,:,-1:]+torch.arange(1,size-n+1,device=positions.device))
  last.fill_(n-1);graph.replay()
  return out.clone()