Fix warm-cache new token blocks in generation preparation

#11
by Qubitium - opened
Files changed (1) hide show
  1. modeling_ricursive.py +20 -11
modeling_ricursive.py CHANGED
@@ -908,17 +908,26 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
908
  past_length = _cache_seq_length(past_key_values)
909
 
910
  if cache_position is None:
911
- # With a non-empty cache and the full sequence supplied, only the trailing
912
- # (uncached) tokens are new. Otherwise every token in `input_ids` is new.
913
- if past_length > 0 and input_ids.shape[1] > 1:
914
- new_len = input_ids.shape[1] - past_length
915
- cache_position = torch.arange(
916
- past_length, past_length + new_len, device=input_ids.device
917
- )
918
- else:
919
- cache_position = torch.arange(
920
- past_length, past_length + input_ids.shape[1], device=input_ids.device
921
- )
 
 
 
 
 
 
 
 
 
922
 
923
  # Only slice when the cache is warm and `input_ids` still carries cached
924
  # tokens. Slicing unconditionally turned a `None` cache position into
 
908
  past_length = _cache_seq_length(past_key_values)
909
 
910
  if cache_position is None:
911
+ # A full prompt can include cached tokens, while verification may pass
912
+ # only a new token block. A block no longer than the cache must be new;
913
+ # for a longer block, a mask with exactly the cached prefix plus new
914
+ # tokens identifies new-only input when its final slot is populated.
915
+ # A longer mask, or one with unused capacity at the end, may be padded
916
+ # to a static cache's capacity and describes a full prompt instead.
917
+ input_length = input_ids.shape[1]
918
+ candidate_full_prompt = past_length > 0 and input_length > past_length
919
+ new_block_mask = (
920
+ candidate_full_prompt
921
+ and attention_mask is not None
922
+ and attention_mask.dim() == 2
923
+ and attention_mask.shape[-1] == past_length + input_length
924
+ and bool(attention_mask[..., -1].any())
925
+ )
926
+ full_prompt = candidate_full_prompt and not new_block_mask
927
+ new_len = input_length - past_length if full_prompt else input_length
928
+ cache_position = torch.arange(
929
+ past_length, past_length + new_len, device=input_ids.device
930
+ )
931
 
932
  # Only slice when the cache is warm and `input_ids` still carries cached
933
  # tokens. Slicing unconditionally turned a `None` cache position into