Fix warm-cache new token blocks in generation preparation
#11
by Qubitium - opened
- modeling_ricursive.py +20 -11
modeling_ricursive.py
CHANGED
|
@@ -908,17 +908,26 @@ class RicursiveForCausalLM(RicursivePreTrainedModel, GenerationMixin):
|
|
| 908 |
past_length = _cache_seq_length(past_key_values)
|
| 909 |
|
| 910 |
if cache_position is None:
|
| 911 |
-
#
|
| 912 |
-
#
|
| 913 |
-
|
| 914 |
-
|
| 915 |
-
|
| 916 |
-
|
| 917 |
-
|
| 918 |
-
|
| 919 |
-
|
| 920 |
-
|
| 921 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 922 |
|
| 923 |
# Only slice when the cache is warm and `input_ids` still carries cached
|
| 924 |
# tokens. Slicing unconditionally turned a `None` cache position into
|
|
|
|
| 908 |
past_length = _cache_seq_length(past_key_values)
|
| 909 |
|
| 910 |
if cache_position is None:
|
| 911 |
+
# A full prompt can include cached tokens, while verification may pass
|
| 912 |
+
# only a new token block. A block no longer than the cache must be new;
|
| 913 |
+
# for a longer block, a mask with exactly the cached prefix plus new
|
| 914 |
+
# tokens identifies new-only input when its final slot is populated.
|
| 915 |
+
# A longer mask, or one with unused capacity at the end, may be padded
|
| 916 |
+
# to a static cache's capacity and describes a full prompt instead.
|
| 917 |
+
input_length = input_ids.shape[1]
|
| 918 |
+
candidate_full_prompt = past_length > 0 and input_length > past_length
|
| 919 |
+
new_block_mask = (
|
| 920 |
+
candidate_full_prompt
|
| 921 |
+
and attention_mask is not None
|
| 922 |
+
and attention_mask.dim() == 2
|
| 923 |
+
and attention_mask.shape[-1] == past_length + input_length
|
| 924 |
+
and bool(attention_mask[..., -1].any())
|
| 925 |
+
)
|
| 926 |
+
full_prompt = candidate_full_prompt and not new_block_mask
|
| 927 |
+
new_len = input_length - past_length if full_prompt else input_length
|
| 928 |
+
cache_position = torch.arange(
|
| 929 |
+
past_length, past_length + new_len, device=input_ids.device
|
| 930 |
+
)
|
| 931 |
|
| 932 |
# Only slice when the cache is warm and `input_ids` still carries cached
|
| 933 |
# tokens. Slicing unconditionally turned a `None` cache position into
|