Andrej Karpathy commited on
Commit ·
baf0b3f
1
Parent(s): f1db6b4
also add a test that failed before the fix and passes now with the fix for kv cache resize
Browse files- tests/test_engine.py +66 -0
tests/test_engine.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Test Engine class. Example run:
|
| 3 |
+
|
| 4 |
+
python -m pytest tests/test_engine.py -v
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from nanochat.engine import KVCache
|
| 9 |
+
|
| 10 |
+
def test_kv_cache_resize():
|
| 11 |
+
"""
|
| 12 |
+
The KV cache was not resized correctly, more information here:
|
| 13 |
+
https://github.com/karpathy/nanochat/pull/186
|
| 14 |
+
This test reproduces the issue and will be merged alongside the fix.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
batch_size = 2
|
| 18 |
+
num_heads = 3
|
| 19 |
+
seq_len = 4
|
| 20 |
+
head_dim = 5
|
| 21 |
+
num_layers = 6
|
| 22 |
+
|
| 23 |
+
kv_cache = KVCache(
|
| 24 |
+
batch_size=batch_size,
|
| 25 |
+
num_heads=num_heads,
|
| 26 |
+
seq_len=seq_len,
|
| 27 |
+
head_dim=head_dim,
|
| 28 |
+
num_layers=num_layers
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
# Insert a single token with a distinct fill value to all layers
|
| 32 |
+
def insert_token(token_idx):
|
| 33 |
+
for layer_idx in range(num_layers):
|
| 34 |
+
k = torch.full((batch_size, num_heads, 1, head_dim), fill_value=float(token_idx), dtype=torch.float32)
|
| 35 |
+
v = torch.full((batch_size, num_heads, 1, head_dim), fill_value=float(token_idx * 100), dtype=torch.float32)
|
| 36 |
+
kv_cache.insert_kv(layer_idx, k, v)
|
| 37 |
+
|
| 38 |
+
# Insert 4 tokens (fills the initial seq_len=4)
|
| 39 |
+
for i in range(4):
|
| 40 |
+
insert_token(i)
|
| 41 |
+
|
| 42 |
+
# Record the original state of the cache
|
| 43 |
+
original_cache = kv_cache.kv_cache.clone()
|
| 44 |
+
original_seq_len = original_cache.shape[4]
|
| 45 |
+
|
| 46 |
+
# Insert the 5th token, which will trigger a resize
|
| 47 |
+
insert_token(4)
|
| 48 |
+
# Verify that the cache actually resized
|
| 49 |
+
new_seq_len = kv_cache.kv_cache.shape[4]
|
| 50 |
+
assert new_seq_len > original_seq_len, f"Cache did not resize: original seq_len={original_seq_len}, new seq_len={new_seq_len}"
|
| 51 |
+
|
| 52 |
+
# Verify that the original 4 tokens are still intact after resize
|
| 53 |
+
for layer_idx in range(num_layers):
|
| 54 |
+
for token_idx in range(4):
|
| 55 |
+
# Check that resized cache matches expected values
|
| 56 |
+
expected_k = float(token_idx)
|
| 57 |
+
expected_v = float(token_idx * 100)
|
| 58 |
+
actual_k = kv_cache.kv_cache[layer_idx, 0, :, :, token_idx, :]
|
| 59 |
+
actual_v = kv_cache.kv_cache[layer_idx, 1, :, :, token_idx, :]
|
| 60 |
+
assert (actual_k == expected_k).all(), f"Layer {layer_idx}, token {token_idx}: key corrupted, expected {expected_k}"
|
| 61 |
+
assert (actual_v == expected_v).all(), f"Layer {layer_idx}, token {token_idx}: value corrupted, expected {expected_v}"
|
| 62 |
+
# And that the original cache matches resized cache
|
| 63 |
+
original_k = original_cache[layer_idx, 0, :, :, token_idx, :]
|
| 64 |
+
original_v = original_cache[layer_idx, 1, :, :, token_idx, :]
|
| 65 |
+
assert (actual_k == original_k).all(), f"Layer {layer_idx}, token {token_idx}: key doesn't match original"
|
| 66 |
+
assert (actual_v == original_v).all(), f"Layer {layer_idx}, token {token_idx}: value doesn't match original"
|