Andrej Karpathy commited on
Commit
baf0b3f
·
1 Parent(s): f1db6b4

also add a test that failed before the fix and passes now with the fix for kv cache resize

Browse files
Files changed (1) hide show
  1. tests/test_engine.py +66 -0
tests/test_engine.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Test Engine class. Example run:
3
+
4
+ python -m pytest tests/test_engine.py -v
5
+ """
6
+
7
+ import torch
8
+ from nanochat.engine import KVCache
9
+
10
+ def test_kv_cache_resize():
11
+ """
12
+ The KV cache was not resized correctly, more information here:
13
+ https://github.com/karpathy/nanochat/pull/186
14
+ This test reproduces the issue and will be merged alongside the fix.
15
+ """
16
+
17
+ batch_size = 2
18
+ num_heads = 3
19
+ seq_len = 4
20
+ head_dim = 5
21
+ num_layers = 6
22
+
23
+ kv_cache = KVCache(
24
+ batch_size=batch_size,
25
+ num_heads=num_heads,
26
+ seq_len=seq_len,
27
+ head_dim=head_dim,
28
+ num_layers=num_layers
29
+ )
30
+
31
+ # Insert a single token with a distinct fill value to all layers
32
+ def insert_token(token_idx):
33
+ for layer_idx in range(num_layers):
34
+ k = torch.full((batch_size, num_heads, 1, head_dim), fill_value=float(token_idx), dtype=torch.float32)
35
+ v = torch.full((batch_size, num_heads, 1, head_dim), fill_value=float(token_idx * 100), dtype=torch.float32)
36
+ kv_cache.insert_kv(layer_idx, k, v)
37
+
38
+ # Insert 4 tokens (fills the initial seq_len=4)
39
+ for i in range(4):
40
+ insert_token(i)
41
+
42
+ # Record the original state of the cache
43
+ original_cache = kv_cache.kv_cache.clone()
44
+ original_seq_len = original_cache.shape[4]
45
+
46
+ # Insert the 5th token, which will trigger a resize
47
+ insert_token(4)
48
+ # Verify that the cache actually resized
49
+ new_seq_len = kv_cache.kv_cache.shape[4]
50
+ assert new_seq_len > original_seq_len, f"Cache did not resize: original seq_len={original_seq_len}, new seq_len={new_seq_len}"
51
+
52
+ # Verify that the original 4 tokens are still intact after resize
53
+ for layer_idx in range(num_layers):
54
+ for token_idx in range(4):
55
+ # Check that resized cache matches expected values
56
+ expected_k = float(token_idx)
57
+ expected_v = float(token_idx * 100)
58
+ actual_k = kv_cache.kv_cache[layer_idx, 0, :, :, token_idx, :]
59
+ actual_v = kv_cache.kv_cache[layer_idx, 1, :, :, token_idx, :]
60
+ assert (actual_k == expected_k).all(), f"Layer {layer_idx}, token {token_idx}: key corrupted, expected {expected_k}"
61
+ assert (actual_v == expected_v).all(), f"Layer {layer_idx}, token {token_idx}: value corrupted, expected {expected_v}"
62
+ # And that the original cache matches resized cache
63
+ original_k = original_cache[layer_idx, 0, :, :, token_idx, :]
64
+ original_v = original_cache[layer_idx, 1, :, :, token_idx, :]
65
+ assert (actual_k == original_k).all(), f"Layer {layer_idx}, token {token_idx}: key doesn't match original"
66
+ assert (actual_v == original_v).all(), f"Layer {layer_idx}, token {token_idx}: value doesn't match original"