File size: 5,229 Bytes
79df115 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | from types import SimpleNamespace
import pytest
import torch
from engine import Settings, POLICIES, schedule, prompt_ids, generate, rank_candidates
class Tokenizer:
def apply_chat_template(self, messages, **kwargs):
return torch.tensor([[1, 2, 9]]) # Includes EOS in the prefix on purpose.
def decode(self, ids, **kwargs):
return " ".join(map(str, ids))
class FakeModel:
config = SimpleNamespace(mask_token_id=15, eos_token_id=9, max_sequence_length=4096)
def __init__(self, eos=False):
self.calls = []
self.eos = eos
def __call__(self, x, **kw):
self.calls.append((x.clone(), kw))
b, length = x.shape
logits = torch.zeros(b, length, 16)
logits[..., 4] = 5
if self.eos and not kw.get("update_kv_cache"):
logits[:, 1, 9] = 8
return SimpleNamespace(logits=logits, confidences=torch.arange(length).float()[None])
@pytest.mark.parametrize("length", [1, 7, 16, 31, 32, 64])
@pytest.mark.parametrize("steps", [1, 3, 16, 32, 64])
def test_schedule(length, steps):
counts = schedule(length, steps)
assert len(counts) == steps and counts[-1] == length
assert all(0 <= a <= b <= length for a, b in zip([0] + counts, counts))
@pytest.mark.parametrize("kwargs", [dict(max_tokens=0), dict(max_tokens=257), dict(block_size=7), dict(steps=0), dict(policy="bad"), dict(max_tokens=256, block_size=16, steps=64), dict(seed=1.5)])
def test_bad_settings(kwargs):
with pytest.raises(ValueError):
Settings(**kwargs).validate()
def test_empty_and_long_prompts():
for value in ("", " ", "a" * 16001):
with pytest.raises(ValueError):
prompt_ids(Tokenizer(), value)
class LongTokenizer(Tokenizer):
def apply_chat_template(self, *a, **kw):
return torch.ones(1, 1025, dtype=torch.long)
with pytest.raises(ValueError, match="not truncated"):
prompt_ids(LongTokenizer(), "hello")
@pytest.mark.parametrize("policy", POLICIES)
def test_partial_blocks_no_prompt_eos_stopping(policy):
model = FakeModel()
result = generate(model, Tokenizer(), torch.tensor([[1, 9]]), Settings(35, 16, 3, policy), "cpu")
assert result["metrics"]["output_tokens"] == 35
assert result["metrics"]["finish_reason"] == "length"
assert 15 not in result["token_ids"]
assert len(result["trace"]) == 9
assert result["metrics"]["forward_passes"] == 12 # prefix + 9 denoise + 2 block cache
# Prompt cache is built once, then only completed blocks are appended.
assert [x.shape[1] for x, k in model.calls if k["update_kv_cache"]] == [2, 16, 16]
assert model.calls[-1][1]["position_ids"].tolist() == [[34, 35, 36]]
def test_eos_stops_before_next_block_and_excludes_eos():
model = FakeModel(eos=True)
result = generate(model, Tokenizer(), torch.tensor([[1, 9]]), Settings(64, 16, 2), "cpu")
assert result["token_ids"] == [4]
assert result["metrics"]["finish_reason"] == "eos"
assert len(model.calls) == 3
def test_request_caches_are_isolated():
m = FakeModel()
for _ in range(2):
generate(m, Tokenizer(), torch.tensor([[1]]), Settings(1, 16, 1), "cpu")
assert m.calls[0][1]["kv_cache"] is not m.calls[2][1]["kv_cache"]
def test_context_limit_and_deadline():
m = FakeModel()
with pytest.raises(ValueError, match="context"):
generate(m, Tokenizer(), torch.ones(1, 4096, dtype=torch.long), Settings(), "cpu")
with pytest.raises(TimeoutError):
generate(m, Tokenizer(), torch.tensor([[1]]), Settings(), "cpu", deadline_seconds=-1)
def test_no_trace():
result = generate(FakeModel(), Tokenizer(), torch.tensor([[1]]), Settings(1, 16, 1, trace=False), "cpu")
assert result["trace"] == []
def test_non_finite_rejected():
out = SimpleNamespace(logits=torch.full((1, 2, 4), float("nan")))
with pytest.raises(RuntimeError, match="non-finite"):
rank_candidates(out, torch.ones(1, 2, dtype=torch.long), POLICIES[0])
def test_policy_scores_and_normalisation():
out = SimpleNamespace(logits=torch.tensor([[[1., 3.], [4., 2.]]]), confidences=torch.tensor([[.2, .7]]))
candidates = torch.tensor([[1, 0]])
assert torch.equal(rank_candidates(out, candidates, POLICIES[0]), torch.tensor([[3., 4.]]))
assert torch.equal(rank_candidates(out, candidates, POLICIES[1]), out.confidences)
assert torch.allclose(rank_candidates(out, candidates, POLICIES[2]), torch.tensor([[-.126928, -.126928]]), atol=1e-5)
def test_remasking_revises_previously_kept_position():
class Changing(FakeModel):
def __call__(self, x, **kw):
out = super().__call__(x, **kw)
if not kw["update_kv_cache"]:
n = len([1 for _, k in self.calls if not k["update_kv_cache"]])
out.confidences = torch.zeros(1, x.shape[1])
out.confidences[0, :n] = 10
if n == 2:
out.confidences[0, 0] = -1
out.confidences[0, 2] = 20
return out
result = generate(Changing(), Tokenizer(), torch.tensor([[1]]), Settings(3, 16, 3, POLICIES[1]), "cpu")
assert result["trace"][1]["remasked"] == 1
assert result["metrics"]["remasked_positions"] == 1
|