Q-TensorFormer / tests /test_tokenizer_and_training.py
Premchandyadav369
feat: complete scientific repair, canonical checkpoint training, tokenizer, GQA zero-copy, and release gating
d279df0
Raw History Blame Contribute Delete
7.23 kB
"""
tests/test_tokenizer_and_training.py
Comprehensive tests for:
1. Canonical QTensorFormerTokenizer initialization and deterministic vocabulary
2. Special token IDs (<pad>=0, <bos>=1, <eos>=2, <unk>=3)
3. Lossless text -> token IDs -> text round-trip decode
4. Batch encode with padding, truncation, and attention_mask generation
5. Vocabulary-size alignment validation between tokenizer and model
6. Empty string and invalid input error handling with clear messages
7. Zero-copy GQA: non-expansion of K and V storage
8. NaN-free attention masks with causal and padding combinations
9. Canonical checkpoint saving and strict=True loading verification
"""
import tempfile
from pathlib import Path
import pytest
import torch
from src.checkpoint import (
inspect_checkpoint_compatibility,
load_canonical_checkpoint,
save_canonical_checkpoint,
)
from src.config import ModelConfig
from src.attention import MultiHeadAttention
from src.models import DenseBaseline, QTensorFormer
from src.tokenizer import QTensorFormerTokenizer
ROOT_DIR = Path(__file__).resolve().parent.parent
def test_tokenizer_special_tokens_and_vocab():
tok = QTensorFormerTokenizer(vocab_size=256)
assert tok.pad_token_id == 0
assert tok.bos_token_id == 1
assert tok.eos_token_id == 2
assert tok.unk_token_id == 3
assert tok.decode([0, 1, 2, 3], skip_special_tokens=False) == "<pad><bos><eos><unk>"
assert len(tok) == 256
assert tok.vocab_size == 256
def test_tokenizer_lossless_roundtrip():
tok = QTensorFormerTokenizer(vocab_size=256)
sample_text = "Q-TensorFormer: dynamic information-to-resource allocation at the edge."
ids = tok.encode(sample_text, add_bos=False, add_eos=False)
decoded = tok.decode(ids, skip_special_tokens=True)
assert decoded == sample_text
def test_tokenizer_batch_encode_padding_and_masks():
tok = QTensorFormerTokenizer(vocab_size=256, model_max_length=32)
texts = [
"Short text.",
"A significantly longer text sentence designed to verify batch padding and attention masks.",
]
encoded = tok.batch_encode(
texts,
add_bos=True,
add_eos=True,
padding=True,
truncation=True,
max_length=20,
return_tensors="pt",
)
input_ids = encoded["input_ids"]
attention_mask = encoded["attention_mask"]
assert input_ids.shape == (2, 20)
assert attention_mask.shape == (2, 20)
assert (attention_mask[0, :] == 0).any(), "Shorter sequence must contain pad positions"
# Verify that pad positions in input_ids match attention_mask == 0
assert (input_ids[0][attention_mask[0] == 0] == tok.pad_token_id).all()
assert (input_ids[0][attention_mask[0] == 1] != tok.pad_token_id).all()
def test_tokenizer_error_handling_clear_messages():
tok = QTensorFormerTokenizer(vocab_size=256)
# Empty string with no special tokens should raise ValueError
with pytest.raises(ValueError, match="Cannot encode an empty string"):
tok.encode("", add_bos=False, add_eos=False)
# Non-string input should raise TypeError
with pytest.raises(TypeError, match="expected a string"):
tok.encode(12345)
# Empty batch should raise ValueError
with pytest.raises(ValueError, match="non-empty list or tuple"):
tok.batch_encode([])
# Out of range token ID in decode should raise ValueError
with pytest.raises(ValueError, match="Out-of-vocabulary"):
tok.decode([99999])
def test_tokenizer_model_alignment_check():
tok = QTensorFormerTokenizer(vocab_size=256)
cfg_matching = ModelConfig(vocab_size=256, d_model=64, n_heads=4, n_layers=2)
cfg_mismatch = ModelConfig(vocab_size=512, d_model=64, n_heads=4, n_layers=2)
assert tok.validate_model_alignment(cfg_matching) is True
with pytest.raises(ValueError, match="VocabularyMismatchError"):
tok.validate_model_alignment(cfg_mismatch)
model_matching = QTensorFormer(cfg_matching)
assert tok.validate_model_alignment(model_matching) is True
def test_gqa_zero_copy_storage():
"""Verify that GQA attention does not allocate an expanded copy of K and V."""
d_model = 64
n_heads = 8
n_kv_heads = 2
attn = MultiHeadAttention(d_model=d_model, n_heads=n_heads, n_kv_heads=n_kv_heads)
x = torch.randn(2, 16, d_model)
out, weights = attn(x)
assert out.shape == (2, 16, d_model)
assert not torch.isnan(out).any()
# Check that storage was shared without materializing full n_heads K/V copy
assert getattr(attn, "last_kv_shared_storage", False) is True
# Storage element count must equal B * n_kv_heads * T * head_dim, not n_heads!
expected_elements = 2 * n_kv_heads * 16 * (d_model // n_heads)
assert attn.last_kv_storage_elements == expected_elements
def test_attention_masks_no_nans_with_padding():
"""Verify that attention with padding masks (including left and right padding) never produces NaNs."""
d_model = 64
n_heads = 4
attn = MultiHeadAttention(d_model=d_model, n_heads=n_heads)
x = torch.randn(2, 8, d_model)
# Mask with left-padded zeros and right-padded zeros
mask = torch.tensor([
[0, 0, 1, 1, 1, 1, 1, 1], # Left-padded
[1, 1, 1, 1, 1, 0, 0, 0], # Right-padded
], dtype=torch.long)
out, weights = attn(x, mask=mask, return_attn_weights=True)
assert not torch.isnan(out).any(), "Attention output contains NaNs with padding mask!"
assert not torch.isinf(out).any()
assert not torch.isnan(weights).any(), "Attention weights contain NaNs with padding mask!"
# Verify right-padded positions have zero attention weight
assert (weights[1, :, :, 5:] == 0.0).all(), "Padded positions in sequence 1 should have zero attention weight!"
def test_save_and_strict_load_canonical_checkpoint():
cfg = ModelConfig(vocab_size=256, d_model=64, n_heads=4, n_layers=2, tt_rank=4)
model = QTensorFormer(cfg)
tok = QTensorFormerTokenizer(vocab_size=256)
with tempfile.TemporaryDirectory() as tmpdir:
ckpt_path = Path(tmpdir) / "test_canonical.pt"
meta = save_canonical_checkpoint(
model=model,
checkpoint_path=ckpt_path,
tokenizer=tok,
is_trained=True,
training_provenance={"test": True, "epochs": 1},
)
assert ckpt_path.exists()
assert meta["is_trained"] is True
assert meta["file_sha256"] is not None
# Inspect compatibility
compat = inspect_checkpoint_compatibility(ckpt_path, model)
assert compat["strict_compatible"] is True
assert compat["missing_key_count"] == 0
assert compat["unexpected_key_count"] == 0
# Strict load should succeed
model2 = QTensorFormer(cfg)
loaded_model, info = load_canonical_checkpoint(
ckpt_path,
model=model2,
strict=True,
require_trained=True,
tokenizer=tok,
)
assert loaded_model is model2
# Check weights match
for (k1, p1), (k2, p2) in zip(model.named_parameters(), model2.named_parameters()):
assert k1 == k2
assert torch.allclose(p1, p2)