""" tests/test_tokenizer_and_training.py Comprehensive tests for: 1. Canonical QTensorFormerTokenizer initialization and deterministic vocabulary 2. Special token IDs (=0, =1, =2, =3) 3. Lossless text -> token IDs -> text round-trip decode 4. Batch encode with padding, truncation, and attention_mask generation 5. Vocabulary-size alignment validation between tokenizer and model 6. Empty string and invalid input error handling with clear messages 7. Zero-copy GQA: non-expansion of K and V storage 8. NaN-free attention masks with causal and padding combinations 9. Canonical checkpoint saving and strict=True loading verification """ import tempfile from pathlib import Path import pytest import torch from src.checkpoint import ( inspect_checkpoint_compatibility, load_canonical_checkpoint, save_canonical_checkpoint, ) from src.config import ModelConfig from src.attention import MultiHeadAttention from src.models import DenseBaseline, QTensorFormer from src.tokenizer import QTensorFormerTokenizer ROOT_DIR = Path(__file__).resolve().parent.parent def test_tokenizer_special_tokens_and_vocab(): tok = QTensorFormerTokenizer(vocab_size=256) assert tok.pad_token_id == 0 assert tok.bos_token_id == 1 assert tok.eos_token_id == 2 assert tok.unk_token_id == 3 assert tok.decode([0, 1, 2, 3], skip_special_tokens=False) == "" assert len(tok) == 256 assert tok.vocab_size == 256 def test_tokenizer_lossless_roundtrip(): tok = QTensorFormerTokenizer(vocab_size=256) sample_text = "Q-TensorFormer: dynamic information-to-resource allocation at the edge." ids = tok.encode(sample_text, add_bos=False, add_eos=False) decoded = tok.decode(ids, skip_special_tokens=True) assert decoded == sample_text def test_tokenizer_batch_encode_padding_and_masks(): tok = QTensorFormerTokenizer(vocab_size=256, model_max_length=32) texts = [ "Short text.", "A significantly longer text sentence designed to verify batch padding and attention masks.", ] encoded = tok.batch_encode( texts, add_bos=True, add_eos=True, padding=True, truncation=True, max_length=20, return_tensors="pt", ) input_ids = encoded["input_ids"] attention_mask = encoded["attention_mask"] assert input_ids.shape == (2, 20) assert attention_mask.shape == (2, 20) assert (attention_mask[0, :] == 0).any(), "Shorter sequence must contain pad positions" # Verify that pad positions in input_ids match attention_mask == 0 assert (input_ids[0][attention_mask[0] == 0] == tok.pad_token_id).all() assert (input_ids[0][attention_mask[0] == 1] != tok.pad_token_id).all() def test_tokenizer_error_handling_clear_messages(): tok = QTensorFormerTokenizer(vocab_size=256) # Empty string with no special tokens should raise ValueError with pytest.raises(ValueError, match="Cannot encode an empty string"): tok.encode("", add_bos=False, add_eos=False) # Non-string input should raise TypeError with pytest.raises(TypeError, match="expected a string"): tok.encode(12345) # Empty batch should raise ValueError with pytest.raises(ValueError, match="non-empty list or tuple"): tok.batch_encode([]) # Out of range token ID in decode should raise ValueError with pytest.raises(ValueError, match="Out-of-vocabulary"): tok.decode([99999]) def test_tokenizer_model_alignment_check(): tok = QTensorFormerTokenizer(vocab_size=256) cfg_matching = ModelConfig(vocab_size=256, d_model=64, n_heads=4, n_layers=2) cfg_mismatch = ModelConfig(vocab_size=512, d_model=64, n_heads=4, n_layers=2) assert tok.validate_model_alignment(cfg_matching) is True with pytest.raises(ValueError, match="VocabularyMismatchError"): tok.validate_model_alignment(cfg_mismatch) model_matching = QTensorFormer(cfg_matching) assert tok.validate_model_alignment(model_matching) is True def test_gqa_zero_copy_storage(): """Verify that GQA attention does not allocate an expanded copy of K and V.""" d_model = 64 n_heads = 8 n_kv_heads = 2 attn = MultiHeadAttention(d_model=d_model, n_heads=n_heads, n_kv_heads=n_kv_heads) x = torch.randn(2, 16, d_model) out, weights = attn(x) assert out.shape == (2, 16, d_model) assert not torch.isnan(out).any() # Check that storage was shared without materializing full n_heads K/V copy assert getattr(attn, "last_kv_shared_storage", False) is True # Storage element count must equal B * n_kv_heads * T * head_dim, not n_heads! expected_elements = 2 * n_kv_heads * 16 * (d_model // n_heads) assert attn.last_kv_storage_elements == expected_elements def test_attention_masks_no_nans_with_padding(): """Verify that attention with padding masks (including left and right padding) never produces NaNs.""" d_model = 64 n_heads = 4 attn = MultiHeadAttention(d_model=d_model, n_heads=n_heads) x = torch.randn(2, 8, d_model) # Mask with left-padded zeros and right-padded zeros mask = torch.tensor([ [0, 0, 1, 1, 1, 1, 1, 1], # Left-padded [1, 1, 1, 1, 1, 0, 0, 0], # Right-padded ], dtype=torch.long) out, weights = attn(x, mask=mask, return_attn_weights=True) assert not torch.isnan(out).any(), "Attention output contains NaNs with padding mask!" assert not torch.isinf(out).any() assert not torch.isnan(weights).any(), "Attention weights contain NaNs with padding mask!" # Verify right-padded positions have zero attention weight assert (weights[1, :, :, 5:] == 0.0).all(), "Padded positions in sequence 1 should have zero attention weight!" def test_save_and_strict_load_canonical_checkpoint(): cfg = ModelConfig(vocab_size=256, d_model=64, n_heads=4, n_layers=2, tt_rank=4) model = QTensorFormer(cfg) tok = QTensorFormerTokenizer(vocab_size=256) with tempfile.TemporaryDirectory() as tmpdir: ckpt_path = Path(tmpdir) / "test_canonical.pt" meta = save_canonical_checkpoint( model=model, checkpoint_path=ckpt_path, tokenizer=tok, is_trained=True, training_provenance={"test": True, "epochs": 1}, ) assert ckpt_path.exists() assert meta["is_trained"] is True assert meta["file_sha256"] is not None # Inspect compatibility compat = inspect_checkpoint_compatibility(ckpt_path, model) assert compat["strict_compatible"] is True assert compat["missing_key_count"] == 0 assert compat["unexpected_key_count"] == 0 # Strict load should succeed model2 = QTensorFormer(cfg) loaded_model, info = load_canonical_checkpoint( ckpt_path, model=model2, strict=True, require_trained=True, tokenizer=tok, ) assert loaded_model is model2 # Check weights match for (k1, p1), (k2, p2) in zip(model.named_parameters(), model2.named_parameters()): assert k1 == k2 assert torch.allclose(p1, p2)