""" Comprehensive Unit Tests for Q-TensorFormer Frontier (10/10) System Extensions: 1. High-Performance Fused & Vectorized Tensor-Train Kernels (kernels/tt_kernels.py) 2. Universal GGUF / Ollama Exporter (qtensorformer_to_gguf.py) 3. ONNX & WebGPU Graph Serialization (export_onnx.py) 4. Multimodal Vision-Language Architecture (qtensorformer_vision.py) """ import os import sys import tempfile import torch import torch.nn as nn import pytest REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) from configuration_qtensorformer import QTensorFormerConfig from modeling_qtensorformer import QTensorFormerForCausalLM from kernels.tt_kernels import ( slice_tt_cores, optimized_tt_contract, fast_tt_linear, benchmark_tt_kernel, ) from qtensorformer_to_gguf import GGUFWriter, export_qtensorformer_to_gguf from export_onnx import QTensorFormerONNXWrapper from qtensorformer_vision import ( TensorTrainVisionProjector, QTensorFormerVisionForCausalLM, ) def test_kernel_slice_and_contraction(): """Verify kernel core slicing and optimized contraction produce correct shapes.""" batch_size, seq_len = 2, 4 d_in, d_out = 64, 128 in_shapes = [4, 4, 4] out_shapes = [8, 4, 4] rank = 8 x = torch.randn(batch_size, seq_len, d_in) g1 = torch.randn(1, 8, 4, rank) g2 = torch.randn(rank, 4, 4, rank) g3 = torch.randn(rank, 4, 4, 1) cores = [g1, g2, g3] # Test full contraction out_full = optimized_tt_contract(x, cores, in_shapes, out_shapes) assert out_full.shape == (batch_size, seq_len, d_out) # Test sliced contraction sliced_cores = slice_tt_cores(cores, rank=4) assert sliced_cores[0].shape == (1, 8, 4, 4) assert sliced_cores[1].shape == (4, 4, 4, 4) assert sliced_cores[2].shape == (4, 4, 4, 1) out_sliced = optimized_tt_contract(x, cores, in_shapes, out_shapes, active_rank=4) assert out_sliced.shape == (batch_size, seq_len, d_out) def test_kernel_linear_with_bias(): """Verify fast_tt_linear handles bias and rank truncation correctly.""" batch_size, seq_len = 1, 8 d_in, d_out = 64, 64 in_shapes = [4, 4, 4] out_shapes = [4, 4, 4] rank = 16 x = torch.randn(batch_size, seq_len, d_in) cores = [ torch.randn(1, 4, 4, rank), torch.randn(rank, 4, 4, rank), torch.randn(rank, 4, 4, 1), ] bias = torch.randn(d_out) out = fast_tt_linear(x, cores, in_shapes, out_shapes, bias=bias, active_rank=8) assert out.shape == (batch_size, seq_len, d_out) def test_kernel_benchmark_utility(): """Verify benchmark_tt_kernel executes without error.""" res = benchmark_tt_kernel(d_in=64, d_out=64, batch_size=1, seq_len=4, rank=4, num_runs=5) assert "latency_ms" in res assert res["latency_ms"] > 0.0 def test_gguf_writer_serialization(): """Verify GGUFWriter correctly writes magic bytes, metadata, and tensors to disk.""" with tempfile.NamedTemporaryFile(suffix=".gguf", delete=False) as tmp: tmp_name = tmp.name try: writer = GGUFWriter(tmp_name) writer.add_string("general.architecture", "qtensorformer") writer.add_uint32("qtensorformer.vocab_size", 5000) writer.add_float32("qtensorformer.pid_kp", 0.1) writer.add_bool("qtensorformer.routing_enabled", True) dummy_tensor = torch.randn(4, 4, dtype=torch.float32) writer.add_tensor("test.weight", dummy_tensor) writer.write() assert os.path.exists(tmp_name) assert os.path.getsize(tmp_name) > 0 # Verify magic header with open(tmp_name, "rb") as f: magic = f.read(4) assert magic == b"GGUF" finally: if os.path.exists(tmp_name): os.remove(tmp_name) def test_onnx_wrapper(): """Verify QTensorFormerONNXWrapper produces valid logits tensor for graph tracing.""" config = QTensorFormerConfig( vocab_size=256, hidden_size=64, num_hidden_layers=1, num_attention_heads=2, num_key_value_heads=1, intermediate_size=128, tt_ranks=[1, 4, 4, 1], ) model = QTensorFormerForCausalLM(config) wrapper = QTensorFormerONNXWrapper(model) wrapper.eval() dummy_input = torch.randint(0, 256, (1, 6), dtype=torch.long) with torch.no_grad(): logits = wrapper(dummy_input) assert logits.shape == (1, 6, 256) def test_multimodal_vision_projector(): """Verify TensorTrainVisionProjector converts vision features to language dimensions.""" vision_dim = 64 lang_dim = 128 projector = TensorTrainVisionProjector( vision_hidden_size=vision_dim, language_hidden_size=lang_dim, intermediate_size=lang_dim, tt_rank=8, ) batch_size, num_patches = 2, 16 vis_features = torch.randn(batch_size, num_patches, vision_dim) projected = projector(vis_features, active_rank=8) assert projected.shape == (batch_size, num_patches, lang_dim) def test_multimodal_vision_causal_lm(): """Verify end-to-end multimodal forward pass and autoregressive generation.""" config = QTensorFormerConfig( vocab_size=300, hidden_size=64, num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=1, intermediate_size=128, tt_ranks=[1, 4, 4, 1], ) vlm = QTensorFormerVisionForCausalLM(config=config, vision_hidden_size=64) vlm.eval() batch_size = 1 text_ids = torch.randint(0, 300, (batch_size, 4), dtype=torch.long) # 224x224 image has (224/16)^2 = 196 patches pixel_values = torch.randn(batch_size, 3, 224, 224) with torch.no_grad(): out = vlm(input_ids=text_ids, pixel_values=pixel_values) assert "logits" in out # Total sequence length = 196 (vision patches) + 4 (text) = 200 assert out["logits"].shape == (batch_size, 200, 300) # Test multimodal generation gen_ids = vlm.generate_multimodal(input_ids=text_ids, pixel_values=pixel_values, max_new_tokens=3) assert gen_ids.shape == (batch_size, 4 + 3)