Q-TensorFormer / tests /test_frontier_v10.py
Premchandyadav369
feat(frontier): achieve 10/10 with Triton kernels, GGUF/Ollama exporter, WebGPU runtime, multimodal vision, technical report, and 81 tests
4e689f6
Raw History Blame Contribute Delete
6.18 kB
"""
Comprehensive Unit Tests for Q-TensorFormer Frontier (10/10) System Extensions:
1. High-Performance Fused & Vectorized Tensor-Train Kernels (kernels/tt_kernels.py)
2. Universal GGUF / Ollama Exporter (qtensorformer_to_gguf.py)
3. ONNX & WebGPU Graph Serialization (export_onnx.py)
4. Multimodal Vision-Language Architecture (qtensorformer_vision.py)
"""
import os
import sys
import tempfile
import torch
import torch.nn as nn
import pytest
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if REPO_ROOT not in sys.path:
sys.path.insert(0, REPO_ROOT)
from configuration_qtensorformer import QTensorFormerConfig
from modeling_qtensorformer import QTensorFormerForCausalLM
from kernels.tt_kernels import (
slice_tt_cores,
optimized_tt_contract,
fast_tt_linear,
benchmark_tt_kernel,
)
from qtensorformer_to_gguf import GGUFWriter, export_qtensorformer_to_gguf
from export_onnx import QTensorFormerONNXWrapper
from qtensorformer_vision import (
TensorTrainVisionProjector,
QTensorFormerVisionForCausalLM,
)
def test_kernel_slice_and_contraction():
"""Verify kernel core slicing and optimized contraction produce correct shapes."""
batch_size, seq_len = 2, 4
d_in, d_out = 64, 128
in_shapes = [4, 4, 4]
out_shapes = [8, 4, 4]
rank = 8
x = torch.randn(batch_size, seq_len, d_in)
g1 = torch.randn(1, 8, 4, rank)
g2 = torch.randn(rank, 4, 4, rank)
g3 = torch.randn(rank, 4, 4, 1)
cores = [g1, g2, g3]
# Test full contraction
out_full = optimized_tt_contract(x, cores, in_shapes, out_shapes)
assert out_full.shape == (batch_size, seq_len, d_out)
# Test sliced contraction
sliced_cores = slice_tt_cores(cores, rank=4)
assert sliced_cores[0].shape == (1, 8, 4, 4)
assert sliced_cores[1].shape == (4, 4, 4, 4)
assert sliced_cores[2].shape == (4, 4, 4, 1)
out_sliced = optimized_tt_contract(x, cores, in_shapes, out_shapes, active_rank=4)
assert out_sliced.shape == (batch_size, seq_len, d_out)
def test_kernel_linear_with_bias():
"""Verify fast_tt_linear handles bias and rank truncation correctly."""
batch_size, seq_len = 1, 8
d_in, d_out = 64, 64
in_shapes = [4, 4, 4]
out_shapes = [4, 4, 4]
rank = 16
x = torch.randn(batch_size, seq_len, d_in)
cores = [
torch.randn(1, 4, 4, rank),
torch.randn(rank, 4, 4, rank),
torch.randn(rank, 4, 4, 1),
]
bias = torch.randn(d_out)
out = fast_tt_linear(x, cores, in_shapes, out_shapes, bias=bias, active_rank=8)
assert out.shape == (batch_size, seq_len, d_out)
def test_kernel_benchmark_utility():
"""Verify benchmark_tt_kernel executes without error."""
res = benchmark_tt_kernel(d_in=64, d_out=64, batch_size=1, seq_len=4, rank=4, num_runs=5)
assert "latency_ms" in res
assert res["latency_ms"] > 0.0
def test_gguf_writer_serialization():
"""Verify GGUFWriter correctly writes magic bytes, metadata, and tensors to disk."""
with tempfile.NamedTemporaryFile(suffix=".gguf", delete=False) as tmp:
tmp_name = tmp.name
try:
writer = GGUFWriter(tmp_name)
writer.add_string("general.architecture", "qtensorformer")
writer.add_uint32("qtensorformer.vocab_size", 5000)
writer.add_float32("qtensorformer.pid_kp", 0.1)
writer.add_bool("qtensorformer.routing_enabled", True)
dummy_tensor = torch.randn(4, 4, dtype=torch.float32)
writer.add_tensor("test.weight", dummy_tensor)
writer.write()
assert os.path.exists(tmp_name)
assert os.path.getsize(tmp_name) > 0
# Verify magic header
with open(tmp_name, "rb") as f:
magic = f.read(4)
assert magic == b"GGUF"
finally:
if os.path.exists(tmp_name):
os.remove(tmp_name)
def test_onnx_wrapper():
"""Verify QTensorFormerONNXWrapper produces valid logits tensor for graph tracing."""
config = QTensorFormerConfig(
vocab_size=256,
hidden_size=64,
num_hidden_layers=1,
num_attention_heads=2,
num_key_value_heads=1,
intermediate_size=128,
tt_ranks=[1, 4, 4, 1],
)
model = QTensorFormerForCausalLM(config)
wrapper = QTensorFormerONNXWrapper(model)
wrapper.eval()
dummy_input = torch.randint(0, 256, (1, 6), dtype=torch.long)
with torch.no_grad():
logits = wrapper(dummy_input)
assert logits.shape == (1, 6, 256)
def test_multimodal_vision_projector():
"""Verify TensorTrainVisionProjector converts vision features to language dimensions."""
vision_dim = 64
lang_dim = 128
projector = TensorTrainVisionProjector(
vision_hidden_size=vision_dim,
language_hidden_size=lang_dim,
intermediate_size=lang_dim,
tt_rank=8,
)
batch_size, num_patches = 2, 16
vis_features = torch.randn(batch_size, num_patches, vision_dim)
projected = projector(vis_features, active_rank=8)
assert projected.shape == (batch_size, num_patches, lang_dim)
def test_multimodal_vision_causal_lm():
"""Verify end-to-end multimodal forward pass and autoregressive generation."""
config = QTensorFormerConfig(
vocab_size=300,
hidden_size=64,
num_hidden_layers=2,
num_attention_heads=2,
num_key_value_heads=1,
intermediate_size=128,
tt_ranks=[1, 4, 4, 1],
)
vlm = QTensorFormerVisionForCausalLM(config=config, vision_hidden_size=64)
vlm.eval()
batch_size = 1
text_ids = torch.randint(0, 300, (batch_size, 4), dtype=torch.long)
# 224x224 image has (224/16)^2 = 196 patches
pixel_values = torch.randn(batch_size, 3, 224, 224)
with torch.no_grad():
out = vlm(input_ids=text_ids, pixel_values=pixel_values)
assert "logits" in out
# Total sequence length = 196 (vision patches) + 4 (text) = 200
assert out["logits"].shape == (batch_size, 200, 300)
# Test multimodal generation
gen_ids = vlm.generate_multimodal(input_ids=text_ids, pixel_values=pixel_values, max_new_tokens=3)
assert gen_ids.shape == (batch_size, 4 + 3)