Text Generation
Transformers
Safetensors
PyTorch
English
qtensorformer
q-tensorformer
high-throughput-inference
tensor-train
tensor-networks
tensorrt-llm
vllm
continuous-batching
paged-attention
speculative-decoding
zero-svd
adaptive-computation
dynamic-neural-networks
kv-cache
kv-cache-quantization
attention-sinks
lyapunov-stability
control-theory
contextual-bandits
edge-ai
edge-inference
on-device-ai
model-compression
hopper-h100
blackwell-b200
jetson-orin
jetson-thor
grouped-query-attention
rotary-position-embeddings
custom_code
academic-paper
ieee
neurips
vit-ap-university
Premchandyadav369
feat(benchmarks): Add comprehensive baseline comparison system with 14 architectures, 17 metrics, Pareto frontiers, and provenance tags
7becc06 | """ | |
| tests/test_baseline_comparator.py | |
| Unit tests for the Comprehensive Baseline Comparison Engine. | |
| """ | |
| import pytest | |
| import os | |
| import json | |
| import tempfile | |
| from pathlib import Path | |
| import sys | |
| import os | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from src.baseline_comparator import ( | |
| ProvenanceTag, | |
| MetricDirection, | |
| MetricSpec, | |
| METRIC_CATALOG, | |
| BaselineEvaluationRecord, | |
| BaselineComparator, | |
| ) | |
| def test_metric_catalog_integrity(): | |
| assert len(METRIC_CATALOG) >= 17 | |
| assert "parameters_total_m" in METRIC_CATALOG | |
| assert "tpot_ms" in METRIC_CATALOG | |
| assert "tokens_per_sec" in METRIC_CATALOG | |
| assert "cosine_fidelity" in METRIC_CATALOG | |
| assert "cost_per_million_tokens_usd" in METRIC_CATALOG | |
| # Verify every metric has valid direction and provenance | |
| for key, spec in METRIC_CATALOG.items(): | |
| assert isinstance(spec.direction, MetricDirection) | |
| assert isinstance(spec.default_provenance, ProvenanceTag) | |
| assert len(spec.display_name) > 0 | |
| assert len(spec.unit) > 0 | |
| def test_compute_percentage_improvement(): | |
| # Lower is better: reduction is an improvement (+) | |
| # Latency 10ms -> 5ms: +50% | |
| res = BaselineComparator.compute_percentage_improvement(10.0, 5.0, MetricDirection.LOWER_IS_BETTER) | |
| assert res == pytest.approx(50.0) | |
| # Latency 10ms -> 15ms: -50% | |
| res = BaselineComparator.compute_percentage_improvement(10.0, 15.0, MetricDirection.LOWER_IS_BETTER) | |
| assert res == pytest.approx(-50.0) | |
| # Higher is better: increase is an improvement (+) | |
| # Tokens/sec 100 -> 150: +50% | |
| res = BaselineComparator.compute_percentage_improvement(100.0, 150.0, MetricDirection.HIGHER_IS_BETTER) | |
| assert res == pytest.approx(50.0) | |
| # Tokens/sec 100 -> 50: -50% | |
| res = BaselineComparator.compute_percentage_improvement(100.0, 50.0, MetricDirection.HIGHER_IS_BETTER) | |
| assert res == pytest.approx(-50.0) | |
| # Missing values return None | |
| assert BaselineComparator.compute_percentage_improvement(None, 10.0, MetricDirection.LOWER_IS_BETTER) is None | |
| assert BaselineComparator.compute_percentage_improvement(10.0, None, MetricDirection.HIGHER_IS_BETTER) is None | |
| assert BaselineComparator.compute_percentage_improvement(0.0, 10.0, MetricDirection.LOWER_IS_BETTER) is None | |
| def test_comparator_matrix_and_formatting(): | |
| rec_dense = BaselineEvaluationRecord( | |
| model_id="dense", | |
| display_name="Dense Baseline (FP32)", | |
| category="dense", | |
| metrics={ | |
| "parameters_total_m": 0.237, | |
| "tpot_ms": 2.50, | |
| "tokens_per_sec": 400.0, | |
| "peak_ram_mb": 0.94, | |
| "cosine_fidelity": 1.000, | |
| }, | |
| ) | |
| rec_qtf = BaselineEvaluationRecord( | |
| model_id="qtf_edge", | |
| display_name="Q-TensorFormer (Edge-SLA)", | |
| category="adaptive_tt", | |
| metrics={ | |
| "parameters_total_m": 0.112, | |
| "tpot_ms": 2.78, | |
| "tokens_per_sec": 359.7, | |
| "peak_ram_mb": 0.44, | |
| "cosine_fidelity": 0.948, | |
| }, | |
| ) | |
| comparator = BaselineComparator([rec_dense, rec_qtf]) | |
| matrix = comparator.compute_improvement_matrix( | |
| target_model_id="qtf_edge", | |
| baseline_model_ids=["dense"], | |
| metric_keys=["peak_ram_mb", "tpot_ms", "tokens_per_sec"], | |
| ) | |
| # RAM: 0.94 -> 0.44 => (0.94 - 0.44)/0.94 * 100 = +53.19% | |
| assert matrix["dense"]["peak_ram_mb"] == pytest.approx(53.19, abs=0.1) | |
| # TPOT: 2.50 -> 2.78 => (2.50 - 2.78)/2.50 * 100 = -11.2% | |
| assert matrix["dense"]["tpot_ms"] == pytest.approx(-11.2, abs=0.1) | |
| # Table generation | |
| md_table = comparator.to_markdown_table(metric_keys=["peak_ram_mb", "tpot_ms"]) | |
| assert "| **Dense Baseline (FP32)** |" in md_table | |
| assert "| **Q-TensorFormer (Edge-SLA)** |" in md_table | |
| # Percentage matrix markdown | |
| pct_md = comparator.to_percentage_matrix_markdown( | |
| target_model_id="qtf_edge", | |
| baseline_model_ids=["dense"], | |
| metric_keys=["peak_ram_mb", "tpot_ms"], | |
| ) | |
| assert "Relative % Improvement: Q-TensorFormer (Edge-SLA) vs Baselines" in pct_md | |
| assert "**+53.2%**" in pct_md or "+53.2%" in pct_md | |
| def test_regime_analyzer(): | |
| comparator = BaselineComparator() | |
| regimes = comparator.identify_regimes() | |
| assert "best_case_regimes" in regimes | |
| assert "worst_case_regimes" in regimes | |
| assert len(regimes["best_case_regimes"]) >= 3 | |
| assert len(regimes["worst_case_regimes"]) >= 3 | |
| def test_json_and_csv_export(tmp_path): | |
| rec = BaselineEvaluationRecord( | |
| model_id="dense", | |
| display_name="Dense Baseline", | |
| category="dense", | |
| metrics={"parameters_total_m": 0.237, "tpot_ms": 2.50}, | |
| ) | |
| comparator = BaselineComparator([rec]) | |
| json_path = tmp_path / "baseline_out.json" | |
| csv_path = tmp_path / "baseline_out.csv" | |
| comparator.to_json(json_path) | |
| comparator.to_csv(csv_path) | |
| assert json_path.exists() | |
| assert csv_path.exists() | |
| with open(json_path, "r") as f: | |
| data = json.load(f) | |
| assert data["metadata"]["system"] == "Q-TensorFormer Baseline Comparison System" | |
| assert len(data["models"]) == 1 | |