Blame for tests/test_baseline_comparator.py

"""
tests/test_baseline_comparator.py
Unit tests for the Comprehensive Baseline Comparison Engine.
"""

import pytest
import os
import json
import tempfile
from pathlib import Path

import sys
import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from src.baseline_comparator import (
    ProvenanceTag,
    MetricDirection,
    MetricSpec,
    METRIC_CATALOG,
    BaselineEvaluationRecord,
    BaselineComparator,
)


def test_metric_catalog_integrity():
    assert len(METRIC_CATALOG) >= 17
    assert "parameters_total_m" in METRIC_CATALOG
    assert "tpot_ms" in METRIC_CATALOG
    assert "tokens_per_sec" in METRIC_CATALOG
    assert "cosine_fidelity" in METRIC_CATALOG
    assert "cost_per_million_tokens_usd" in METRIC_CATALOG

    # Verify every metric has valid direction and provenance
    for key, spec in METRIC_CATALOG.items():
        assert isinstance(spec.direction, MetricDirection)
        assert isinstance(spec.default_provenance, ProvenanceTag)
        assert len(spec.display_name) > 0
        assert len(spec.unit) > 0


def test_compute_percentage_improvement():
    # Lower is better: reduction is an improvement (+)
    # Latency 10ms -> 5ms: +50%
    res = BaselineComparator.compute_percentage_improvement(10.0, 5.0, MetricDirection.LOWER_IS_BETTER)
    assert res == pytest.approx(50.0)

    # Latency 10ms -> 15ms: -50%
    res = BaselineComparator.compute_percentage_improvement(10.0, 15.0, MetricDirection.LOWER_IS_BETTER)
    assert res == pytest.approx(-50.0)

    # Higher is better: increase is an improvement (+)
    # Tokens/sec 100 -> 150: +50%
    res = BaselineComparator.compute_percentage_improvement(100.0, 150.0, MetricDirection.HIGHER_IS_BETTER)
    assert res == pytest.approx(50.0)

    # Tokens/sec 100 -> 50: -50%
    res = BaselineComparator.compute_percentage_improvement(100.0, 50.0, MetricDirection.HIGHER_IS_BETTER)
    assert res == pytest.approx(-50.0)

    # Missing values return None
    assert BaselineComparator.compute_percentage_improvement(None, 10.0, MetricDirection.LOWER_IS_BETTER) is None
    assert BaselineComparator.compute_percentage_improvement(10.0, None, MetricDirection.HIGHER_IS_BETTER) is None
    assert BaselineComparator.compute_percentage_improvement(0.0, 10.0, MetricDirection.LOWER_IS_BETTER) is None


def test_comparator_matrix_and_formatting():
    rec_dense = BaselineEvaluationRecord(
        model_id="dense",
        display_name="Dense Baseline (FP32)",
        category="dense",
        metrics={
            "parameters_total_m": 0.237,
            "tpot_ms": 2.50,
            "tokens_per_sec": 400.0,
            "peak_ram_mb": 0.94,
            "cosine_fidelity": 1.000,
        },
    )

    rec_qtf = BaselineEvaluationRecord(
        model_id="qtf_edge",
        display_name="Q-TensorFormer (Edge-SLA)",
        category="adaptive_tt",
        metrics={
            "parameters_total_m": 0.112,
            "tpot_ms": 2.78,
            "tokens_per_sec": 359.7,
            "peak_ram_mb": 0.44,
            "cosine_fidelity": 0.948,
        },
    )

    comparator = BaselineComparator([rec_dense, rec_qtf])
    matrix = comparator.compute_improvement_matrix(
        target_model_id="qtf_edge",
        baseline_model_ids=["dense"],
        metric_keys=["peak_ram_mb", "tpot_ms", "tokens_per_sec"],
    )

    # RAM: 0.94 -> 0.44 => (0.94 - 0.44)/0.94 * 100 = +53.19%
    assert matrix["dense"]["peak_ram_mb"] == pytest.approx(53.19, abs=0.1)

    # TPOT: 2.50 -> 2.78 => (2.50 - 2.78)/2.50 * 100 = -11.2%
    assert matrix["dense"]["tpot_ms"] == pytest.approx(-11.2, abs=0.1)

    # Table generation
    md_table = comparator.to_markdown_table(metric_keys=["peak_ram_mb", "tpot_ms"])
    assert "| **Dense Baseline (FP32)** |" in md_table
    assert "| **Q-TensorFormer (Edge-SLA)** |" in md_table

    # Percentage matrix markdown
    pct_md = comparator.to_percentage_matrix_markdown(
        target_model_id="qtf_edge",
        baseline_model_ids=["dense"],
        metric_keys=["peak_ram_mb", "tpot_ms"],
    )
    assert "Relative % Improvement: Q-TensorFormer (Edge-SLA) vs Baselines" in pct_md
    assert "**+53.2%**" in pct_md or "+53.2%" in pct_md


def test_regime_analyzer():
    comparator = BaselineComparator()
    regimes = comparator.identify_regimes()
    assert "best_case_regimes" in regimes
    assert "worst_case_regimes" in regimes
    assert len(regimes["best_case_regimes"]) >= 3
    assert len(regimes["worst_case_regimes"]) >= 3


def test_json_and_csv_export(tmp_path):
    rec = BaselineEvaluationRecord(
        model_id="dense",
        display_name="Dense Baseline",
        category="dense",
        metrics={"parameters_total_m": 0.237, "tpot_ms": 2.50},
    )
    comparator = BaselineComparator([rec])

    json_path = tmp_path / "baseline_out.json"
    csv_path = tmp_path / "baseline_out.csv"

    comparator.to_json(json_path)
    comparator.to_csv(csv_path)

    assert json_path.exists()
    assert csv_path.exists()

    with open(json_path, "r") as f:
        data = json.load(f)
        assert data["metadata"]["system"] == "Q-TensorFormer Baseline Comparison System"
        assert len(data["models"]) == 1