code/tests/test_alpha_grid_aggregate.py

from __future__ import annotations

import json

from scripts.aggregate_alpha_grid import BLOCKS, METHODS, SEEDS, aggregate


def test_aggregate_reports_cells_and_paired_alpha_results(tmp_path):
    for alpha in (0.25, 0.5):
        for block_index, (model, task) in enumerate(BLOCKS):
            block_dir = tmp_path / f"alpha{alpha:g}" / "comparison" / f"{model}_{task}"
            block_dir.mkdir(parents=True)
            rows = {}
            for method_index, method in enumerate(METHODS):
                for seed_index, seed in enumerate(SEEDS):
                    accuracy = 0.7 + block_index * 0.001 + method_index * 0.01 + seed_index * 0.0001
                    rows[f"{method}__rank32__seed{seed}"] = {
                        "status": "complete",
                        "eval_accuracy": accuracy,
                    }
            (block_dir / "results.json").write_text(json.dumps(rows))

    result = aggregate(tmp_path, alphas=(0.25, 0.5))

    assert result["cell_counts"] == {"expected": 280, "complete": 280, "errors": 0}
    assert result["per_alpha"]["0.25"]["blocks"]["qwen3_xnli"]["fpeft_low"]["n"] == 5
    comparison = result["per_alpha"]["0.25"]["paired"]["fpeft_low__vs__random_orthogonal"]
    assert comparison["n"] == 35
    assert comparison["mean"] < 0
    assert len(comparison["block_bootstrap_95"]) == 2