"""Validate every smoke request and aggregate acceptance/active-run throughput."""

import csv
import json
import sys
from pathlib import Path

root = Path(sys.argv[1])
subsets = (
    "HumanEval",
    "math_reasoning",
    "qa",
    "question",
    "rag",
    "summarization",
    "tool_call",
    "translation",
    "writing",
)
results = {}
table = [
    "# NGram-Candidates functional evaluation",
    "",
    "50-step checkpoint; 4 requests per subset, concurrency 1, spec=7, "
    "at most 512 output tokens. This is not a convergence/quality evaluation.",
    "",
    "Throughput is successful output tokens divided by active benchmark "
    "duration, excluding server startup and gaps between subsets. "
    "Aggregate acceptance is weighted by draft count, "
    "not an average of subset ratios.",
    "",
    "| Mode | Subset | Requests | Acceptance length | Output tokens/s |",
    "| --- | --- | ---: | ---: | ---: |",
]
for mode in ("greedy", "rs"):
    with (root / mode / "acceptance.csv").open() as source:
        acceptance = {row["subset"]: row for row in csv.DictReader(source)}
    if set(acceptance) != set(subsets):
        raise RuntimeError(
            f"Incomplete acceptance subsets for {mode}: {sorted(acceptance)}"
        )
    rows = []
    for subset in subsets:
        artifact = json.loads(
            (root / mode / "artifacts" / f"run_{subset}.json").read_text()
        )
        if len(artifact["benchmarks"]) != 1:
            raise RuntimeError(f"Expected one throughput benchmark for {mode}/{subset}")
        benchmark = artifact["benchmarks"][0]
        counts = benchmark["scheduler_metrics"]["requests_made"]
        if counts != {"successful": 4, "errored": 0, "incomplete": 0, "total": 4}:
            raise RuntimeError(f"Unsuccessful requests in {mode}/{subset}: {counts}")
        output_tokens = benchmark["metrics"]["output_token_count"]["successful"][
            "total_sum"
        ]
        duration = benchmark["duration"]
        if output_tokens <= 0 or duration <= 0:
            raise RuntimeError(f"Empty output or invalid duration in {mode}/{subset}")
        spec = acceptance[subset]
        drafts = float(spec["num_drafts"])
        accepted = float(spec["num_accepted_tokens"])
        if drafts <= 0 or float(spec["num_draft_tokens"]) != 7 * drafts:
            raise RuntimeError(f"Expected actual spec=7 decoding in {mode}/{subset}")
        row = {
            "subset": subset,
            "successful_requests": 4,
            "output_tokens": output_tokens,
            "duration_seconds": duration,
            "output_tokens_per_second": output_tokens / duration,
            "num_drafts": drafts,
            "num_accepted_tokens": accepted,
            "acceptance_length": 1 + accepted / drafts,
        }
        rows.append(row)
        table.append(
            f"| {mode} | {subset} | 4 | {row['acceptance_length']:.4f} | "
            f"{row['output_tokens_per_second']:.2f} |"
        )
    aggregate = {
        key: sum(row[key] for row in rows)
        for key in (
            "successful_requests",
            "output_tokens",
            "duration_seconds",
            "num_drafts",
            "num_accepted_tokens",
        )
    }
    aggregate["acceptance_length"] = (
        1 + aggregate["num_accepted_tokens"] / aggregate["num_drafts"]
    )
    aggregate["output_tokens_per_second"] = (
        aggregate["output_tokens"] / aggregate["duration_seconds"]
    )
    results[mode] = {"subsets": rows, "aggregate": aggregate}
    table.append(
        f"| {mode} | **ALL** | 36 | {aggregate['acceptance_length']:.4f} | "
        f"{aggregate['output_tokens_per_second']:.2f} |"
    )
report = {
    "passed": True,
    "scope": "functional smoke only; not full-epoch convergence or a quality benchmark",
    "modes": results,
}
(root / "summary.json").write_text(json.dumps(report, indent=2) + "\n")
(root / "summary.md").write_text("\n".join(table) + "\n")
print(
    json.dumps(
        {mode: result["aggregate"] for mode, result in results.items()}, indent=2
    )
)