| """Validate every smoke request and aggregate acceptance/active-run throughput.""" |
|
|
| import csv |
| import json |
| import sys |
| from pathlib import Path |
|
|
| root = Path(sys.argv[1]) |
| subsets = ( |
| "HumanEval", |
| "math_reasoning", |
| "qa", |
| "question", |
| "rag", |
| "summarization", |
| "tool_call", |
| "translation", |
| "writing", |
| ) |
| results = {} |
| table = [ |
| "# NGram-Candidates functional evaluation", |
| "", |
| "50-step checkpoint; 4 requests per subset, concurrency 1, spec=7, " |
| "at most 512 output tokens. This is not a convergence/quality evaluation.", |
| "", |
| "Throughput is successful output tokens divided by active benchmark " |
| "duration, excluding server startup and gaps between subsets. " |
| "Aggregate acceptance is weighted by draft count, " |
| "not an average of subset ratios.", |
| "", |
| "| Mode | Subset | Requests | Acceptance length | Output tokens/s |", |
| "| --- | --- | ---: | ---: | ---: |", |
| ] |
| for mode in ("greedy", "rs"): |
| with (root / mode / "acceptance.csv").open() as source: |
| acceptance = {row["subset"]: row for row in csv.DictReader(source)} |
| if set(acceptance) != set(subsets): |
| raise RuntimeError( |
| f"Incomplete acceptance subsets for {mode}: {sorted(acceptance)}" |
| ) |
| rows = [] |
| for subset in subsets: |
| artifact = json.loads( |
| (root / mode / "artifacts" / f"run_{subset}.json").read_text() |
| ) |
| if len(artifact["benchmarks"]) != 1: |
| raise RuntimeError(f"Expected one throughput benchmark for {mode}/{subset}") |
| benchmark = artifact["benchmarks"][0] |
| counts = benchmark["scheduler_metrics"]["requests_made"] |
| if counts != {"successful": 4, "errored": 0, "incomplete": 0, "total": 4}: |
| raise RuntimeError(f"Unsuccessful requests in {mode}/{subset}: {counts}") |
| output_tokens = benchmark["metrics"]["output_token_count"]["successful"][ |
| "total_sum" |
| ] |
| duration = benchmark["duration"] |
| if output_tokens <= 0 or duration <= 0: |
| raise RuntimeError(f"Empty output or invalid duration in {mode}/{subset}") |
| spec = acceptance[subset] |
| drafts = float(spec["num_drafts"]) |
| accepted = float(spec["num_accepted_tokens"]) |
| if drafts <= 0 or float(spec["num_draft_tokens"]) != 7 * drafts: |
| raise RuntimeError(f"Expected actual spec=7 decoding in {mode}/{subset}") |
| row = { |
| "subset": subset, |
| "successful_requests": 4, |
| "output_tokens": output_tokens, |
| "duration_seconds": duration, |
| "output_tokens_per_second": output_tokens / duration, |
| "num_drafts": drafts, |
| "num_accepted_tokens": accepted, |
| "acceptance_length": 1 + accepted / drafts, |
| } |
| rows.append(row) |
| table.append( |
| f"| {mode} | {subset} | 4 | {row['acceptance_length']:.4f} | " |
| f"{row['output_tokens_per_second']:.2f} |" |
| ) |
| aggregate = { |
| key: sum(row[key] for row in rows) |
| for key in ( |
| "successful_requests", |
| "output_tokens", |
| "duration_seconds", |
| "num_drafts", |
| "num_accepted_tokens", |
| ) |
| } |
| aggregate["acceptance_length"] = ( |
| 1 + aggregate["num_accepted_tokens"] / aggregate["num_drafts"] |
| ) |
| aggregate["output_tokens_per_second"] = ( |
| aggregate["output_tokens"] / aggregate["duration_seconds"] |
| ) |
| results[mode] = {"subsets": rows, "aggregate": aggregate} |
| table.append( |
| f"| {mode} | **ALL** | 36 | {aggregate['acceptance_length']:.4f} | " |
| f"{aggregate['output_tokens_per_second']:.2f} |" |
| ) |
| report = { |
| "passed": True, |
| "scope": "functional smoke only; not full-epoch convergence or a quality benchmark", |
| "modes": results, |
| } |
| (root / "summary.json").write_text(json.dumps(report, indent=2) + "\n") |
| (root / "summary.md").write_text("\n".join(table) + "\n") |
| print( |
| json.dumps( |
| {mode: result["aggregate"] for mode, result in results.items()}, indent=2 |
| ) |
| ) |
|
|
