File size: 4,075 Bytes
932bc69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | """Validate every smoke request and aggregate acceptance/active-run throughput."""
import csv
import json
import sys
from pathlib import Path
root = Path(sys.argv[1])
subsets = (
"HumanEval",
"math_reasoning",
"qa",
"question",
"rag",
"summarization",
"tool_call",
"translation",
"writing",
)
results = {}
table = [
"# NGram-Candidates functional evaluation",
"",
"50-step checkpoint; 4 requests per subset, concurrency 1, spec=7, "
"at most 512 output tokens. This is not a convergence/quality evaluation.",
"",
"Throughput is successful output tokens divided by active benchmark "
"duration, excluding server startup and gaps between subsets. "
"Aggregate acceptance is weighted by draft count, "
"not an average of subset ratios.",
"",
"| Mode | Subset | Requests | Acceptance length | Output tokens/s |",
"| --- | --- | ---: | ---: | ---: |",
]
for mode in ("greedy", "rs"):
with (root / mode / "acceptance.csv").open() as source:
acceptance = {row["subset"]: row for row in csv.DictReader(source)}
if set(acceptance) != set(subsets):
raise RuntimeError(
f"Incomplete acceptance subsets for {mode}: {sorted(acceptance)}"
)
rows = []
for subset in subsets:
artifact = json.loads(
(root / mode / "artifacts" / f"run_{subset}.json").read_text()
)
if len(artifact["benchmarks"]) != 1:
raise RuntimeError(f"Expected one throughput benchmark for {mode}/{subset}")
benchmark = artifact["benchmarks"][0]
counts = benchmark["scheduler_metrics"]["requests_made"]
if counts != {"successful": 4, "errored": 0, "incomplete": 0, "total": 4}:
raise RuntimeError(f"Unsuccessful requests in {mode}/{subset}: {counts}")
output_tokens = benchmark["metrics"]["output_token_count"]["successful"][
"total_sum"
]
duration = benchmark["duration"]
if output_tokens <= 0 or duration <= 0:
raise RuntimeError(f"Empty output or invalid duration in {mode}/{subset}")
spec = acceptance[subset]
drafts = float(spec["num_drafts"])
accepted = float(spec["num_accepted_tokens"])
if drafts <= 0 or float(spec["num_draft_tokens"]) != 7 * drafts:
raise RuntimeError(f"Expected actual spec=7 decoding in {mode}/{subset}")
row = {
"subset": subset,
"successful_requests": 4,
"output_tokens": output_tokens,
"duration_seconds": duration,
"output_tokens_per_second": output_tokens / duration,
"num_drafts": drafts,
"num_accepted_tokens": accepted,
"acceptance_length": 1 + accepted / drafts,
}
rows.append(row)
table.append(
f"| {mode} | {subset} | 4 | {row['acceptance_length']:.4f} | "
f"{row['output_tokens_per_second']:.2f} |"
)
aggregate = {
key: sum(row[key] for row in rows)
for key in (
"successful_requests",
"output_tokens",
"duration_seconds",
"num_drafts",
"num_accepted_tokens",
)
}
aggregate["acceptance_length"] = (
1 + aggregate["num_accepted_tokens"] / aggregate["num_drafts"]
)
aggregate["output_tokens_per_second"] = (
aggregate["output_tokens"] / aggregate["duration_seconds"]
)
results[mode] = {"subsets": rows, "aggregate": aggregate}
table.append(
f"| {mode} | **ALL** | 36 | {aggregate['acceptance_length']:.4f} | "
f"{aggregate['output_tokens_per_second']:.2f} |"
)
report = {
"passed": True,
"scope": "functional smoke only; not full-epoch convergence or a quality benchmark",
"modes": results,
}
(root / "summary.json").write_text(json.dumps(report, indent=2) + "\n")
(root / "summary.md").write_text("\n".join(table) + "\n")
print(
json.dumps(
{mode: result["aggregate"] for mode, result in results.items()}, indent=2
)
)
|