File size: 6,484 Bytes
932bc69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | """Aggregate a draft-checkpoint sweep into a single comparison table.
Reads ``<run>/perf_results.csv`` for every ``<variant>_spec<N>`` subdirectory of a
sweep directory and prints the per-subset acceptance / throughput / latency tables
plus a drafts-weighted overall row.
"""
import argparse
import csv
from pathlib import Path
SUBSET_ORDER = [
"HumanEval",
"math_reasoning",
"qa",
"question",
"rag",
"summarization",
"tool_call",
"translation",
"writing",
]
HEADER = """Qwen3.6-35B-A3B draft checkpoints, RedHatAI/speculator_benchmarks
single-GPU vLLM, concurrency 1, temperature 0, max_output_tokens 4096,
verifier Qwen/Qwen3.6-35B-A3B. Every subset runs its full row count
(HumanEval 164, tool_call 200, the other seven 80) -- the earlier
`max_requests 200/subset` cap never truncated any subset, and guidellm does
not resample, so capped and uncapped runs are the same measurement.
NOTE: `dflash_muon-*` was run alone on one card, sequentially; the other
columns come from 8 concurrent single-GPU replicas. Acceptance metrics
are per-draft and unaffected, but output_tps / itl / ttft carry a small
bias because the 8-replica runs contend for host CPU.
NOTE: the dataset ships question.jsonl and writing.jsonl with identical
content (same git oid), so those two rows are expected to match.
NOTE: acceptance positions are 0-indexed, so `pos14` is the 15th speculative
position. A run that proposes fewer tokens than a position asks for
leaves that cell blank rather than reporting a zero."""
SUBSET_WIDTH = 15
OVERALL_WIDTH = 22
COL_WIDTH = 15
# Acceptance rates are per-draft, so they aggregate across subsets weighted by
# each subset's draft count; throughput and latency medians do not.
DRAFTS_WEIGHTED_FIELDS = (
"acceptance_length",
"acceptance_at_pos_0",
"acceptance_at_pos_6",
"acceptance_at_pos_14",
)
def variant_sort_key(name: str) -> tuple[str, int]:
"""Sort ``dflash_spec15`` as ``("dflash", 15)`` so specs stay numeric."""
variant, _, spec = name.partition("_spec")
return variant, int(spec) if spec.isdigit() else 0
def label_of(run_dir: Path) -> str:
variant, _, spec = run_dir.name.partition("_spec")
return f"{variant}-{spec}"
def load_run(run_dir: Path) -> dict[str, dict[str, float]]:
rows = {}
with (run_dir / "perf_results.csv").open(newline="") as handle:
for row in csv.DictReader(handle):
rows[row["subset"]] = {
k: float(v)
for k, v in row.items()
if k not in ("subset", "strategy") and v not in ("", None)
}
return rows
def format_table(
title: str,
labels: list[str],
data: dict[str, dict[str, dict[str, float]]],
field: str,
digits: int,
) -> list[str]:
lines = [f"=== {title} ===", "subset".ljust(SUBSET_WIDTH)]
lines[1] += "".join(label.rjust(COL_WIDTH) for label in labels)
for subset in SUBSET_ORDER:
line = subset.ljust(SUBSET_WIDTH)
for label in labels:
value = data[label].get(subset, {}).get(field)
cell = "-" if value is None else f"{value:.{digits}f}"
line += cell.rjust(COL_WIDTH)
lines.append(line)
return lines
def overall_row(rows: dict[str, dict[str, float]], field: str) -> float | None:
"""Drafts-weighted for acceptance fields, plain mean for perf fields.
Returns ``None`` when no subset reports the field. That is how a run with
fewer speculative positions than the row asks for stays blank: a spec=7 run
has no ``acceptance_at_pos_14`` column at all, and averaging the positions
it does have would silently answer a different question.
"""
subsets = [s for s in SUBSET_ORDER if s in rows and field in rows[s]]
if not subsets:
return None
if field in DRAFTS_WEIGHTED_FIELDS:
drafts = sum(rows[s]["num_drafts"] for s in subsets)
if field == "acceptance_length":
# per-subset acceptance_length == accepted/drafts + 1
accepted = sum(rows[s]["num_accepted_tokens"] for s in subsets)
return accepted / drafts + 1.0
return sum(rows[s]["num_drafts"] * rows[s][field] for s in subsets) / drafts
return sum(rows[s][field] for s in subsets) / len(subsets)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("run_dir", type=Path, help="sweep directory")
parser.add_argument(
"-o",
"--output",
type=Path,
default=None,
help="write to this file instead of stdout",
)
args = parser.parse_args()
run_dirs = sorted(
(p for p in args.run_dir.iterdir() if (p / "perf_results.csv").is_file()),
key=lambda p: variant_sort_key(p.name),
)
if not run_dirs:
raise SystemExit(f"no perf_results.csv found under {args.run_dir}")
labels = [label_of(p) for p in run_dirs]
data = {label_of(p): load_run(p) for p in run_dirs}
lines = [HEADER, ""]
for title, field, digits in (
("Acceptance length", "acceptance_length", 3),
("Output tokens/s (median)", "output_tps_median", 2),
("Inter-token latency ms (median)", "itl_median_ms", 2),
("TTFT ms (median)", "ttft_median_ms", 2),
):
lines += format_table(title, labels, data, field, digits)
lines.append("")
lines.append(
"=== Overall (drafts-weighted acceptance, "
"arithmetic mean of subset medians) ==="
)
head = "metric".ljust(OVERALL_WIDTH)
lines.append(head + "".join(label.rjust(COL_WIDTH) for label in labels))
for metric, field, digits in (
("acceptance_length", "acceptance_length", 3),
("pos0 accept rate", "acceptance_at_pos_0", 4),
("pos6 accept rate", "acceptance_at_pos_6", 4),
("pos14 accept rate", "acceptance_at_pos_14", 4),
("output_tps", "output_tps_median", 2),
("itl_ms", "itl_median_ms", 2),
):
line = metric.ljust(OVERALL_WIDTH)
for label in labels:
value = overall_row(data[label], field)
cell = "" if value is None else f"{value:.{digits}f}"
line += cell.rjust(COL_WIDTH)
lines.append(line.rstrip())
text = "\n".join(lines) + "\n"
if args.output:
args.output.write_text(text)
print(f"wrote {args.output}")
else:
print(text, end="")
if __name__ == "__main__":
main()
|