| |
| """Tabulate the vision performance record: the ds41 vision surface, one boot. |
| |
| Reads the three attach passes (vision-run-N.csv, the speed-bench CSV standard |
| with the vision extras) and the server's own per-request log lines |
| (tower-lines.txt: the tower seconds and the WxH patch -> token plan, in |
| request order), then writes tables.md, summary.json and vision-rows.png |
| beside them. Fails loud, never silently drops a row. |
| """ |
| import csv |
| import json |
| import math |
| import re |
| from pathlib import Path |
| from statistics import median |
|
|
| import matplotlib |
|
|
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
|
|
| DATA = Path(__file__).with_name("ds41-spark-vision-2026-10-14") |
| PASSES = 3 |
| IMAGES = ["review-market-panel.png", "bug.jpeg", "homer.png"] |
| TEXT_ROWS = 2 |
| GENERATED = 128 |
| HOMER_TOKENS = 59 |
| TOWER_LINE = re.compile(r"vision tower: (\d+) image\(s\) ([0-9.]+)s") |
| PLAN_LINE = re.compile(r"image \d+: (\d+x\d+) patch -> (\d+x\d+) token \(span (\d+)\)") |
| COLOR = "#087f8c" |
|
|
| COLUMNS = ["ctx_tokens", "prefill_tokens", "prefill_cached_tokens", |
| "prefill_tps", "gen_tokens", "gen_tps", "first_token_sec", |
| "tok_per_step", "wall_s", "clock_mhz", "kind", "image", |
| "patch", "span", "tower_s"] |
|
|
|
|
| def read(path): |
| with path.open() as source: |
| rows = list(csv.DictReader(source)) |
| assert list(rows[0].keys()) == COLUMNS, f"{path.name}: unexpected columns" |
| return rows |
|
|
|
|
| def stats(values): |
| return {"median": median(values), "min": min(values), "max": max(values), |
| "samples": values} |
|
|
|
|
| |
| |
| |
| |
| def load_log(): |
| towers, plans = [], [] |
| for line in (DATA / "tower-lines.txt").read_text().splitlines(): |
| tower = TOWER_LINE.search(line) |
| if tower: |
| assert int(tower.group(1)) == 1, f"multi-image tower line: {line}" |
| towers.append(float(tower.group(2))) |
| continue |
| plan = PLAN_LINE.search(line) |
| if plan: |
| plans.append({"patch": plan.group(1), "token_grid": plan.group(2), |
| "span": int(plan.group(3))}) |
| assert len(towers) == PASSES * len(IMAGES), f"{len(towers)} tower lines" |
| assert len(plans) == PASSES * len(IMAGES), f"{len(plans)} plan lines" |
| assert plans[0:3] == plans[3:6] == plans[6:9], "the image plans differ across passes" |
| return towers, plans |
|
|
|
|
| def load_passes(): |
| passes = [read(DATA / f"vision-run-{i}.csv") for i in range(1, PASSES + 1)] |
| for index, rows in enumerate(passes, start=1): |
| assert len(rows) == len(IMAGES) + TEXT_ROWS, f"vision-run-{index}: rows" |
| assert [row["kind"] for row in rows] == ["image"] * 3 + ["text"] * 2, \ |
| f"vision-run-{index}: row kinds" |
| assert [row["image"] for row in rows[:3]] == IMAGES, \ |
| f"vision-run-{index}: fixture order" |
| for row in rows: |
| assert int(row["prefill_cached_tokens"]) == 0, \ |
| f"vision-run-{index}: reused prompt tokens" |
| for key in ("prefill_tps", "gen_tps", "first_token_sec"): |
| value = float(row[key]) |
| assert math.isfinite(value) and value > 0, f"vision-run-{index}: {key}" |
| generations = [int(row["gen_tokens"]) for row in rows] |
| assert generations == [GENERATED, GENERATED, HOMER_TOKENS] + [GENERATED] * 2, \ |
| f"vision-run-{index}: generated tokens {generations}" |
| return passes |
|
|
|
|
| def vision_report(passes, towers, plans): |
| rows = [] |
| for index in range(len(passes[0])): |
| first = passes[0][index] |
| entry = {"tag": first["image"] if first["kind"] == "image" |
| else f"text {int(first['ctx_tokens'])} tok", |
| "kind": first["kind"], "image": first["image"]} |
| if first["kind"] == "image": |
| position = IMAGES.index(first["image"]) |
| entry["patch"] = plans[position]["patch"] |
| entry["token_grid"] = plans[position]["token_grid"] |
| entry["span"] = plans[position]["span"] |
| entry["tower_s"] = stats([towers[p * len(IMAGES) + position] |
| for p in range(PASSES)]) |
| for key in ("ctx_tokens", "prefill_tps", "gen_tps", "first_token_sec"): |
| entry[key] = stats([float(run[index][key]) for run in passes]) |
| rows.append(entry) |
| return {"passes": PASSES, "rows": rows, |
| "tower_lines": towers, "plan_lines": plans} |
|
|
|
|
| def write_tables(report): |
| lines = [ |
| "# DeepSeek V4.1 Flash vision on the feat/optimize head", |
| "", |
| "One DGX Spark / GB10, one vision server process, three attach passes.", |
| "Image rows send the repo fixtures with one fixed prompt; text rows", |
| "bracket the image spans. Tower seconds and the per-image plan come", |
| "from the server's own log lines (tower-lines.txt). Every sample is", |
| "retained in the run CSVs; medians carry the min-max band.", |
| "", |
| "[Conditions and evidence](README.md).", |
| "", |
| "## Rows (median of three passes)", |
| "", |
| "| Row | Kind | Prompt tokens | Span | Patch -> token | Tower, s | TTFT, ms | Prefill, tok/s | Decode, tok/s |", |
| "| --- | --- | ---: | ---: | --- | ---: | ---: | ---: | ---: |", |
| ] |
| for row in report["rows"]: |
| if row["kind"] == "image": |
| tower = (f"{row['tower_s']['median']:.2f} " |
| f"({row['tower_s']['min']:.2f}-{row['tower_s']['max']:.2f})") |
| span = row["span"] |
| plan = f"{row['patch']} -> {row['token_grid']}" |
| else: |
| tower, span, plan = "-", "-", "-" |
| lines.append( |
| f"| {row['tag']} | {row['kind']} | {row['ctx_tokens']['median']:,.0f} | " |
| f"{span} | {plan} | {tower} | " |
| f"{row['first_token_sec']['median'] * 1000:,.0f} " |
| f"({row['first_token_sec']['min'] * 1000:,.0f}-{row['first_token_sec']['max'] * 1000:,.0f}) | " |
| f"{row['prefill_tps']['median']:,.1f} | {row['gen_tps']['median']:,.2f} |") |
| lines += ["", |
| "Image-row TTFT includes the tower seconds; the prefill rate is the", |
| "server's own prefill timer over the whole prompt build, so it also", |
| "includes the tower. homer.png ends early (finish=stop, 59 tokens)", |
| "in every pass.", |
| ""] |
| (DATA / "tables.md").write_text("\n".join(lines) + "\n") |
|
|
|
|
| def plot(report): |
| rows = report["rows"] |
| tags = [row["tag"] for row in rows] |
| towers = [row["tower_s"]["median"] * 1000 if row["kind"] == "image" else 0.0 |
| for row in rows] |
| ttfts = [row["first_token_sec"]["median"] * 1000 for row in rows] |
| residuals = [ttft - tower for ttft, tower in zip(ttfts, towers)] |
| lo = [row["first_token_sec"]["min"] * 1000 for row in rows] |
| hi = [row["first_token_sec"]["max"] * 1000 for row in rows] |
|
|
| fig, (left, right) = plt.subplots(1, 2, figsize=(11.5, 4.8)) |
| xs = range(len(rows)) |
| left.bar(xs, towers, color="#b45309", label="Tower") |
| left.bar(xs, residuals, bottom=towers, color=COLOR, label="Rest of TTFT") |
| left.errorbar(xs, ttfts, |
| yerr=[[t - l for t, l in zip(ttfts, lo)], |
| [h - t for t, h in zip(ttfts, hi)]], |
| fmt="none", ecolor="#333333", elinewidth=1, capsize=3) |
| left.set_xticks(xs) |
| left.set_xticklabels(tags, rotation=18, ha="right", fontsize=8) |
| left.set_ylabel("First token, ms") |
| left.set_title("TTFT (tower share highlighted)") |
| left.legend(fontsize=8) |
|
|
| width = 0.38 |
| prefill = [row["prefill_tps"]["median"] for row in rows] |
| decode = [row["gen_tps"]["median"] for row in rows] |
| right.bar([x - width / 2 for x in xs], prefill, width, color=COLOR, label="Prefill") |
| right.bar([x + width / 2 for x in xs], decode, width, color="#b45309", label="Decode") |
| right.set_xticks(xs) |
| right.set_xticklabels(tags, rotation=18, ha="right", fontsize=8) |
| right.set_ylabel("tok/s") |
| right.set_title("Reported rates") |
| right.legend(fontsize=8) |
|
|
| fig.suptitle("DeepSeek V4.1 Flash vision on one DGX Spark") |
| fig.tight_layout() |
| fig.savefig(DATA / "vision-rows.png", dpi=180) |
|
|
|
|
| def main(): |
| towers, plans = load_log() |
| passes = load_passes() |
| report = vision_report(passes, towers, plans) |
| write_tables(report) |
| (DATA / "summary.json").write_text(json.dumps(report, indent=2) + "\n") |
| plot(report) |
| print("tables.md, summary.json, vision-rows.png written") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
