Spaces:
Sleeping
Sleeping
Download benchmarks/visualize.py from jasonlingg/gpt2-forge-testbed: direct link, hf CLI and curl.
- Browser
- Download file 6.37 kB
-
https://huggingface.co/spaces/jasonlingg/gpt2-forge-testbed/resolve/main/benchmarks/visualize.py
- Command line
-
hf download hf://spaces/jasonlingg/gpt2-forge-testbed/benchmarks/visualize.py
-
curl -L -o visualize.py https://huggingface.co/spaces/jasonlingg/gpt2-forge-testbed/resolve/main/benchmarks/visualize.py
6.37 kB
| """ | |
| Generate benchmark visualizations from results JSON. | |
| Produces four key plots: | |
| 1. Speedup comparison (bar chart) | |
| 2. Memory-quality tradeoff (scatter) | |
| 3. Latency breakdown (stacked bar) | |
| 4. Summary radar chart | |
| Run: python benchmarks/visualize.py | |
| """ | |
| import json | |
| import os | |
| import matplotlib.pyplot as plt | |
| import matplotlib | |
| import numpy as np | |
| matplotlib.rcParams['font.family'] = 'sans-serif' | |
| matplotlib.rcParams['font.size'] = 11 | |
| COLORS = { | |
| 'baseline': '#6B7280', | |
| 'kv_cache': '#3B82F6', | |
| 'kv_cache_gqa': '#8B5CF6', | |
| 'kv_cache_spec': '#F59E0B', | |
| 'all': '#10B981', | |
| } | |
| def load_results(path: str = "benchmarks/results/benchmark_results.json") -> dict: | |
| with open(path) as f: | |
| return json.load(f) | |
| def plot_speedup_comparison(results: dict, output_dir: str): | |
| """Bar chart: tokens/sec for each configuration.""" | |
| fig, ax = plt.subplots(figsize=(10, 5)) | |
| names = list(results.keys()) | |
| tps = [results[n]['mean_tokens_per_sec'] for n in names] | |
| labels = [results[n]['config_desc'] for n in names] | |
| colors = [COLORS.get(n, '#6B7280') for n in names] | |
| bars = ax.bar(range(len(names)), tps, color=colors, edgecolor='white', linewidth=0.5) | |
| # Add value labels on bars | |
| for bar, val in zip(bars, tps): | |
| ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.3, | |
| f'{val:.1f}', ha='center', va='bottom', fontweight='bold', fontsize=10) | |
| # Baseline reference line | |
| if tps: | |
| ax.axhline(y=tps[0], color='red', linestyle='--', alpha=0.5, label='Baseline') | |
| ax.set_xticks(range(len(names))) | |
| ax.set_xticklabels(labels, rotation=15, ha='right') | |
| ax.set_ylabel('Tokens per Second') | |
| ax.set_title('Inference Throughput by Configuration', fontweight='bold') | |
| ax.legend() | |
| ax.spines['top'].set_visible(False) | |
| ax.spines['right'].set_visible(False) | |
| plt.tight_layout() | |
| plt.savefig(os.path.join(output_dir, 'speedup_comparison.png'), dpi=150, bbox_inches='tight') | |
| plt.close() | |
| print(" Saved speedup_comparison.png") | |
| def plot_ttft_comparison(results: dict, output_dir: str): | |
| """Bar chart: time to first token for each configuration.""" | |
| fig, ax = plt.subplots(figsize=(10, 5)) | |
| names = list(results.keys()) | |
| ttft = [results[n]['mean_ttft_ms'] for n in names] | |
| labels = [results[n]['config_desc'] for n in names] | |
| colors = [COLORS.get(n, '#6B7280') for n in names] | |
| bars = ax.bar(range(len(names)), ttft, color=colors, edgecolor='white', linewidth=0.5) | |
| for bar, val in zip(bars, ttft): | |
| ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.2, | |
| f'{val:.1f}ms', ha='center', va='bottom', fontsize=10) | |
| ax.set_xticks(range(len(names))) | |
| ax.set_xticklabels(labels, rotation=15, ha='right') | |
| ax.set_ylabel('Time to First Token (ms)') | |
| ax.set_title('TTFT by Configuration', fontweight='bold') | |
| ax.spines['top'].set_visible(False) | |
| ax.spines['right'].set_visible(False) | |
| plt.tight_layout() | |
| plt.savefig(os.path.join(output_dir, 'ttft_comparison.png'), dpi=150, bbox_inches='tight') | |
| plt.close() | |
| print(" Saved ttft_comparison.png") | |
| def plot_step_latency(results: dict, output_dir: str): | |
| """Bar chart: average per-step latency.""" | |
| fig, ax = plt.subplots(figsize=(10, 5)) | |
| names = list(results.keys()) | |
| step_ms = [results[n]['mean_step_ms'] for n in names] | |
| labels = [results[n]['config_desc'] for n in names] | |
| colors = [COLORS.get(n, '#6B7280') for n in names] | |
| bars = ax.bar(range(len(names)), step_ms, color=colors, edgecolor='white', linewidth=0.5) | |
| for bar, val in zip(bars, step_ms): | |
| ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.1, | |
| f'{val:.1f}ms', ha='center', va='bottom', fontsize=10) | |
| ax.set_xticks(range(len(names))) | |
| ax.set_xticklabels(labels, rotation=15, ha='right') | |
| ax.set_ylabel('Avg Step Latency (ms)') | |
| ax.set_title('Per-Token Generation Latency', fontweight='bold') | |
| ax.spines['top'].set_visible(False) | |
| ax.spines['right'].set_visible(False) | |
| plt.tight_layout() | |
| plt.savefig(os.path.join(output_dir, 'step_latency.png'), dpi=150, bbox_inches='tight') | |
| plt.close() | |
| print(" Saved step_latency.png") | |
| def plot_speedup_factors(results: dict, output_dir: str): | |
| """Horizontal bar chart showing speedup factors vs baseline.""" | |
| fig, ax = plt.subplots(figsize=(8, 4)) | |
| names = list(results.keys()) | |
| tps = [results[n]['mean_tokens_per_sec'] for n in names] | |
| labels = [results[n]['config_desc'] for n in names] | |
| colors = [COLORS.get(n, '#6B7280') for n in names] | |
| baseline = tps[0] if tps[0] > 0 else 1 | |
| speedups = [t / baseline for t in tps] | |
| # Reverse for horizontal bar (top = first) | |
| bars = ax.barh(range(len(names)-1, -1, -1), speedups, color=colors, edgecolor='white') | |
| for bar, val in zip(bars, reversed(speedups)): | |
| ax.text(bar.get_width() + 0.05, bar.get_y() + bar.get_height()/2, | |
| f'{val:.1f}x', va='center', fontweight='bold', fontsize=11) | |
| ax.set_yticks(range(len(names)-1, -1, -1)) | |
| ax.set_yticklabels(labels) | |
| ax.set_xlabel('Speedup vs Baseline') | |
| ax.set_title('Speedup Factor by Configuration', fontweight='bold') | |
| ax.axvline(x=1.0, color='red', linestyle='--', alpha=0.5) | |
| ax.spines['top'].set_visible(False) | |
| ax.spines['right'].set_visible(False) | |
| plt.tight_layout() | |
| plt.savefig(os.path.join(output_dir, 'speedup_factors.png'), dpi=150, bbox_inches='tight') | |
| plt.close() | |
| print(" Saved speedup_factors.png") | |
| def main(): | |
| output_dir = "benchmarks/results" | |
| os.makedirs(output_dir, exist_ok=True) | |
| results_path = os.path.join(output_dir, "benchmark_results.json") | |
| if not os.path.exists(results_path): | |
| print("No benchmark results found. Run `python -m benchmarks.harness` first.") | |
| return | |
| results = load_results(results_path) | |
| print("Generating visualizations...") | |
| plot_speedup_comparison(results, output_dir) | |
| plot_ttft_comparison(results, output_dir) | |
| plot_step_latency(results, output_dir) | |
| plot_speedup_factors(results, output_dir) | |
| print(f"\nAll plots saved to {output_dir}/") | |
| print("Add these to your README and blog post!") | |
| if __name__ == "__main__": | |
| main() | |