| """ |
| Benchmark Report - Report generation for benchmark results |
| """ |
|
|
| import json |
| from typing import List, Optional |
| import pandas as pd |
|
|
|
|
| def generate_report(results_file: str, output_file: Optional[str] = None) -> str: |
| """ |
| Generate benchmark report |
| |
| Args: |
| results_file: Path to results JSON file |
| output_file: Path to output report file, if None prints to console |
| |
| Returns: |
| Report text |
| """ |
| with open(results_file, "r", encoding="utf-8") as f: |
| results = json.load(f) |
|
|
| config = results["config"] |
| metrics = results["metrics"] |
|
|
| |
| report_lines = [] |
| append_line = lambda line: report_lines.append(line) |
| append_line("=" * 80) |
| append_line("Diffulex Benchmark Report") |
| append_line("=" * 80) |
| append_line("") |
| append_line("Configuration:") |
| append_line(f" Model: {config.get('model_path', 'N/A')}") |
| append_line(f" Model Name: {config.get('model_name', 'N/A')}") |
| append_line(f" Decoding Strategy: {config.get('decoding_strategy', 'N/A')}") |
| append_line(f" Dataset: {config.get('dataset_name', 'N/A')}") |
| append_line(f" Tensor Parallel Size: {config.get('tensor_parallel_size', 'N/A')}") |
| append_line(f" Data Parallel Size: {config.get('data_parallel_size', 'N/A')}") |
| append_line("") |
| append_line("Metrics:") |
| append_line(f" Number of Samples: {metrics.get('num_samples', 'N/A')}") |
| append_line(f" Total Tokens: {metrics.get('total_tokens', 'N/A')}") |
| append_line(f" Average Tokens per Sample: {metrics.get('avg_tokens_per_sample', 0):.2f}") |
| append_line(f" Average NFE: {metrics.get('avg_nfe', 0):.2f}") |
| append_line(f" Total Time: {metrics.get('total_time', 0):.2f} seconds") |
| append_line(f" E2E Time: {metrics.get('e2e_total_time_s', 0):.2f} seconds") |
| append_line(f" TTFT: {metrics.get('ttft_s', 0):.2f} seconds") |
| append_line(f" TPOT: {metrics.get('tpot_s', 0):.2f} seconds") |
| append_line(f" E2E Th: {metrics.get('e2e_throughput_tok_s', 0):.2f} tok/s") |
| append_line(f" Prefill Th: {metrics.get('prefill_throughput_tok_s', 0):.2f} tok/s") |
| append_line(f" Decode Th: {metrics.get('decode_throughput_tok_s', 0):.2f} tok/s") |
|
|
| if "accuracy" in metrics and metrics["accuracy"] is not None: |
| report_lines.append(f" Accuracy: {metrics['accuracy']:.4f}") |
|
|
| report_lines.append("") |
| report_lines.append(f"Timestamp: {results.get('timestamp', 'N/A')}") |
| report_lines.append("=" * 80) |
|
|
| report_text = "\n".join(report_lines) |
|
|
| |
| if output_file: |
| with open(output_file, "w", encoding="utf-8") as f: |
| f.write(report_text) |
| print(f"Report saved to: {output_file}") |
| else: |
| print(report_text) |
|
|
| return report_text |
|
|
|
|
| def compare_results(result_files: List[str], output_file: Optional[str] = None) -> pd.DataFrame: |
| """ |
| Compare multiple benchmark results |
| |
| Args: |
| result_files: List of result file paths |
| output_file: Path to output CSV file, if None only returns DataFrame |
| |
| Returns: |
| DataFrame with comparison results |
| """ |
| rows = [] |
|
|
| for result_file in result_files: |
| with open(result_file, "r", encoding="utf-8") as f: |
| results = json.load(f) |
|
|
| config = results["config"] |
| metrics = results["metrics"] |
|
|
| row = { |
| "model_path": config.get("model_path", "N/A"), |
| "model_name": config.get("model_name", "N/A"), |
| "decoding_strategy": config.get("decoding_strategy", "N/A"), |
| "dataset": config.get("dataset_name", "N/A"), |
| "num_samples": metrics.get("num_samples", 0), |
| "total_tokens": metrics.get("total_tokens", 0), |
| "avg_tokens_per_sample": metrics.get("avg_tokens_per_sample", 0), |
| "avg_nfe": metrics.get("avg_nfe", 0), |
| "e2e_total_time_s": metrics.get("e2e_total_time_s", 0), |
| "ttft_s": metrics.get("ttft_s", 0), |
| "tpot_s": metrics.get("tpot_s", 0), |
| "e2e_throughput_tok_s": metrics.get("e2e_throughput_tok_s", 0), |
| "prefill_throughput_tok_s": metrics.get("prefill_throughput_tok_s", 0), |
| "decode_throughput_tok_s": metrics.get("decode_throughput_tok_s", 0), |
| "accuracy": metrics.get("accuracy", None), |
| "timestamp": results.get("timestamp", "N/A"), |
| } |
| rows.append(row) |
|
|
| df = pd.DataFrame(rows) |
|
|
| if output_file: |
| df.to_csv(output_file, index=False, encoding="utf-8") |
| print(f"Comparison saved to: {output_file}") |
|
|
| return df |
|
|
