File size: 4,572 Bytes
31dc8dc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | """
Benchmark Report - Report generation for benchmark results
"""
import json
from typing import List, Optional
import pandas as pd
def generate_report(results_file: str, output_file: Optional[str] = None) -> str:
"""
Generate benchmark report
Args:
results_file: Path to results JSON file
output_file: Path to output report file, if None prints to console
Returns:
Report text
"""
with open(results_file, "r", encoding="utf-8") as f:
results = json.load(f)
config = results["config"]
metrics = results["metrics"]
# Generate report
report_lines = []
append_line = lambda line: report_lines.append(line)
append_line("=" * 80)
append_line("Diffulex Benchmark Report")
append_line("=" * 80)
append_line("")
append_line("Configuration:")
append_line(f" Model: {config.get('model_path', 'N/A')}")
append_line(f" Model Name: {config.get('model_name', 'N/A')}")
append_line(f" Decoding Strategy: {config.get('decoding_strategy', 'N/A')}")
append_line(f" Dataset: {config.get('dataset_name', 'N/A')}")
append_line(f" Tensor Parallel Size: {config.get('tensor_parallel_size', 'N/A')}")
append_line(f" Data Parallel Size: {config.get('data_parallel_size', 'N/A')}")
append_line("")
append_line("Metrics:")
append_line(f" Number of Samples: {metrics.get('num_samples', 'N/A')}")
append_line(f" Total Tokens: {metrics.get('total_tokens', 'N/A')}")
append_line(f" Average Tokens per Sample: {metrics.get('avg_tokens_per_sample', 0):.2f}")
append_line(f" Average NFE: {metrics.get('avg_nfe', 0):.2f}")
append_line(f" Total Time: {metrics.get('total_time', 0):.2f} seconds")
append_line(f" E2E Time: {metrics.get('e2e_total_time_s', 0):.2f} seconds")
append_line(f" TTFT: {metrics.get('ttft_s', 0):.2f} seconds")
append_line(f" TPOT: {metrics.get('tpot_s', 0):.2f} seconds")
append_line(f" E2E Th: {metrics.get('e2e_throughput_tok_s', 0):.2f} tok/s")
append_line(f" Prefill Th: {metrics.get('prefill_throughput_tok_s', 0):.2f} tok/s")
append_line(f" Decode Th: {metrics.get('decode_throughput_tok_s', 0):.2f} tok/s")
if "accuracy" in metrics and metrics["accuracy"] is not None:
report_lines.append(f" Accuracy: {metrics['accuracy']:.4f}")
report_lines.append("")
report_lines.append(f"Timestamp: {results.get('timestamp', 'N/A')}")
report_lines.append("=" * 80)
report_text = "\n".join(report_lines)
# Save or output
if output_file:
with open(output_file, "w", encoding="utf-8") as f:
f.write(report_text)
print(f"Report saved to: {output_file}")
else:
print(report_text)
return report_text
def compare_results(result_files: List[str], output_file: Optional[str] = None) -> pd.DataFrame:
"""
Compare multiple benchmark results
Args:
result_files: List of result file paths
output_file: Path to output CSV file, if None only returns DataFrame
Returns:
DataFrame with comparison results
"""
rows = []
for result_file in result_files:
with open(result_file, "r", encoding="utf-8") as f:
results = json.load(f)
config = results["config"]
metrics = results["metrics"]
row = {
"model_path": config.get("model_path", "N/A"),
"model_name": config.get("model_name", "N/A"),
"decoding_strategy": config.get("decoding_strategy", "N/A"),
"dataset": config.get("dataset_name", "N/A"),
"num_samples": metrics.get("num_samples", 0),
"total_tokens": metrics.get("total_tokens", 0),
"avg_tokens_per_sample": metrics.get("avg_tokens_per_sample", 0),
"avg_nfe": metrics.get("avg_nfe", 0),
"e2e_total_time_s": metrics.get("e2e_total_time_s", 0),
"ttft_s": metrics.get("ttft_s", 0),
"tpot_s": metrics.get("tpot_s", 0),
"e2e_throughput_tok_s": metrics.get("e2e_throughput_tok_s", 0),
"prefill_throughput_tok_s": metrics.get("prefill_throughput_tok_s", 0),
"decode_throughput_tok_s": metrics.get("decode_throughput_tok_s", 0),
"accuracy": metrics.get("accuracy", None),
"timestamp": results.get("timestamp", "N/A"),
}
rows.append(row)
df = pd.DataFrame(rows)
if output_file:
df.to_csv(output_file, index=False, encoding="utf-8")
print(f"Comparison saved to: {output_file}")
return df
|