"""Merge independently saved one-session browser benchmark reports. Running one fresh ORT session per page/report avoids conflating session teardown and reinitialization failures with the timed request population. The merged report is still passed through the strict raw-sample verifier. """ from __future__ import annotations import argparse import hashlib import json from pathlib import Path from typing import Any from verify_browser_benchmark import file_sha256, percentile, verify_report def _read_report(path: Path) -> dict[str, Any]: value = json.loads(path.read_text(encoding="utf-8")) if not isinstance(value, dict): raise TypeError(f"{path} must contain a JSON object") return value def _summary(samples: list[dict[str, Any]]) -> dict[str, Any]: successful = [sample for sample in samples if sample["status"] == "ok"] failed = len(samples) - len(successful) def values(field: str) -> list[float]: return [float(sample[field]) for sample in successful] def memory_values(field: str) -> list[float]: return [float(sample[field]) for sample in successful if sample.get(field) is not None] before = memory_values("js_heap_used_bytes_before") after = memory_values("js_heap_used_bytes_after") return { "successful_samples": len(successful), "failed_samples": failed, "total_request_ms_p50": percentile(values("totalRequestMs"), 0.5), "total_request_ms_p95": percentile(values("totalRequestMs"), 0.95), "inference_and_readback_ms_p50": percentile(values("inferenceAndReadbackMs"), 0.5), "inference_and_readback_ms_p95": percentile(values("inferenceAndReadbackMs"), 0.95), "js_heap_before_max": max(before) if before else None, "js_heap_after_max": max(after) if after else None, "memory_note": "performance.memory before/after observations only; not a peak or GPU-memory measurement", } def merge_reports(paths: list[Path]) -> dict[str, Any]: if len(paths) < 2: raise ValueError("at least two one-session reports are required") raw_reports = [_read_report(path) for path in paths] for path in paths: verify_report(path) baseline = raw_reports[0] comparable_keys = ( "backend", "requested_provider", "warmup_excluded_per_session", "repeats_per_session", "cases", "environment", "runtime_info", "execution_provider_evidence", ) for index, report in enumerate(raw_reports, start=1): if report.get("sessions") != 1: raise ValueError(f"source report {index} must have sessions=1") for key in comparable_keys: if report.get(key) != baseline.get(key): raise ValueError(f"source report {index} does not match baseline field {key}") source_hashes = [file_sha256(path) for path in paths] merged_run_id = "merged-" + hashlib.sha256("\n".join(source_hashes).encode()).hexdigest()[:16] samples: list[dict[str, Any]] = [] session_records: list[dict[str, Any]] = [] for session_start, report in enumerate(raw_reports, start=1): record = dict(report["session_records"][0]) record["session_start"] = session_start session_records.append(record) for raw_sample in report["samples"]: sample = dict(raw_sample) sample["source_run_id"] = sample.get("run_id") sample["run_id"] = merged_run_id sample["session_start"] = session_start samples.append(sample) return { "schema": "vons.browser-benchmark/v1", "run_id": merged_run_id, "backend": baseline["backend"], "requested_provider": baseline["requested_provider"], "sessions": len(raw_reports), "warmup_excluded_per_session": baseline["warmup_excluded_per_session"], "repeats_per_session": baseline["repeats_per_session"], "cases": baseline["cases"], "session_start_semantics": "one fresh page/session per source report; browser caches may persist", "execution_provider_evidence": baseline["execution_provider_evidence"], "environment": baseline["environment"], "runtime_info": baseline.get("runtime_info"), "merge": { "source_count": len(paths), "source_reports": [ {"path": str(path), "sha256": digest} for path, digest in zip(paths, source_hashes, strict=True) ], }, "session_records": session_records, "samples": samples, "summary": _summary(samples), } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--report", type=Path, action="append", required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() merged = merge_reports(args.report) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(json.dumps(merged, indent=2, sort_keys=True)) if __name__ == "__main__": main()