Spaces:
Running on Zero
Running on Zero
Download scripts/summarize_univfd_benchmark.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 4.02 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/summarize_univfd_benchmark.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/summarize_univfd_benchmark.py
-
curl -L -o summarize_univfd_benchmark.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/summarize_univfd_benchmark.py
4.02 kB
| """ | |
| UnivFD Benchmark Summarizer | |
| Generates an operational diagnostic summary from benchmark results. | |
| DISCLAIMER: This script does NOT compute accuracy, precision, recall, or F1 scores | |
| because the benchmark is a diagnostic operational harness and not a certified statistical dataset. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| from collections import Counter | |
| from typing import Dict, Any, List | |
| BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| RESULTS_FILE = os.path.join(BASE_DIR, "benchmark", "results", "univfd_results.json") | |
| def summarize_benchmark(): | |
| print("=" * 75) | |
| print("UNIVFD BENCHMARK OPERATIONAL SUMMARY") | |
| print("Harness: Diagnostic Operational Analysis") | |
| print("=" * 75 + "\n") | |
| if not os.path.exists(RESULTS_FILE): | |
| print(f"Results file not found: {RESULTS_FILE}") | |
| print("Please run scripts/run_univfd_benchmark.py first.") | |
| return | |
| with open(RESULTS_FILE, "r", encoding="utf-8") as f: | |
| records: List[Dict[str, Any]] = json.load(f) | |
| total_inputs = len(records) | |
| available_inputs = [r for r in records if r.get("available", False)] | |
| missing_inputs = [r for r in records if not r.get("available", False)] | |
| successful_inferences = [ | |
| r for r in available_inputs if r.get("prediction") in {"REAL", "FAKE", "UNCERTAIN"} | |
| ] | |
| rejected_or_corrupted = [ | |
| r for r in available_inputs if r.get("prediction") == "ERROR" | |
| ] | |
| predictions_counter = Counter(r.get("prediction") for r in records) | |
| categories_counter = Counter(r.get("category") for r in records) | |
| unknown_gt_entries = [r for r in records if r.get("ground_truth") == "UNKNOWN"] | |
| # Calculate average inference time over successful inferences | |
| times = [ | |
| r["inference_time_ms"] for r in successful_inferences | |
| if r.get("inference_time_ms") is not None | |
| ] | |
| avg_inference_time = sum(times) / len(times) if times else 0.0 | |
| print("--- 1. Input Inventory ---") | |
| print(f"Total entries registered: {total_inputs}") | |
| print(f"Physically available inputs: {len(available_inputs)}") | |
| print(f"Missing inputs on disk: {len(missing_inputs)}") | |
| print(f"UNKNOWN ground-truth labels: {len(unknown_gt_entries)}") | |
| print() | |
| print("--- 2. Inference & Error Breakdown ---") | |
| print(f"Successful model passes: {len(successful_inferences)}") | |
| print(f"Rejected / corrupted inputs: {len(rejected_or_corrupted)}") | |
| print(f"Average inference latency: {avg_inference_time:.2f} ms") | |
| print() | |
| print("--- 3. Prediction Counts ---") | |
| for pred, count in predictions_counter.items(): | |
| print(f" {pred:<25}: {count}") | |
| print() | |
| print("--- 4. Counts by Category ---") | |
| for cat, count in categories_counter.items(): | |
| print(f" {cat:<25}: {count}") | |
| print() | |
| print("--- 5. Missing / Skipped Inputs ---") | |
| if missing_inputs: | |
| for r in missing_inputs: | |
| print(f" - [{r['id']}] {r['filename']} ({r['error']})") | |
| else: | |
| print(" None. All registered inputs were present.") | |
| print() | |
| print("--- 6. Captured Errors & Warnings ---") | |
| warnings_found = set() | |
| errors_found = [] | |
| for r in records: | |
| if r.get("warning"): | |
| warnings_found.add(r["warning"]) | |
| if r.get("error"): | |
| errors_found.append((r["id"], r["error"])) | |
| print(f"Unique Warnings ({len(warnings_found)}):") | |
| for w in warnings_found: | |
| print(f" - \"{w}\"") | |
| print(f"Recorded Errors ({len(errors_found)}):") | |
| for eid, err in errors_found: | |
| print(f" - [{eid}]: {err}") | |
| print() | |
| print("=" * 75) | |
| print("IMPORTANT DISCLAIMER ON EVALUATION METRICS:") | |
| print("In accordance with scientific benchmarking standards, no accuracy, precision,") | |
| print("recall, or F1 scores are reported. The benchmark inputs serve as operational") | |
| print("diagnostics, not a statistically certified ground-truth evaluation dataset.") | |
| print("=" * 75) | |
| if __name__ == "__main__": | |
| summarize_benchmark() | |