File size: 4,019 Bytes
7591a5b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
"""
UnivFD Benchmark Summarizer
Generates an operational diagnostic summary from benchmark results.
DISCLAIMER: This script does NOT compute accuracy, precision, recall, or F1 scores
because the benchmark is a diagnostic operational harness and not a certified statistical dataset.
"""

import os
import sys
import json
from collections import Counter
from typing import Dict, Any, List

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RESULTS_FILE = os.path.join(BASE_DIR, "benchmark", "results", "univfd_results.json")


def summarize_benchmark():
    print("=" * 75)
    print("UNIVFD BENCHMARK OPERATIONAL SUMMARY")
    print("Harness: Diagnostic Operational Analysis")
    print("=" * 75 + "\n")

    if not os.path.exists(RESULTS_FILE):
        print(f"Results file not found: {RESULTS_FILE}")
        print("Please run scripts/run_univfd_benchmark.py first.")
        return

    with open(RESULTS_FILE, "r", encoding="utf-8") as f:
        records: List[Dict[str, Any]] = json.load(f)

    total_inputs = len(records)
    available_inputs = [r for r in records if r.get("available", False)]
    missing_inputs = [r for r in records if not r.get("available", False)]

    successful_inferences = [
        r for r in available_inputs if r.get("prediction") in {"REAL", "FAKE", "UNCERTAIN"}
    ]
    rejected_or_corrupted = [
        r for r in available_inputs if r.get("prediction") == "ERROR"
    ]

    predictions_counter = Counter(r.get("prediction") for r in records)
    categories_counter = Counter(r.get("category") for r in records)
    unknown_gt_entries = [r for r in records if r.get("ground_truth") == "UNKNOWN"]

    # Calculate average inference time over successful inferences
    times = [
        r["inference_time_ms"] for r in successful_inferences
        if r.get("inference_time_ms") is not None
    ]
    avg_inference_time = sum(times) / len(times) if times else 0.0

    print("--- 1. Input Inventory ---")
    print(f"Total entries registered:    {total_inputs}")
    print(f"Physically available inputs: {len(available_inputs)}")
    print(f"Missing inputs on disk:      {len(missing_inputs)}")
    print(f"UNKNOWN ground-truth labels: {len(unknown_gt_entries)}")
    print()

    print("--- 2. Inference & Error Breakdown ---")
    print(f"Successful model passes:     {len(successful_inferences)}")
    print(f"Rejected / corrupted inputs: {len(rejected_or_corrupted)}")
    print(f"Average inference latency:   {avg_inference_time:.2f} ms")
    print()

    print("--- 3. Prediction Counts ---")
    for pred, count in predictions_counter.items():
        print(f"  {pred:<25}: {count}")
    print()

    print("--- 4. Counts by Category ---")
    for cat, count in categories_counter.items():
        print(f"  {cat:<25}: {count}")
    print()

    print("--- 5. Missing / Skipped Inputs ---")
    if missing_inputs:
        for r in missing_inputs:
            print(f"  - [{r['id']}] {r['filename']} ({r['error']})")
    else:
        print("  None. All registered inputs were present.")
    print()

    print("--- 6. Captured Errors & Warnings ---")
    warnings_found = set()
    errors_found = []
    for r in records:
        if r.get("warning"):
            warnings_found.add(r["warning"])
        if r.get("error"):
            errors_found.append((r["id"], r["error"]))

    print(f"Unique Warnings ({len(warnings_found)}):")
    for w in warnings_found:
        print(f"  - \"{w}\"")
    print(f"Recorded Errors ({len(errors_found)}):")
    for eid, err in errors_found:
        print(f"  - [{eid}]: {err}")
    print()

    print("=" * 75)
    print("IMPORTANT DISCLAIMER ON EVALUATION METRICS:")
    print("In accordance with scientific benchmarking standards, no accuracy, precision,")
    print("recall, or F1 scores are reported. The benchmark inputs serve as operational")
    print("diagnostics, not a statistically certified ground-truth evaluation dataset.")
    print("=" * 75)


if __name__ == "__main__":
    summarize_benchmark()