Spaces:
Running on Zero
Running on Zero
File size: 3,606 Bytes
61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | """
Benchmark Summary Generator
Dynamically parses detector_comparison.json and prints an objective, comparative summary
for all evaluated detectors without combining predictions.
"""
import os
import sys
import json
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RESULTS_FILE = os.path.join(BASE_DIR, "benchmark", "results", "detector_comparison.json")
def summarize():
if not os.path.exists(RESULTS_FILE):
print(f"Results file not found: {RESULTS_FILE}")
return
with open(RESULTS_FILE, "r", encoding="utf-8") as f:
records = json.load(f)
if not records:
print("No benchmark records found.")
return
total = len(records)
first_record = records[0]
# Discover detectors dynamically from record keys (ending with '_prediction')
detector_keys = [
k[:-11] for k in first_record.keys()
if k.endswith("_prediction")
]
agreements = [r for r in records if r.get("agreement_status") == "AGREE"]
disagreements = [r for r in records if r.get("agreement_status") == "DISAGREE"]
col_width = 18
header_detectors = " | ".join(f"{d.upper() + ' (Conf)':<{col_width}}" for d in detector_keys)
table_header = f"{'ID':<10} | {'Category':<14} | {'Label':<12} | {header_detectors} | {'Status'}"
separator = "-" * len(table_header)
print("=" * len(table_header))
print(f"MULTI-MODEL DETECTOR BENCHMARK SUMMARY ({len(detector_keys)} Detectors Registered)")
print(f"Detectors evaluated independently: {', '.join(detector_keys)}")
print("Policy: Independent evaluation. Predictions NOT combined into ensemble.")
print("=" * len(table_header))
print(table_header)
print(separator)
for r in records:
entry_id = r["id"]
cat = r["category"]
lbl = r["explicit_label"]
det_cols = []
for d in detector_keys:
pred = r.get(f"{d}_prediction", "N/A")
conf_val = r.get(f"{d}_confidence")
conf_str = f"{conf_val * 100:.1f}%" if conf_val is not None else "N/A"
det_cols.append(f"{pred} ({conf_str}):<{col_width}")
# Format nicely
det_cols[-1] = f"{pred} ({conf_str})"[:col_width].ljust(col_width)
det_str = " | ".join(det_cols)
status = r.get("agreement_status", "UNKNOWN")
print(f"{entry_id:<10} | {cat:<14} | {lbl:<12} | {det_str} | {status}")
print("=" * len(table_header))
print(f"TOTAL SAMPLES: {total}")
print(f"IDENTICAL PREDICTIONS: {len(agreements)} / {total} ({len(agreements)/total*100:.1f}%)")
print(f"DISAGREEMENTS: {len(disagreements)} / {total} ({len(disagreements)/total*100:.1f}%)")
print("=" * len(table_header))
if disagreements:
print("\nDETAILED DISAGREEMENTS (Predictions NOT combined):")
for d in disagreements:
print(f" Sample ID: {d['id']} ({d['filename']})")
print(f" Explicit Label: {d['explicit_label']}")
for det in detector_keys:
print(f" {det.upper()} Output: {d.get(f'{det}_prediction')} (fake_prob={d.get(f'{det}_fake_prob')}, logit={d.get(f'{det}_logit')})")
print(" Note: Each detector operated in total isolation; predictions are not combined.\n")
else:
print("\nObservational Concurrence: Both detectors produced identical discrete predictions across all 5 samples.")
print("Note: Discrete agreement at 0.50 threshold does NOT indicate identical feature representations or accuracy.")
if __name__ == "__main__":
summarize()
|