Spaces:
Running on Zero
Running on Zero
File size: 24,742 Bytes
151f388 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 | """
Independent Adversarial Empirical Verification Suite for Milestone 1: Phase 4 Baseline Evaluation.
Verifies data integrity, partition isolation, statistical metrics, Wilson CIs, non-degeneracy, and report consistency.
"""
import os
import sys
import json
import csv
import math
from typing import Dict, Any, List, Tuple
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MANIFEST_CSV = os.path.join(PROJECT_ROOT, "benchmark", "evaluation_manifest.csv")
PREDICTIONS_CSV = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase4_baseline_predictions.csv")
METRICS_JSON = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase4_baseline_metrics.json")
REPORT_MD = os.path.join(PROJECT_ROOT, "docs", "PHASE4_BASELINE_EVALUATION_REPORT.md")
def independent_wilson_ci(k: int, n: int, z: float = 1.959963984540054) -> Tuple[float, float]:
"""
Independent implementation of Wilson score interval:
center = (p + z^2 / (2n)) / (1 + z^2 / n)
margin = (z / (1 + z^2 / n)) * sqrt(p(1-p)/n + z^2 / (4n^2))
"""
if n <= 0:
return (0.0, 0.0)
p = k / n
z2 = z * z
denom = 1.0 + (z2 / n)
center = (p + (z2 / (2.0 * n))) / denom
margin = (z / denom) * math.sqrt((p * (1.0 - p) / n) + (z2 / (4.0 * n * n)))
lower = max(0.0, center - margin)
upper = min(1.0, center + margin)
return (round(lower, 4), round(upper, 4))
def independent_cohen_kappa(both_fake: int, both_real: int, m1_f_m2_r: int, m1_r_m2_f: int) -> float:
"""Independent implementation of Cohen's Kappa for 2x2 agreement matrix."""
n = both_fake + both_real + m1_f_m2_r + m1_r_m2_f
if n == 0:
return 0.0
po = (both_fake + both_real) / n
p_f1 = (both_fake + m1_f_m2_r) / n
p_r1 = (m1_r_m2_f + both_real) / n
p_f2 = (both_fake + m1_r_m2_f) / n
p_r2 = (m1_f_m2_r + both_real) / n
pe = (p_f1 * p_f2) + (p_r1 * p_r2)
if abs(1.0 - pe) < 1e-9:
return 1.0 if abs(po - 1.0) < 1e-9 else 0.0
kappa = (po - pe) / (1.0 - pe)
return round(kappa, 4)
def run_verification():
findings = []
successes = []
print("=" * 80)
print("INDEPENDENT ADVERSARIAL VERIFICATION: PHASE 4 BASELINE EVALUATION")
print("=" * 80)
# 1. Load Manifest
if not os.path.exists(MANIFEST_CSV):
raise FileNotFoundError(f"Manifest not found: {MANIFEST_CSV}")
manifest_rows = {}
with open(MANIFEST_CSV, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
for row in reader:
manifest_rows[row["sample_id"]] = row
held_out_ids = {sid for sid, r in manifest_rows.items() if r["split"] == "held_out_test"}
val_ids = {sid for sid, r in manifest_rows.items() if r["split"] == "development_val"}
smoke_ids = {sid for sid, r in manifest_rows.items() if r["split"] == "operational_smoke"}
print(f"Manifest loaded: Total={len(manifest_rows)}, held_out_test={len(held_out_ids)}, development_val={len(val_ids)}, operational_smoke={len(smoke_ids)}")
# 2. Load Predictions CSV
if not os.path.exists(PREDICTIONS_CSV):
raise FileNotFoundError(f"Predictions CSV not found: {PREDICTIONS_CSV}")
with open(PREDICTIONS_CSV, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
prediction_rows = list(reader)
print(f"Predictions CSV loaded: Total rows = {len(prediction_rows)}")
# Check 1: Partition Leakage & Contamination
print("\n--- CHECK 1: DATA CONTAMINATION & PARTITION LEAKAGE ---")
expected_detectors = {"chen2024_convnext", "univfd", "vit"}
detector_samples = {det: set() for det in expected_detectors}
detector_rows = {det: [] for det in expected_detectors}
for row in prediction_rows:
sid = row["sample_id"]
det = row["detector_key"]
split = row["split"]
if det not in expected_detectors:
findings.append(f"Unexpected detector_key in predictions: {det}")
continue
detector_samples[det].add(sid)
detector_rows[det].append(row)
# Check contamination
if sid in val_ids or "val_face_" in sid:
findings.append(f"CRITICAL CONTAMINATION: development_val sample {sid} found in predictions under {det}!")
if sid in smoke_ids or "smoke_" in sid:
findings.append(f"CRITICAL CONTAMINATION: operational_smoke sample {sid} found in predictions under {det}!")
if sid not in held_out_ids:
findings.append(f"CRITICAL CONTAMINATION: sample {sid} is not in held_out_test!")
if split != "held_out_test":
findings.append(f"Split column value '{split}' is not 'held_out_test' for sample {sid}")
# Check metadata consistency against manifest
m_row = manifest_rows.get(sid)
if m_row:
if m_row["ground_truth"] != row["ground_truth"]:
findings.append(f"Ground truth mismatch for {sid}: manifest={m_row['ground_truth']} vs pred_csv={row['ground_truth']}")
if m_row["domain"] != row["domain"]:
findings.append(f"Domain mismatch for {sid}: manifest={m_row['domain']} vs pred_csv={row['domain']}")
if m_row["category"] != row["category"]:
findings.append(f"Category mismatch for {sid}: manifest={m_row['category']} vs pred_csv={row['category']}")
for det in expected_detectors:
count = len(detector_samples[det])
total_rows = len(detector_rows[det])
if count != 105:
findings.append(f"Detector {det} evaluated {count} distinct samples, expected exactly 105!")
else:
successes.append(f"Detector {det} evaluated exactly 105 distinct samples.")
if total_rows != 105:
findings.append(f"Detector {det} has {total_rows} rows (duplicate sample IDs detected!)")
else:
successes.append(f"Detector {det} has exactly 105 rows (zero duplicate rows).")
# Check set difference between evaluated samples and held_out_ids
missing = held_out_ids - detector_samples[det]
if missing:
findings.append(f"Detector {det} is missing held_out samples: {missing}")
extra = detector_samples[det] - held_out_ids
if extra:
findings.append(f"Detector {det} has extra non-held_out samples: {extra}")
# Check 2: Row-level consistency in predictions.csv
print("\n--- CHECK 2: ROW-LEVEL CONSISTENCY & NUMERICAL VALIDITY ---")
for row in prediction_rows:
sid = row["sample_id"]
det = row["detector_key"]
status = row["status"]
pred = row["prediction"]
pred_class = row["predicted_class"]
gt = row["ground_truth"]
# Check status and prediction consistency
if gt == "REJECTION_CONTROL":
# Controls are expected to return ERROR prediction
if pred != "ERROR":
findings.append(f"Rejection control {sid} under {det} has prediction={pred}, expected ERROR")
if pred_class != "ERROR":
findings.append(f"Rejection control {sid} under {det} has predicted_class={pred_class}, expected ERROR")
if not row["error_message"]:
findings.append(f"Rejection control {sid} under {det} missing error_message")
if status != "ERROR":
findings.append(f"Rejection control {sid} under {det} has unexpected status: {status}")
successes.append(f"Rejection control {sid} gracefully rejected under {det} with prediction=ERROR.")
else:
if status != "SUCCESS":
findings.append(f"Sample {sid} under {det} has non-SUCCESS status: {status}")
if pred not in ("REAL", "FAKE"):
findings.append(f"Sample {sid} under {det} has non-standard prediction: {pred}")
if pred != pred_class:
findings.append(f"Sample {sid} under {det} prediction ({pred}) != predicted_class ({pred_class})")
# Check probability bounds and threshold consistency
try:
p_fake = float(row["fake_probability"])
p_real = float(row["real_probability"])
conf = float(row["confidence"])
thresh = float(row["threshold"])
latency = float(row["latency_ms"])
if not (0.0 <= p_fake <= 1.0):
findings.append(f"Sample {sid} under {det} p_fake out of bounds: {p_fake}")
if not (0.0 <= p_real <= 1.0):
findings.append(f"Sample {sid} under {det} p_real out of bounds: {p_real}")
if abs((p_fake + p_real) - 1.0) > 1e-4:
findings.append(f"Sample {sid} under {det} p_fake + p_real != 1.0: {p_fake + p_real}")
# Threshold decision consistency
expected_pred = "FAKE" if p_fake >= thresh else "REAL"
if pred != expected_pred:
findings.append(f"Sample {sid} under {det} prediction {pred} contradicts p_fake={p_fake} with threshold={thresh}")
expected_conf = max(p_fake, p_real)
if abs(conf - expected_conf) > 1e-4:
findings.append(f"Sample {sid} under {det} confidence {conf} != max prob {expected_conf}")
# Latency check: non-zero, positive, realistic
if latency <= 0:
findings.append(f"Sample {sid} under {det} has non-positive latency: {latency}")
except ValueError as ve:
findings.append(f"Sample {sid} under {det} numerical parse error: {ve}")
# Check 3: Model Non-Degeneracy
print("\n--- CHECK 3: MODEL DIVERSITY & NON-DEGENERACY ---")
for det in expected_detectors:
preds = [r["prediction"] for r in detector_rows[det] if r["ground_truth"] != "REJECTION_CONTROL"]
unique_preds = set(preds)
fake_probs = [float(r["fake_probability"]) for r in detector_rows[det] if r["ground_truth"] != "REJECTION_CONTROL"]
prob_variance = sum((p - (sum(fake_probs)/len(fake_probs)))**2 for p in fake_probs) / len(fake_probs)
print(f"Detector {det}: unique predictions = {unique_preds}, prob variance = {prob_variance:.6f}")
if len(unique_preds) < 2:
findings.append(f"DEGENERATE MODEL WARNING: Detector {det} produced only one prediction class: {unique_preds}")
else:
successes.append(f"Detector {det} produced diverse predictions: {unique_preds} with variance {prob_variance:.6f}")
# Check 4: Independent Metric Calculation vs JSON vs Report Markdown
print("\n--- CHECK 4: INDEPENDENT METRIC CALCULATION & CROSS-CHECK ---")
if not os.path.exists(METRICS_JSON):
raise FileNotFoundError(f"Metrics JSON not found: {METRICS_JSON}")
with open(METRICS_JSON, "r", encoding="utf-8") as f:
metrics_json = json.load(f)
with open(REPORT_MD, "r", encoding="utf-8") as f:
report_md = f.read()
for det in expected_detectors:
rows = detector_rows[det]
# 4a: Overall Eligible Samples (103 samples: 51 REAL, 52 FAKE)
overall_rows = [r for r in rows if r["ground_truth"] in ("REAL", "FAKE")]
if len(overall_rows) != 103:
findings.append(f"Detector {det} has {len(overall_rows)} overall eligible samples, expected 103")
tp = sum(1 for r in overall_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "FAKE")
fp = sum(1 for r in overall_rows if r["ground_truth"] == "REAL" and r["prediction"] == "FAKE")
tn = sum(1 for r in overall_rows if r["ground_truth"] == "REAL" and r["prediction"] == "REAL")
fn = sum(1 for r in overall_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "REAL")
n_valid = tp + fp + tn + fn
acc = (tp + tn) / n_valid if n_valid > 0 else 0.0
prec = tp / (tp + fp) if (tp + fp) > 0 else 0.0
rec = tp / (tp + fn) if (tp + fn) > 0 else 0.0
spec = tn / (tn + fp) if (tn + fp) > 0 else 0.0
f1 = (2 * prec * rec) / (prec + rec) if (prec + rec) > 0 else 0.0
bal_acc = (rec + spec) / 2.0
fps = [r["sample_id"] for r in overall_rows if r["ground_truth"] == "REAL" and r["prediction"] == "FAKE"]
fns = [r["sample_id"] for r in overall_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "REAL"]
# Cross-check with JSON
json_overall = metrics_json["detectors"][det]["overall_classification_metrics"]
cm_json = json_overall["confusion_matrix"]
if (cm_json["tp"], cm_json["fp"], cm_json["tn"], cm_json["fn"]) != (tp, fp, tn, fn):
findings.append(f"Overall confusion matrix mismatch for {det}: Calc={(tp, fp, tn, fn)} vs JSON={(cm_json['tp'], cm_json['fp'], cm_json['tn'], cm_json['fn'])}")
else:
successes.append(f"Overall confusion matrix matches for {det}: TP={tp}, FP={fp}, TN={tn}, FN={fn}")
if round(acc, 4) != json_overall["accuracy"]:
findings.append(f"Overall accuracy mismatch for {det}: Calc={round(acc, 4)} vs JSON={json_overall['accuracy']}")
if round(prec, 4) != json_overall["precision"]:
findings.append(f"Overall precision mismatch for {det}: Calc={round(prec, 4)} vs JSON={json_overall['precision']}")
if round(rec, 4) != json_overall["recall"]:
findings.append(f"Overall recall mismatch for {det}: Calc={round(rec, 4)} vs JSON={json_overall['recall']}")
if round(spec, 4) != json_overall["specificity"]:
findings.append(f"Overall specificity mismatch for {det}: Calc={round(spec, 4)} vs JSON={json_overall['specificity']}")
if round(f1, 4) != json_overall["f1_score"]:
findings.append(f"Overall F1 mismatch for {det}: Calc={round(f1, 4)} vs JSON={json_overall['f1_score']}")
if round(bal_acc, 4) != json_overall["balanced_accuracy"]:
findings.append(f"Overall balanced accuracy mismatch for {det}: Calc={round(bal_acc, 4)} vs JSON={json_overall['balanced_accuracy']}")
if fps != json_overall["false_positives"]:
findings.append(f"Overall FP list mismatch for {det}")
if fns != json_overall["false_negatives"]:
findings.append(f"Overall FN list mismatch for {det}")
# 4b: Facial Evaluation (100 samples: 50 REAL, 50 FAKE)
facial_rows = [r for r in rows if r["domain"] == "facial"]
if len(facial_rows) != 100:
findings.append(f"Detector {det} has {len(facial_rows)} facial samples, expected 100")
f_tp = sum(1 for r in facial_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "FAKE")
f_fp = sum(1 for r in facial_rows if r["ground_truth"] == "REAL" and r["prediction"] == "FAKE")
f_tn = sum(1 for r in facial_rows if r["ground_truth"] == "REAL" and r["prediction"] == "REAL")
f_fn = sum(1 for r in facial_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "REAL")
f_valid = f_tp + f_fp + f_tn + f_fn
f_acc = (f_tp + f_tn) / f_valid if f_valid > 0 else 0.0
f_prec = f_tp / (f_tp + f_fp) if (f_tp + f_fp) > 0 else 0.0
f_rec = f_tp / (f_tp + f_fn) if (f_tp + f_fn) > 0 else 0.0
f_spec = f_tn / (f_tn + f_fp) if (f_tn + f_fp) > 0 else 0.0
f_f1 = (2 * f_prec * f_rec) / (f_prec + f_rec) if (f_prec + f_rec) > 0 else 0.0
f_bal_acc = (f_rec + f_spec) / 2.0
# Wilson Confidence Intervals for facial
f_acc_ci = independent_wilson_ci(f_tp + f_tn, f_valid)
f_prec_ci = independent_wilson_ci(f_tp, f_tp + f_fp)
f_rec_ci = independent_wilson_ci(f_tp, f_tp + f_fn)
f_spec_ci = independent_wilson_ci(f_tn, f_tn + f_fp)
json_facial = metrics_json["detectors"][det]["facial_evaluation_metrics"]
f_cm_json = json_facial["confusion_matrix"]
if (f_cm_json["tp"], f_cm_json["fp"], f_cm_json["tn"], f_cm_json["fn"]) != (f_tp, f_fp, f_tn, f_fn):
findings.append(f"Facial confusion matrix mismatch for {det}: Calc={(f_tp, f_fp, f_tn, f_fn)} vs JSON={(f_cm_json['tp'], f_cm_json['fp'], f_cm_json['tn'], f_cm_json['fn'])}")
else:
successes.append(f"Facial confusion matrix matches for {det}: TP={f_tp}, FP={f_fp}, TN={f_tn}, FN={f_fn}")
if round(f_acc, 4) != json_facial["accuracy"]:
findings.append(f"Facial accuracy mismatch for {det}: Calc={round(f_acc, 4)} vs JSON={json_facial['accuracy']}")
if list(f_acc_ci) != json_facial["accuracy_95ci"]:
findings.append(f"Facial accuracy 95% CI mismatch for {det}: Calc={list(f_acc_ci)} vs JSON={json_facial['accuracy_95ci']}")
else:
successes.append(f"Facial accuracy 95% CI matches for {det}: {list(f_acc_ci)}")
if round(f_prec, 4) != json_facial["precision"]:
findings.append(f"Facial precision mismatch for {det}: Calc={round(f_prec, 4)} vs JSON={json_facial['precision']}")
if list(f_prec_ci) != json_facial["precision_95ci"]:
findings.append(f"Facial precision 95% CI mismatch for {det}: Calc={list(f_prec_ci)} vs JSON={json_facial['precision_95ci']}")
if round(f_rec, 4) != json_facial["recall"]:
findings.append(f"Facial recall mismatch for {det}: Calc={round(f_rec, 4)} vs JSON={json_facial['recall']}")
if list(f_rec_ci) != json_facial["recall_95ci"]:
findings.append(f"Facial recall 95% CI mismatch for {det}: Calc={list(f_rec_ci)} vs JSON={json_facial['recall_95ci']}")
if round(f_spec, 4) != json_facial["specificity"]:
findings.append(f"Facial specificity mismatch for {det}: Calc={round(f_spec, 4)} vs JSON={json_facial['specificity']}")
if list(f_spec_ci) != json_facial["specificity_95ci"]:
findings.append(f"Facial specificity 95% CI mismatch for {det}: Calc={list(f_spec_ci)} vs JSON={json_facial['specificity_95ci']}")
if round(f_f1, 4) != json_facial["f1_score"]:
findings.append(f"Facial F1 mismatch for {det}: Calc={round(f_f1, 4)} vs JSON={json_facial['f1_score']}")
if round(f_bal_acc, 4) != json_facial["balanced_accuracy"]:
findings.append(f"Facial balanced accuracy mismatch for {det}: Calc={round(f_bal_acc, 4)} vs JSON={json_facial['balanced_accuracy']}")
# Separation margin
real_probs = [float(r["fake_probability"]) for r in facial_rows if r["ground_truth"] == "REAL"]
fake_probs = [float(r["fake_probability"]) for r in facial_rows if r["ground_truth"] == "FAKE"]
mean_real = sum(real_probs) / len(real_probs) if real_probs else 0.0
mean_fake = sum(fake_probs) / len(fake_probs) if fake_probs else 0.0
sep_margin = mean_fake - mean_real
if round(mean_real, 4) != json_facial["mean_fake_prob_real_faces"]:
findings.append(f"Mean real prob mismatch for {det}: Calc={round(mean_real, 4)} vs JSON={json_facial['mean_fake_prob_real_faces']}")
if round(mean_fake, 4) != json_facial["mean_fake_prob_fake_faces"]:
findings.append(f"Mean fake prob mismatch for {det}: Calc={round(mean_fake, 4)} vs JSON={json_facial['mean_fake_prob_fake_faces']}")
# Verify separation margin in markdown report table 6.2
margin_str = f"**{sep_margin*100:+.2f}%**" if sep_margin >= 0 else f"**{sep_margin*100:.2f}%**"
if f"{sep_margin*100:.2f}%" not in report_md:
findings.append(f"Separation margin {sep_margin*100:.2f}% not found in report markdown for {det}")
else:
successes.append(f"Separation margin for {det} ({sep_margin*100:.2f}%) verified in markdown report.")
# Check 5: Pairwise Concordance
print("\n--- CHECK 5: PAIRWISE AGREEMENT & COHEN'S KAPPA ---")
pairs = [
("chen2024_convnext", "univfd"),
("chen2024_convnext", "vit"),
("univfd", "vit")
]
for det1, det2 in pairs:
pair_key = f"{det1}_vs_{det2}"
rows1 = {r["sample_id"]: r for r in detector_rows[det1] if r["ground_truth"] != "REJECTION_CONTROL"}
rows2 = {r["sample_id"]: r for r in detector_rows[det2] if r["ground_truth"] != "REJECTION_CONTROL"}
common_ids = sorted(list(set(rows1.keys()) & set(rows2.keys())))
if len(common_ids) != 103:
findings.append(f"Pair {pair_key} evaluated {len(common_ids)} common valid samples, expected 103")
both_fake = sum(1 for sid in common_ids if rows1[sid]["prediction"] == "FAKE" and rows2[sid]["prediction"] == "FAKE")
both_real = sum(1 for sid in common_ids if rows1[sid]["prediction"] == "REAL" and rows2[sid]["prediction"] == "REAL")
m1_f_m2_r = sum(1 for sid in common_ids if rows1[sid]["prediction"] == "FAKE" and rows2[sid]["prediction"] == "REAL")
m1_r_m2_f = sum(1 for sid in common_ids if rows1[sid]["prediction"] == "REAL" and rows2[sid]["prediction"] == "FAKE")
concordant = both_fake + both_real
discordant = m1_f_m2_r + m1_r_m2_f
total_p = concordant + discordant
conc_rate = concordant / total_p if total_p > 0 else 0.0
kappa = independent_cohen_kappa(both_fake, both_real, m1_f_m2_r, m1_r_m2_f)
json_pair = metrics_json["pairwise_agreement"][pair_key]
if json_pair["concordant_samples"] != concordant:
findings.append(f"Concordant samples mismatch for {pair_key}: Calc={concordant} vs JSON={json_pair['concordant_samples']}")
if json_pair["discordant_samples"] != discordant:
findings.append(f"Discordant samples mismatch for {pair_key}: Calc={discordant} vs JSON={json_pair['discordant_samples']}")
if round(conc_rate, 4) != json_pair["concordance_rate"]:
findings.append(f"Concordance rate mismatch for {pair_key}: Calc={round(conc_rate, 4)} vs JSON={json_pair['concordance_rate']}")
if round(kappa, 4) != json_pair["cohen_kappa"]:
findings.append(f"Cohen Kappa mismatch for {pair_key}: Calc={round(kappa, 4)} vs JSON={json_pair['cohen_kappa']}")
else:
successes.append(f"Pairwise agreement matches for {pair_key}: Concordant={concordant}/{total_p} ({conc_rate*100:.2f}%), Kappa={kappa}")
# Check 6: Markdown Report vs JSON cross-check
print("\n--- CHECK 6: MARKDOWN REPORT NARRATIVE & TABLES CONSISTENCY ---")
for det_key, det_metrics in metrics_json["detectors"].items():
det_name = det_metrics["model_info"]["detector_name"]
o_acc = det_metrics["overall_classification_metrics"]["accuracy_fraction"]
f_acc = det_metrics["facial_evaluation_metrics"]["accuracy_fraction"]
if o_acc not in report_md:
findings.append(f"Markdown report missing overall accuracy fraction {o_acc} for {det_name}")
if f_acc not in report_md:
findings.append(f"Markdown report missing facial accuracy fraction {f_acc} for {det_name}")
# Check 7: Latency Summary
print("\n--- CHECK 7: LATENCY STATISTICS ---")
for det in expected_detectors:
latencies = [float(r["latency_ms"]) for r in detector_rows[det] if r["latency_ms"] != ""]
min_lat = min(latencies)
max_lat = max(latencies)
avg_lat = sum(latencies) / len(latencies)
print(f"Detector {det} latency: count={len(latencies)}, min={min_lat:.2f}ms, avg={avg_lat:.2f}ms, max={max_lat:.2f}ms")
if min_lat <= 0:
findings.append(f"Detector {det} has non-positive min latency: {min_lat}")
else:
successes.append(f"Detector {det} latency is realistic: min={min_lat:.2f}ms, avg={avg_lat:.2f}ms, max={max_lat:.2f}ms")
# Check 8: Single-example Scene Diagnostics (3 cases)
print("\n--- CHECK 8: SCENE & COMPOSITE DIAGNOSTICS ---")
scene_cases = ["test_scene_real_001", "test_scene_diffusion_001", "test_scene_manip_001"]
for sc in scene_cases:
for det in expected_detectors:
row = next((r for r in detector_rows[det] if r["sample_id"] == sc), None)
if not row:
findings.append(f"Scene case {sc} missing for {det}")
else:
successes.append(f"Scene case {sc} evaluated under {det}: pred={row['prediction']}, p_fake={row['fake_probability']}")
# Summary
print("\n" + "=" * 80)
print("VERIFICATION SUMMARY")
print("=" * 80)
print(f"Total Successes Checked: {len(successes)}")
print(f"Total Findings / Discrepancies: {len(findings)}")
if findings:
print("\nDISCREPANCIES FOUND:")
for f in findings:
print(f" [X] {f}")
return False, findings, successes
else:
print("\nALL ADVERSARIAL CHECKS PASSED PERFECTLY!")
return True, findings, successes
if __name__ == "__main__":
passed, findings, successes = run_verification()
sys.exit(0 if passed else 1)
|