Spaces:
Running on Zero
Running on Zero
Download scripts/test_phase4_adversarial_verification.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 25.3 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/test_phase4_adversarial_verification.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/test_phase4_adversarial_verification.py
-
curl -L -o test_phase4_adversarial_verification.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/test_phase4_adversarial_verification.py
25.3 kB
| """ | |
| Adversarial Empirical Verification Oracle for Phase 4 Baseline Evaluation. | |
| Independently inspects and verifies: | |
| 1. benchmark/evaluation_manifest.csv | |
| 2. benchmark/results/phase4_baseline_predictions.csv | |
| 3. benchmark/results/phase4_baseline_metrics.json | |
| 4. docs/PHASE4_BASELINE_EVALUATION_REPORT.md | |
| """ | |
| import os | |
| import sys | |
| import csv | |
| import json | |
| import math | |
| from typing import Dict, List, Any, Tuple | |
| PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "evaluation_manifest.csv") | |
| PREDICTIONS_PATH = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase4_baseline_predictions.csv") | |
| METRICS_PATH = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase4_baseline_metrics.json") | |
| REPORT_PATH = os.path.join(PROJECT_ROOT, "docs", "PHASE4_BASELINE_EVALUATION_REPORT.md") | |
| def independent_wilson_ci(k: int, n: int, confidence: float = 0.95) -> List[float]: | |
| if n <= 0: | |
| return [0.0, 0.0] | |
| p_hat = k / n | |
| z = 1.959963984540054 # 95% standard normal quantile | |
| z2 = z * z | |
| denom = 1.0 + z2 / n | |
| center = (p_hat + z2 / (2.0 * n)) / denom | |
| margin = z * math.sqrt((p_hat * (1.0 - p_hat) / n) + (z2 / (4.0 * n * n))) / denom | |
| lower = max(0.0, center - margin) | |
| upper = min(1.0, center + margin) | |
| return [round(lower, 4), round(upper, 4)] | |
| def independent_cohen_kappa(both_fake: int, both_real: int, m1_f_m2_r: int, m1_r_m2_f: int) -> float: | |
| n = both_fake + both_real + m1_f_m2_r + m1_r_m2_f | |
| if n == 0: | |
| return 0.0 | |
| po = (both_fake + both_real) / n | |
| p_f1 = (both_fake + m1_f_m2_r) / n | |
| p_r1 = (m1_r_m2_f + both_real) / n | |
| p_f2 = (both_fake + m1_r_m2_f) / n | |
| p_r2 = (m1_f_m2_r + both_real) / n | |
| pe = (p_f1 * p_f2) + (p_r1 * p_r2) | |
| if abs(1.0 - pe) < 1e-9: | |
| return 1.0 if abs(po - 1.0) < 1e-9 else 0.0 | |
| kappa = (po - pe) / (1.0 - pe) | |
| return round(kappa, 4) | |
| def run_all_checks() -> Dict[str, Any]: | |
| print("=" * 80) | |
| print("ADVERSARIAL EMPIRICAL VERIFICATION ORACLE: PHASE 4 BASELINE EVALUATION") | |
| print("=" * 80) | |
| summary = { | |
| "manifest_checks": {}, | |
| "csv_integrity": {}, | |
| "metrics_recomputed": {}, | |
| "json_matches": {}, | |
| "report_matches": {}, | |
| "edge_case_checks": {}, | |
| "errors": [] | |
| } | |
| # --------------------------------------------------------- | |
| # 1. Manifest Split Audit | |
| # --------------------------------------------------------- | |
| print("\n[1] Auditing Manifest splits & sample IDs...") | |
| manifest_samples = {} | |
| split_counts = {} | |
| with open(MANIFEST_PATH, "r", encoding="utf-8") as f: | |
| reader = csv.DictReader(f) | |
| for row in reader: | |
| sid = row["sample_id"] | |
| split = row["split"] | |
| manifest_samples[sid] = row | |
| split_counts[split] = split_counts.get(split, 0) + 1 | |
| print(f" Total samples in manifest: {len(manifest_samples)}") | |
| print(f" Split breakdown: {split_counts}") | |
| assert split_counts.get("held_out_test") == 105, f"Expected 105 held_out_test, got {split_counts.get('held_out_test')}" | |
| assert split_counts.get("development_val") == 50, f"Expected 50 development_val, got {split_counts.get('development_val')}" | |
| assert split_counts.get("operational_smoke") == 4, f"Expected 4 operational_smoke, got {split_counts.get('operational_smoke')}" | |
| summary["manifest_checks"]["split_counts"] = split_counts | |
| held_out_ids = {sid for sid, row in manifest_samples.items() if row["split"] == "held_out_test"} | |
| dev_val_ids = {sid for sid, row in manifest_samples.items() if row["split"] == "development_val"} | |
| smoke_ids = {sid for sid, row in manifest_samples.items() if row["split"] == "operational_smoke"} | |
| # --------------------------------------------------------- | |
| # 2. CSV Integrity Audit | |
| # --------------------------------------------------------- | |
| print("\n[2] Auditing phase4_baseline_predictions.csv integrity...") | |
| assert os.path.exists(PREDICTIONS_PATH), f"Predictions CSV missing: {PREDICTIONS_PATH}" | |
| expected_columns = [ | |
| "sample_id", "split", "domain", "category", "ground_truth", | |
| "file_path", "detector_key", "detector_name", "model_name", | |
| "prediction", "predicted_class", "fake_probability", "real_probability", | |
| "raw_logit", "relative_logit", "confidence", "threshold", | |
| "status", "error_message", "latency_ms" | |
| ] | |
| pred_rows = [] | |
| with open(PREDICTIONS_PATH, "r", encoding="utf-8") as f: | |
| reader = csv.DictReader(f) | |
| assert reader.fieldnames == expected_columns, f"Columns mismatch: {reader.fieldnames} vs {expected_columns}" | |
| for row in reader: | |
| pred_rows.append(row) | |
| total_rows = len(pred_rows) | |
| print(f" Total prediction rows: {total_rows} (Expected 315 = 105 * 3)") | |
| assert total_rows == 315, f"Expected exactly 315 prediction rows, found {total_rows}" | |
| # Detector row counts & duplicates | |
| detector_keys = ["chen2024_convnext", "univfd", "vit"] | |
| detector_rows = {k: [] for k in detector_keys} | |
| seen_pairs = set() | |
| data_leakage_detected = [] | |
| probability_violations = [] | |
| for idx, row in enumerate(pred_rows): | |
| sid = row["sample_id"] | |
| dkey = row["detector_key"] | |
| split = row["split"] | |
| status = row["status"] | |
| gt = row["ground_truth"] | |
| pred = row["prediction"] | |
| # Check pair uniqueness | |
| pair = (dkey, sid) | |
| if pair in seen_pairs: | |
| raise AssertionError(f"Duplicate detector-sample pair detected: {pair}") | |
| seen_pairs.add(pair) | |
| # Check detector key valid | |
| assert dkey in detector_rows, f"Unexpected detector key: {dkey}" | |
| detector_rows[dkey].append(row) | |
| # STRICT DATA LEAKAGE CHECK | |
| if split != "held_out_test": | |
| data_leakage_detected.append(f"Row {idx}: {sid} has split '{split}', expected 'held_out_test'") | |
| if sid in dev_val_ids: | |
| data_leakage_detected.append(f"CRITICAL LEAK: {sid} is from development_val!") | |
| if sid in smoke_ids: | |
| data_leakage_detected.append(f"CRITICAL LEAK: {sid} is from operational_smoke!") | |
| if sid not in held_out_ids: | |
| data_leakage_detected.append(f"Row {idx}: {sid} not in held_out_test manifest!") | |
| # Probability and value validations | |
| if status == "SUCCESS": | |
| assert pred in ["REAL", "FAKE"], f"Invalid prediction: {pred}" | |
| assert row["predicted_class"] == pred, f"predicted_class mismatch: {row['predicted_class']} != {pred}" | |
| fake_p = float(row["fake_probability"]) | |
| real_p = float(row["real_probability"]) | |
| conf = float(row["confidence"]) | |
| thresh = float(row["threshold"]) | |
| lat = float(row["latency_ms"]) | |
| assert 0.0 <= fake_p <= 1.0, f"fake_probability out of bounds: {fake_p}" | |
| assert 0.0 <= real_p <= 1.0, f"real_probability out of bounds: {real_p}" | |
| assert abs(fake_p + real_p - 1.0) < 1e-4, f"Probabilities do not sum to 1: {fake_p} + {real_p}" | |
| assert 0.5 <= conf <= 1.0, f"Confidence out of bounds: {conf}" | |
| assert thresh == 0.5, f"Threshold != 0.5: {thresh}" | |
| assert lat > 0, f"Latency <= 0: {lat}" | |
| assert row["error_message"] == "", f"Unexpected error message in SUCCESS row: {row['error_message']}" | |
| elif status == "ERROR": | |
| assert pred == "ERROR", f"Error row prediction is not ERROR: {pred}" | |
| assert row["predicted_class"] == "ERROR", f"Error row predicted_class is not ERROR: {row['predicted_class']}" | |
| assert row["fake_probability"] == "", f"Error row has fake_probability: {row['fake_probability']}" | |
| assert row["real_probability"] == "", f"Error row has real_probability: {row['real_probability']}" | |
| assert row["raw_logit"] == "", f"Error row has raw_logit: {row['raw_logit']}" | |
| assert row["confidence"] == "", f"Error row has confidence: {row['confidence']}" | |
| assert row["latency_ms"] == "", f"Error row has latency_ms: {row['latency_ms']}" | |
| assert row["error_message"] != "", "Error row has empty error_message" | |
| assert gt == "REJECTION_CONTROL", f"Error row ground_truth is not REJECTION_CONTROL: {gt}" | |
| else: | |
| raise AssertionError(f"Unknown status: {status}") | |
| assert len(data_leakage_detected) == 0, f"DATA LEAKAGE DETECTED:\n" + "\n".join(data_leakage_detected) | |
| print(" [PASS] Zero data leakage: 100% of prediction rows belong to held_out_test.") | |
| print(" [PASS] Zero duplicate pairs found.") | |
| print(" [PASS] All probability constraints and field formats strictly satisfied.") | |
| summary["csv_integrity"]["total_rows"] = total_rows | |
| summary["csv_integrity"]["detectors_count"] = {k: len(v) for k, v in detector_rows.items()} | |
| summary["csv_integrity"]["data_leakage_detected"] = len(data_leakage_detected) | |
| # --------------------------------------------------------- | |
| # 3. Independent Metrics Recomputation from Scratch | |
| # --------------------------------------------------------- | |
| print("\n[3] Recomputing all metrics directly from phase4_baseline_predictions.csv...") | |
| with open(METRICS_PATH, "r", encoding="utf-8") as f: | |
| ground_truth_metrics = json.load(f) | |
| computed_metrics = {"detectors": {}, "pairwise_agreement": {}} | |
| for dkey in detector_keys: | |
| rows = detector_rows[dkey] | |
| assert len(rows) == 105 | |
| # 3.1 Separate samples by role | |
| eligible_rows = [r for r in rows if r["ground_truth"] in ["REAL", "FAKE"]] | |
| facial_rows = [r for r in rows if r["domain"] == "facial"] | |
| scene_rows = [r for r in rows if r["domain"] == "general_scene"] | |
| control_rows = [r for r in rows if r["ground_truth"] == "REJECTION_CONTROL"] | |
| assert len(eligible_rows) == 103, f"{dkey}: Expected 103 eligible rows, got {len(eligible_rows)}" | |
| assert len(facial_rows) == 100, f"{dkey}: Expected 100 facial rows, got {len(facial_rows)}" | |
| assert len(scene_rows) == 3, f"{dkey}: Expected 3 scene rows, got {len(scene_rows)}" | |
| assert len(control_rows) == 2, f"{dkey}: Expected 2 control rows, got {len(control_rows)}" | |
| # 3.2 Overall metrics (103 eligible samples) | |
| tp = sum(1 for r in eligible_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "FAKE") | |
| fp = sum(1 for r in eligible_rows if r["ground_truth"] == "REAL" and r["prediction"] == "FAKE") | |
| tn = sum(1 for r in eligible_rows if r["ground_truth"] == "REAL" and r["prediction"] == "REAL") | |
| fn = sum(1 for r in eligible_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "REAL") | |
| errs = sum(1 for r in eligible_rows if r["status"] == "ERROR") | |
| n_eligible = len(eligible_rows) | |
| acc = round((tp + tn) / n_eligible, 4) | |
| prec = round(tp / (tp + fp), 4) if (tp + fp) > 0 else 0.0 | |
| rec = round(tp / (tp + fn), 4) if (tp + fn) > 0 else 0.0 | |
| spec = round(tn / (tn + fp), 4) if (tn + fp) > 0 else 0.0 | |
| f1 = round(2.0 * prec * rec / (prec + rec), 4) if (prec + rec) > 0 else 0.0 | |
| bal_acc = round((rec + spec) / 2.0, 4) | |
| fps = [r["sample_id"] for r in eligible_rows if r["ground_truth"] == "REAL" and r["prediction"] == "FAKE"] | |
| fns = [r["sample_id"] for r in eligible_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "REAL"] | |
| # 3.3 Facial metrics (100 samples) | |
| f_tp = sum(1 for r in facial_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "FAKE") | |
| f_fp = sum(1 for r in facial_rows if r["ground_truth"] == "REAL" and r["prediction"] == "FAKE") | |
| f_tn = sum(1 for r in facial_rows if r["ground_truth"] == "REAL" and r["prediction"] == "REAL") | |
| f_fn = sum(1 for r in facial_rows if r["ground_truth"] == "FAKE" and r["prediction"] == "REAL") | |
| f_acc = round((f_tp + f_tn) / 100, 4) | |
| f_prec = round(f_tp / (f_tp + f_fp), 4) if (f_tp + f_fp) > 0 else 0.0 | |
| f_rec = round(f_tp / (f_tp + f_fn), 4) if (f_tp + f_fn) > 0 else 0.0 | |
| f_spec = round(f_tn / (f_tn + f_fp), 4) if (f_tn + f_fp) > 0 else 0.0 | |
| f_f1 = round(2.0 * f_prec * f_rec / (f_prec + f_rec), 4) if (f_prec + f_rec) > 0 else 0.0 | |
| f_bal_acc = round((f_rec + f_spec) / 2.0, 4) | |
| f_acc_ci = independent_wilson_ci(f_tp + f_tn, 100) | |
| f_prec_ci = independent_wilson_ci(f_tp, f_tp + f_fp) if (f_tp + f_fp) > 0 else [0.0, 0.0] | |
| f_rec_ci = independent_wilson_ci(f_tp, f_tp + f_fn) if (f_tp + f_fn) > 0 else [0.0, 0.0] | |
| f_spec_ci = independent_wilson_ci(f_tn, f_tn + f_fp) if (f_tn + f_fp) > 0 else [0.0, 0.0] | |
| real_face_probs = [float(r["fake_probability"]) for r in facial_rows if r["ground_truth"] == "REAL"] | |
| fake_face_probs = [float(r["fake_probability"]) for r in facial_rows if r["ground_truth"] == "FAKE"] | |
| mean_p_real = round(sum(real_face_probs) / len(real_face_probs), 4) | |
| mean_p_fake = round(sum(fake_face_probs) / len(fake_face_probs), 4) | |
| # 3.4 Rejection controls verification | |
| ctrl_dict = {} | |
| for r in control_rows: | |
| ctrl_dict[r["sample_id"]] = { | |
| "prediction": r["prediction"], | |
| "status": r["status"], | |
| "error_message": r["error_message"], | |
| "rejection_successful": (r["prediction"] == "ERROR" and r["status"] == "ERROR") | |
| } | |
| # 3.5 Scene diagnostic cases | |
| scene_dict = {} | |
| for r in scene_rows: | |
| scene_dict[r["sample_id"]] = { | |
| "prediction": r["prediction"], | |
| "ground_truth": r["ground_truth"], | |
| "fake_probability": round(float(r["fake_probability"]), 6), | |
| "correct": (r["prediction"] == r["ground_truth"]) | |
| } | |
| computed_metrics["detectors"][dkey] = { | |
| "overall": { | |
| "tp": tp, "fp": fp, "tn": tn, "fn": fn, | |
| "accuracy": acc, "precision": prec, "recall": rec, | |
| "specificity": spec, "f1_score": f1, "balanced_accuracy": bal_acc, | |
| "false_positives": sorted(fps), | |
| "false_negatives": sorted(fns), | |
| "errors": errs | |
| }, | |
| "facial": { | |
| "tp": f_tp, "fp": f_fp, "tn": f_tn, "fn": f_fn, | |
| "accuracy": f_acc, "precision": f_prec, "recall": f_rec, | |
| "specificity": f_spec, "f1_score": f_f1, "balanced_accuracy": f_bal_acc, | |
| "accuracy_95ci": f_acc_ci, | |
| "precision_95ci": f_prec_ci, | |
| "recall_95ci": f_rec_ci, | |
| "specificity_95ci": f_spec_ci, | |
| "mean_fake_prob_real_faces": mean_p_real, | |
| "mean_fake_prob_fake_faces": mean_p_fake | |
| }, | |
| "scene": scene_dict, | |
| "controls": ctrl_dict | |
| } | |
| # 3.6 Pairwise agreement recomputation | |
| pair_defs = [ | |
| ("chen2024_convnext", "univfd"), | |
| ("chen2024_convnext", "vit"), | |
| ("univfd", "vit") | |
| ] | |
| for d1, d2 in pair_defs: | |
| pair_key = f"{d1}_vs_{d2}" | |
| rows1 = {r["sample_id"]: r for r in detector_rows[d1]} | |
| rows2 = {r["sample_id"]: r for r in detector_rows[d2]} | |
| valid_sids = [ | |
| sid for sid in rows1 | |
| if rows1[sid]["status"] == "SUCCESS" and rows2[sid]["status"] == "SUCCESS" | |
| ] | |
| assert len(valid_sids) == 103, f"{pair_key}: Expected 103 valid joint samples, got {len(valid_sids)}" | |
| both_fake = sum(1 for sid in valid_sids if rows1[sid]["prediction"] == "FAKE" and rows2[sid]["prediction"] == "FAKE") | |
| both_real = sum(1 for sid in valid_sids if rows1[sid]["prediction"] == "REAL" and rows2[sid]["prediction"] == "REAL") | |
| m1_f_m2_r = sum(1 for sid in valid_sids if rows1[sid]["prediction"] == "FAKE" and rows2[sid]["prediction"] == "REAL") | |
| m1_r_m2_f = sum(1 for sid in valid_sids if rows1[sid]["prediction"] == "REAL" and rows2[sid]["prediction"] == "FAKE") | |
| concordant = both_fake + both_real | |
| discordant = m1_f_m2_r + m1_r_m2_f | |
| assert concordant + discordant == 103 | |
| rate = round(concordant / 103, 4) | |
| kappa = independent_cohen_kappa(both_fake, both_real, m1_f_m2_r, m1_r_m2_f) | |
| computed_metrics["pairwise_agreement"][pair_key] = { | |
| "valid_sample_count": 103, | |
| "concordant_samples": concordant, | |
| "discordant_samples": discordant, | |
| "concordance_rate": rate, | |
| "both_fake": both_fake, | |
| "both_real": both_real, | |
| "model1_fake_model2_real": m1_f_m2_r, | |
| "model1_real_model2_fake": m1_r_m2_f, | |
| "cohen_kappa": kappa | |
| } | |
| # --------------------------------------------------------- | |
| # 4. Strict Comparison against phase4_baseline_metrics.json | |
| # --------------------------------------------------------- | |
| print("\n[4] Comparing computed metrics against phase4_baseline_metrics.json...") | |
| discrepancies = [] | |
| for dkey in detector_keys: | |
| comp_d = computed_metrics["detectors"][dkey] | |
| json_d = ground_truth_metrics["detectors"][dkey] | |
| # Overall | |
| j_ov = json_d["overall_classification_metrics"] | |
| j_cm = j_ov["confusion_matrix"] | |
| c_ov = comp_d["overall"] | |
| for k in ["tp", "fp", "tn", "fn"]: | |
| if j_cm[k] != c_ov[k]: | |
| discrepancies.append(f"{dkey} overall {k}: JSON {j_cm[k]} != Computed {c_ov[k]}") | |
| for k in ["accuracy", "precision", "recall", "specificity", "f1_score", "balanced_accuracy"]: | |
| if abs(j_ov[k] - c_ov[k]) > 1e-4: | |
| discrepancies.append(f"{dkey} overall {k}: JSON {j_ov[k]} != Computed {c_ov[k]}") | |
| if sorted(j_ov["false_positives"]) != c_ov["false_positives"]: | |
| discrepancies.append(f"{dkey} overall false_positives mismatch") | |
| if sorted(j_ov["false_negatives"]) != c_ov["false_negatives"]: | |
| discrepancies.append(f"{dkey} overall false_negatives mismatch") | |
| if j_ov["execution_errors"] != []: | |
| discrepancies.append(f"{dkey} unexpected execution_errors in JSON: {j_ov['execution_errors']}") | |
| # Facial | |
| j_fc = json_d["facial_evaluation_metrics"] | |
| j_fcm = j_fc["confusion_matrix"] | |
| c_fc = comp_d["facial"] | |
| for k in ["tp", "fp", "tn", "fn"]: | |
| if j_fcm[k] != c_fc[k]: | |
| discrepancies.append(f"{dkey} facial {k}: JSON {j_fcm[k]} != Computed {c_fc[k]}") | |
| for k in ["accuracy", "precision", "recall", "specificity", "f1_score", "balanced_accuracy"]: | |
| if abs(j_fc[k] - c_fc[k]) > 1e-4: | |
| discrepancies.append(f"{dkey} facial {k}: JSON {j_fc[k]} != Computed {c_fc[k]}") | |
| for k in ["accuracy_95ci", "precision_95ci", "recall_95ci", "specificity_95ci"]: | |
| if [round(x, 4) for x in j_fc[k]] != [round(x, 4) for x in c_fc[k]]: | |
| discrepancies.append(f"{dkey} facial {k}: JSON {j_fc[k]} != Computed {c_fc[k]}") | |
| for k in ["mean_fake_prob_real_faces", "mean_fake_prob_fake_faces"]: | |
| if abs(j_fc[k] - c_fc[k]) > 1e-4: | |
| discrepancies.append(f"{dkey} facial {k}: JSON {j_fc[k]} != Computed {c_fc[k]}") | |
| # Controls | |
| j_rc = json_d["rejection_control_results"] | |
| c_rc = comp_d["controls"] | |
| for cid in ["test_control_corr_001", "test_control_unsupp_001"]: | |
| if not j_rc[cid]["rejection_successful"]: | |
| discrepancies.append(f"{dkey} control {cid} rejection not successful in JSON") | |
| if not c_rc[cid]["rejection_successful"]: | |
| discrepancies.append(f"{dkey} control {cid} rejection not successful in CSV") | |
| # Scenes | |
| j_sc = json_d["scene_diagnostic_results"] | |
| c_sc = comp_d["scene"] | |
| for sid in ["test_scene_real_001", "test_scene_diffusion_001", "test_scene_manip_001"]: | |
| if j_sc[sid]["prediction"] != c_sc[sid]["prediction"]: | |
| discrepancies.append(f"{dkey} scene {sid} prediction mismatch: JSON {j_sc[sid]['prediction']} != Computed {c_sc[sid]['prediction']}") | |
| if abs(j_sc[sid]["fake_probability"] - c_sc[sid]["fake_probability"]) > 1e-4: | |
| discrepancies.append(f"{dkey} scene {sid} fake_probability mismatch") | |
| # Pairwise agreement comparison | |
| for pair_key, comp_p in computed_metrics["pairwise_agreement"].items(): | |
| json_p = ground_truth_metrics["pairwise_agreement"][pair_key] | |
| for k in ["valid_sample_count", "concordant_samples", "discordant_samples"]: | |
| if json_p[k] != comp_p[k]: | |
| discrepancies.append(f"{pair_key} {k}: JSON {json_p[k]} != Computed {comp_p[k]}") | |
| if abs(json_p["concordance_rate"] - comp_p["concordance_rate"]) > 1e-4: | |
| discrepancies.append(f"{pair_key} concordance_rate: JSON {json_p['concordance_rate']} != Computed {comp_p['concordance_rate']}") | |
| if abs(json_p["cohen_kappa"] - comp_p["cohen_kappa"]) > 1e-4: | |
| discrepancies.append(f"{pair_key} cohen_kappa: JSON {json_p['cohen_kappa']} != Computed {comp_p['cohen_kappa']}") | |
| ab = json_p["agreement_breakdown"] | |
| for k in ["both_fake", "both_real", "model1_fake_model2_real", "model1_real_model2_fake"]: | |
| if ab[k] != comp_p[k]: | |
| discrepancies.append(f"{pair_key} {k}: JSON {ab[k]} != Computed {comp_p[k]}") | |
| if discrepancies: | |
| print(f" [FAIL] Found {len(discrepancies)} discrepancies between CSV and JSON metrics!") | |
| for d in discrepancies: | |
| print(f" - {d}") | |
| raise AssertionError("Discrepancies found between predictions CSV and metrics JSON") | |
| else: | |
| print(" [PASS] 100% exact match between recomputed metrics and phase4_baseline_metrics.json!") | |
| # --------------------------------------------------------- | |
| # 5. Cross-Verification against Markdown Report | |
| # --------------------------------------------------------- | |
| print("\n[5] Cross-verifying against docs/PHASE4_BASELINE_EVALUATION_REPORT.md...") | |
| with open(REPORT_PATH, "r", encoding="utf-8") as f: | |
| report_text = f.read() | |
| # Check key text patterns in report | |
| report_checks = [ | |
| # Confusion matrix numbers | |
| ("Chen2024ConvNeXtDetector | 6 | 14 | 37 | 46 | 103 | 0", "ConvNeXt 103 CM row"), | |
| ("UniversalFakeDetect | 11 | 1 | 50 | 41 | 103 | 0", "UnivFD 103 CM row"), | |
| ("ViTDeepfakeDetector | 50 | 0 | 51 | 2 | 103 | 0", "ViT 103 CM row"), | |
| # Overall metrics | |
| ("41.75% (43/103)", "ConvNeXt overall accuracy"), | |
| ("30.00% (6/20)", "ConvNeXt overall precision"), | |
| ("11.54% (6/52)", "ConvNeXt overall recall"), | |
| ("72.55% (37/51)", "ConvNeXt overall specificity"), | |
| ("0.1667", "ConvNeXt overall F1"), | |
| ("42.04%", "ConvNeXt overall balanced accuracy"), | |
| ("59.22% (61/103)", "UnivFD overall accuracy"), | |
| ("91.67% (11/12)", "UnivFD overall precision"), | |
| ("21.15% (11/52)", "UnivFD overall recall"), | |
| ("98.04% (50/51)", "UnivFD overall specificity"), | |
| ("0.3437", "UnivFD overall F1"), | |
| ("59.60%", "UnivFD overall balanced accuracy"), | |
| ("98.06% (101/103)", "ViT overall accuracy"), | |
| ("100.00% (50/50)", "ViT overall precision"), | |
| ("96.15% (50/52)", "ViT overall recall"), | |
| ("100.00% (51/51)", "ViT overall specificity"), | |
| ("0.9804", "ViT overall F1"), | |
| ("98.08%", "ViT overall balanced accuracy"), | |
| # Facial metrics & Wilson CIs | |
| ("42.00% (42/100)", "ConvNeXt facial accuracy"), | |
| ("[32.8%, 51.8%]", "ConvNeXt facial accuracy CI"), | |
| ("60.00% (60/100)", "UnivFD facial accuracy"), | |
| ("[50.2%, 69.1%]", "UnivFD facial accuracy CI"), | |
| ("100.00% (100/100)", "ViT facial accuracy"), | |
| ("[96.3%, 100.0%]", "ViT facial accuracy CI"), | |
| # Pairwise concordance & Kappa | |
| ("72.82%", "ConvNeXt vs UnivFD concordance rate"), | |
| ("-0.0241", "ConvNeXt vs UnivFD kappa"), | |
| ("43.69%", "ConvNeXt vs ViT concordance rate"), | |
| ("-0.1466", "ConvNeXt vs ViT kappa"), | |
| ("61.17%", "UnivFD vs ViT concordance rate"), | |
| ("0.2056", "UnivFD vs ViT kappa"), | |
| # Diagnostic cases | |
| ("test_scene_real_001", "Scene real ID"), | |
| ("test_scene_diffusion_001", "Scene diffusion ID"), | |
| ("test_scene_manip_001", "Scene manip ID"), | |
| ("test_control_corr_001", "Control corrupted ID"), | |
| ("test_control_unsupp_001", "Control unsupported ID") | |
| ] | |
| report_discrepancies = [] | |
| for pattern, desc in report_checks: | |
| if pattern not in report_text: | |
| report_discrepancies.append(f"Report missing expected string '{pattern}' ({desc})") | |
| if report_discrepancies: | |
| print(f" [FAIL] Report verification discrepancies found:") | |
| for rd in report_discrepancies: | |
| print(f" - {rd}") | |
| raise AssertionError("Report verification discrepancies found") | |
| else: | |
| print(" [PASS] 100% exact match between Markdown report and computed ground truth!") | |
| print("\n" + "=" * 80) | |
| print("ALL ADVERSARIAL EMPIRICAL CHECKS PASSED WITH ZERO DISCREPANCIES!") | |
| print("=" * 80) | |
| return computed_metrics | |
| if __name__ == "__main__": | |
| run_all_checks() | |