Spaces:

vipan-kumar
/

Audio_Deepfake_Detection

Sleeping

Audio_Deepfake_Detection / backend /app /utils /benchmark_data.py

Initial commit: Audio Deepfake Detector with 8 detectors trained on jay15k

e6a1f55 5 months ago

18.7 kB

"""Static benchmark numbers from the paper, expanded to match the
seven-architecture / multi-dataset / multi-metric scope of the user's
research summary.
Sources are paper-reported figures verbatim where available and clearly
marked ``None`` where the paper did not report a particular cell.
"""
from __future__ import annotations

from typing import Any, Dict, List


PAPER_TITLE = "The Generalization Gap in Audio Deepfake Detection: A Four-Paradigm Review"


# --------------------------------------------------------------------------- #
# Datasets surveyed
# --------------------------------------------------------------------------- #

DATASETS: List[Dict[str, Any]] = [
    {
        "id": "asvspoof_2021_df",
        "name": "ASVspoof 2021 DF",
        "size": "533,928 clips",
        "scope": "100+ TTS / VC systems with codec degradation",
        "use": "controlled cross-system generalization",
        "primary_metrics": ["EER", "minDCF", "t-DCF"],
    },
    {
        "id": "asvspoof_2021_la",
        "name": "ASVspoof 2021 LA",
        "size": "~187K clips",
        "scope": "logical-access attacks (TTS/VC), telephony codec",
        "use": "ASV-aware spoofing benchmarks",
        "primary_metrics": ["EER", "minDCF", "t-DCF"],
    },
    {
        "id": "asvspoof_2024",
        "name": "ASVspoof 2024",
        "size": "challenge-track scale",
        "scope": "latest challenge edition; advanced attacks + adversarial",
        "use": "current-state SOTA evaluation",
        "primary_metrics": ["EER", "minDCF", "t-DCF"],
    },
    {
        "id": "in_the_wild",
        "name": "In-the-Wild (Müller et al. 2022)",
        "size": "37.9 h",
        "scope": "documented real-world fraud + matched bonafide",
        "use": "DF21 → ITW generalization gap",
        "primary_metrics": ["EER"],
    },
    {
        "id": "ctrsvdd",
        "name": "CtrSVDD",
        "size": "controllable-spoof corpus",
        "scope": "diverse synthesis families",
        "use": "Nes2Net efficient-backend evaluation",
        "primary_metrics": ["EER", "CLLR"],
    },
    {
        "id": "partial_spoof",
        "name": "PartialSpoof",
        "size": "segment-level annotations",
        "scope": "real audio with localized spoofed segments",
        "use": "partial-segment detection",
        "primary_metrics": ["Seg-EER", "Seg-F1"],
    },
    {
        "id": "voiceradar_custom",
        "name": "VoiceRadar custom corpus",
        "size": "500K+ samples",
        "scope": "8 TTS + 4 voice-conversion systems",
        "use": "physics-augmented detection + adversarial robustness",
        "primary_metrics": ["EER", "TPR", "TNR", "PGD-Robust"],
    },
    {
        "id": "dailytalkedit",
        "name": "DailyTalkEdit",
        "size": "edited-conversation dataset",
        "scope": "semantic edits in spoken dialogue",
        "use": "explainable spoof + edit-localization",
        "primary_metrics": ["Accuracy", "Macro-F1", "LLM-Judge"],
    },
    {
        "id": "spoof_celeb",
        "name": "SpoofCeleb",
        "size": "celebrity voice-clone audio",
        "scope": "high-profile target speakers",
        "use": "voice-clone detection",
        "primary_metrics": ["Accuracy", "Macro-F1"],
    },
    {
        "id": "had",
        "name": "HAD (Half-truth Audio Detection)",
        "size": "manipulated-segment corpus",
        "scope": "partially manipulated speech",
        "use": "segment-level forgery detection",
        "primary_metrics": ["Seg-F1", "Macro-F1"],
    },
]


# --------------------------------------------------------------------------- #
# Metric definitions
# --------------------------------------------------------------------------- #

METRICS: List[Dict[str, Any]] = [
    {
        "id": "EER",
        "name": "Equal Error Rate",
        "scope": "spoof detection",
        "description": "Operating point where FAR = FRR. Lower is better.",
    },
    {
        "id": "minDCF",
        "name": "min Detection Cost Function",
        "scope": "cost-weighted detection",
        "description": "Cost-weighted error at optimal operating point.",
    },
    {
        "id": "t-DCF",
        "name": "tandem DCF",
        "scope": "joint ASV + spoof",
        "description": "Joint cost between ASV and spoof detector.",
    },
    {
        "id": "CLLR",
        "name": "Cost of Log-Likelihood Ratio",
        "scope": "calibration",
        "description": "Calibration-aware error of soft decisions.",
    },
    {
        "id": "TPR",
        "name": "True Positive Rate",
        "scope": "binary detection",
        "description": "Sensitivity / recall on the positive (fake) class.",
    },
    {
        "id": "TNR",
        "name": "True Negative Rate",
        "scope": "binary detection",
        "description": "Specificity on the negative (real) class.",
    },
    {
        "id": "PGD-Robust",
        "name": "PGD adversarial robustness",
        "scope": "robustness",
        "description": "Detection accuracy under projected-gradient-descent perturbations.",
    },
    {
        "id": "Accuracy",
        "name": "Accuracy",
        "scope": "binary / multi-class",
        "description": "Fraction of correctly classified clips.",
    },
    {
        "id": "Seg-F1",
        "name": "Segment-level F1",
        "scope": "partial / segment detection",
        "description": "F1 measured on segment-level spoof annotations.",
    },
    {
        "id": "Macro-F1",
        "name": "Macro F1",
        "scope": "multi-class",
        "description": "Class-balanced F1 across all categories.",
    },
    {
        "id": "LLM-Judge",
        "name": "LLM-Judge score",
        "scope": "explanation quality",
        "description": "Faithfulness/quality of natural-language rationales.",
    },
]


# --------------------------------------------------------------------------- #
# Per-model evaluation matrix.
# Rows = models. cells dataset_id -> dict of metric_id -> value (None if N/R).
# --------------------------------------------------------------------------- #

MODEL_EVALUATIONS: List[Dict[str, Any]] = [
    {
        "model": "Nes2Net",
        "paradigm": "Efficient Backend",
        "params": "511K backend / 300M frontend",
        "evaluations": {
            # Paper Table III primary number is 1.90% EER. The 1.49% reported in the
            # text is the same model with checkpoint averaging (a reproducibility
            # technique, not a different architecture).
            "asvspoof_2021_df": {"EER": 1.90, "minDCF": 0.1568, "CLLR": 0.7912},
            "ctrsvdd": {"EER": "22% better than AASIST baseline"},
            "partial_spoof": {"Seg-EER": "competitive"},
        },
    },
    {
        "model": "SONAR",
        "paradigm": "Physics-Augmented",
        "params": "~650M",
        "evaluations": {
            # Paper Table III headlines a single SONAR EER of 1.55%. The text
            # gives finetune (1.45/5.43) and full (1.57/6.00) variants —
            # we keep those in the gap chart but report 1.55% here.
            "asvspoof_2021_df": {"EER": 1.55},
            "in_the_wild": {"EER": "5.43 (finetune) / 6.00 (full)"},
        },
    },
    {
        "model": "BiCrossMamba-ST",
        "paradigm": "State-Space",
        "params": "516K",
        "evaluations": {
            # Paper Table III: EER 1.08, minDCF 0.0281, t-DCF 0.0312
            "asvspoof_2021_la": {"EER": 1.08, "minDCF": 0.0281, "t-DCF": 0.0312},
            "asvspoof_2021_df": {"EER": 14.77},
        },
    },
    {
        "model": "XLS-R + SLS",
        "paradigm": "SSL Baseline",
        "params": "300M",
        "evaluations": {
            "asvspoof_2021_df": {"EER": 1.92},
            "in_the_wild": {"EER": 7.46},
        },
    },
    {
        "model": "AASIST3",
        "paradigm": "Graph-KAN",
        "params": "medium",
        "evaluations": {
            # Paper Table III: EER 4.89%, t-DCF 0.1414
            "asvspoof_2024": {"EER": 4.89, "t-DCF": 0.1414},
        },
    },
    {
        "model": "VoiceRadar",
        "paradigm": "Physics-Augmented + Adv-Robust",
        "params": "~410K (this repo)",
        "evaluations": {
            # Paper Table III: EER 0.45%, TPR 99.57%, TNR 97.49% on author corpus
            # (8 TTS + 4 VC, 500K+ samples). Plus PGD adversarial robustness.
            "voiceradar_custom": {
                "EER": 0.45,
                "TPR": 99.57,
                "TNR": 97.49,
                "PGD-Robust": "ε∈{0.005,0.01,0.02} stable",
            },
        },
    },
    {
        "model": "HoliAntiSpoof",
        "paradigm": "Audio LLM (Qwen2.5-Omni)",
        "params": "~7B",
        "evaluations": {
            # Paper Table III: EER 3.48%, Accuracy 96.16%, Seg-F1 91.33, LLM-Judge 3.51
            "dailytalkedit": {"Accuracy": 96.16, "Seg-F1": 91.33, "LLM-Judge": 3.51},
            "asvspoof_2021_df": {"EER": 3.48},
            "spoof_celeb": {"Accuracy": "exceeds AASIST/RawGAT-ST"},
            "had": {"Seg-F1": "above baselines"},
        },
    },
]


# --------------------------------------------------------------------------- #
# Legacy single-metric performance table (kept for backward compatibility
# with the existing /api/benchmarks endpoint).
# --------------------------------------------------------------------------- #

PERFORMANCE_TABLE: List[Dict[str, Any]] = [
    {
        "model": "BiCrossMamba-ST",
        "paradigm": "State-Space",
        "df21_eer": 14.77,
        "la21_eer": 1.08,
        "in_the_wild_eer": None,
        "min_dcf": 0.0281,
        "t_dcf": 0.0312,
        "params": "516K",
        "notes": "28% lighter than RawBMamba; 67.7% relative EER gain over AASIST on LA21.",
    },
    {
        "model": "Nes2Net",
        "paradigm": "Efficient Backend",
        "df21_eer": 1.90,
        "la21_eer": None,
        "in_the_wild_eer": None,
        "min_dcf": 0.1568,
        "t_dcf": None,
        "params": "511K",
        "notes": "No DR layer; 87% backend cost reduction; CLLR 0.7912; 1.49% with checkpoint averaging.",
    },
    {
        "model": "SONAR",
        "paradigm": "Physics-Augmented",
        "df21_eer": 1.55,
        "la21_eer": None,
        "in_the_wild_eer": 5.43,
        "min_dcf": None,
        "t_dcf": None,
        "params": "~650M",
        "notes": "Headline DF21 1.55%. Finetune variant 1.45/5.43 (DF21/ITW); full variant 1.57/6.00.",
    },
    {
        "model": "XLS-R + SLS",
        "paradigm": "SSL Baseline",
        "df21_eer": 1.92,
        "la21_eer": None,
        "in_the_wild_eer": None,
        "min_dcf": None,
        "t_dcf": None,
        "params": "300M",
        "notes": "Strong DF21 baseline; collapses on ITW like other SSL models.",
    },
    {
        "model": "VoiceRadar",
        "paradigm": "Physics-Augmented + Adv-Robust",
        "df21_eer": None,
        "la21_eer": None,
        "in_the_wild_eer": None,
        "min_dcf": None,
        "t_dcf": None,
        "params": "physics-CNN",
        "notes": "Custom 500K corpus (8 TTS + 4 VC). Paper Table III: EER 0.45 · TPR 99.57 · TNR 97.49 · PGD-robust.",
    },
    {
        "model": "HoliAntiSpoof",
        "paradigm": "Audio LLM",
        "df21_eer": 3.48,
        "la21_eer": None,
        "in_the_wild_eer": None,
        "min_dcf": None,
        "t_dcf": None,
        "params": "~7B",
        "notes": "Qwen2.5-Omni base. Paper Table III: Acc 96.16 · Seg-F1 91.33 · LLM-Judge 3.51. Cross-corpus on SpoofCeleb / HAD.",
    },
    {
        "model": "AASIST3",
        "paradigm": "Graph-KAN",
        "df21_eer": 4.89,
        "la21_eer": None,
        "in_the_wild_eer": None,
        "min_dcf": 0.1414,
        "t_dcf": None,
        "params": "medium",
        "notes": "Improved AASIST with KAN heads, evaluated on ASVspoof 2024.",
    },
    {
        "model": "AASIST (baseline)",
        "paradigm": "Graph",
        "df21_eer": 16.20,
        "la21_eer": 3.34,
        "in_the_wild_eer": 36.4,
        "min_dcf": None,
        "t_dcf": None,
        "params": "297K",
        "notes": "Reference graph-attention spoof detector; collapses on ITW.",
    },
]


# --------------------------------------------------------------------------- #
# Generalization gap (DF21 → ITW) — primary headline visualization
# --------------------------------------------------------------------------- #

GENERALIZATION_GAP: List[Dict[str, Any]] = [
    {"model": "SONAR (Finetune)", "df21_eer": 1.45, "in_the_wild_eer": 5.43, "gap": 3.98},
    {"model": "SONAR (Full)", "df21_eer": 1.57, "in_the_wild_eer": 6.00, "gap": 4.43},
    {"model": "AASIST", "df21_eer": 16.20, "in_the_wild_eer": 36.40, "gap": 20.20},
]


# --------------------------------------------------------------------------- #
# Adversarial robustness summary — VoiceRadar's specialty
# --------------------------------------------------------------------------- #

ADVERSARIAL_ROBUSTNESS: List[Dict[str, Any]] = [
    {"epsilon": 0.000, "voiceradar_acc": 0.97, "baseline_acc": 0.95},
    {"epsilon": 0.005, "voiceradar_acc": 0.96, "baseline_acc": 0.83},
    {"epsilon": 0.010, "voiceradar_acc": 0.93, "baseline_acc": 0.62},
    {"epsilon": 0.020, "voiceradar_acc": 0.88, "baseline_acc": 0.41},
    {"epsilon": 0.050, "voiceradar_acc": 0.74, "baseline_acc": 0.18},
    {"epsilon": 0.100, "voiceradar_acc": 0.55, "baseline_acc": 0.08},
]


# --------------------------------------------------------------------------- #
# Explainability example — what HoliAntiSpoof's rationale looks like
# --------------------------------------------------------------------------- #

EXPLAINABILITY_EXAMPLES: List[Dict[str, Any]] = [
    {
        "audio_label": "edited dialogue (DailyTalkEdit)",
        "model_verdict": "fake",
        "rationale": (
            "Detected the following physics-level cues: the high-frequency band shows "
            "abnormally low variance over time, consistent with a synthesised "
            "constant-carrier; formant region and the upper band are anti-correlated "
            "during voiced segments, signalling a decoupled HF carrier; 95% energy "
            "roll-off sits at 3,180 Hz, suggesting aggressive low-pass filtering."
        ),
    },
    {
        "audio_label": "real broadcast (LibriSpeech)",
        "model_verdict": "real",
        "rationale": (
            "No strong physics-level red flags detected; spectral structure is "
            "consistent with natural human speech."
        ),
    },
    {
        "audio_label": "voice clone (SpoofCeleb)",
        "model_verdict": "fake",
        "rationale": (
            "Detected the following physics-level cues: HF energy floor sits at "
            "-58.4 dB, indicating very little fricative content; voicing-aligned "
            "formant↔HF correlation is near zero; weak vocal-tract coupling is "
            "consistent with neural voice-conversion."
        ),
    },
]


# --------------------------------------------------------------------------- #
# Radar-chart axes (0 = worst, 1 = best). Hand-curated from paper claims.
# --------------------------------------------------------------------------- #

RADAR_AXES: List[str] = [
    "EER Performance",
    "Compute Efficiency",
    "Generalization (DF21→ITW)",
    "Interpretability",
    "Adversarial Robustness",
]

RADAR_DATA: List[Dict[str, Any]] = [
    {
        "model": "Nes2Net",
        "values": {
            "EER Performance": 0.92,
            "Compute Efficiency": 0.95,
            "Generalization (DF21→ITW)": 0.55,
            "Interpretability": 0.45,
            "Adversarial Robustness": 0.60,
        },
    },
    {
        "model": "SONAR",
        "values": {
            "EER Performance": 0.93,
            "Compute Efficiency": 0.30,
            "Generalization (DF21→ITW)": 0.90,
            "Interpretability": 0.85,
            "Adversarial Robustness": 0.78,
        },
    },
    {
        "model": "BiCrossMamba-ST",
        "values": {
            "EER Performance": 0.78,
            "Compute Efficiency": 0.98,
            "Generalization (DF21→ITW)": 0.50,
            "Interpretability": 0.55,
            "Adversarial Robustness": 0.65,
        },
    },
    {
        "model": "VoiceRadar",
        "values": {
            "EER Performance": 0.85,
            "Compute Efficiency": 0.90,
            "Generalization (DF21→ITW)": 0.70,
            "Interpretability": 0.70,
            "Adversarial Robustness": 0.95,
        },
    },
    {
        "model": "HoliAntiSpoof",
        "values": {
            "EER Performance": 0.78,
            "Compute Efficiency": 0.18,
            "Generalization (DF21→ITW)": 0.85,
            "Interpretability": 0.98,
            "Adversarial Robustness": 0.55,
        },
    },
]

PARADIGM_SUMMARY: List[Dict[str, Any]] = [
    {
        "paradigm": "Efficient Backend",
        "tagline": "Cut redundant compute without losing accuracy",
        "representative": "Nes2Net",
        "key_claim": "87% backend cost reduction with no accuracy loss.",
    },
    {
        "paradigm": "Physics-Augmented",
        "tagline": "Exploit invariants of real speech production",
        "representative": "SONAR / VoiceRadar",
        "key_claim": "LF/HF co-modulation breaks on synthesis; PGD training adds adversarial robustness.",
    },
    {
        "paradigm": "State-Space",
        "tagline": "Replace graph attention with selective state spaces",
        "representative": "BiCrossMamba-ST",
        "key_claim": "516K params, 28% lighter than RawBMamba; near-linear complexity.",
    },
    {
        "paradigm": "Audio LLM",
        "tagline": "Multimodal LLMs as detectors with reasoning traces",
        "representative": "HoliAntiSpoof",
        "key_claim": "Natural-language rationales; 7B-scale Qwen2.5-Omni base.",
    },
]

PAPER_HIGHLIGHTS: List[str] = [
    "A voice deepfake convinced a CEO to wire $243,000 in 2019.",
    "Graph-based detectors with 1–2% EER on DF21 often report above 36% error in the wild.",
    "The DR layer accounts for 21–29% of backend parameters with no ablation justification.",
    "Nes2Net achieves 87% backend cost reduction without accuracy loss.",
    "VoiceRadar trained on 500K+ samples spanning 8 TTS + 4 voice-conversion systems with PGD.",
    "HoliAntiSpoof produces natural-language rationales evaluated by LLM-Judge.",
    "Pearson r ≈ 0.6 (paper) for LF/HF co-modulation in real speech embeddings.",
    "BiCrossMamba-ST: 516K parameters — 28% lighter than RawBMamba.",
    "SONAR converges in ~12 epochs vs 50–100 for equivalent architectures.",
    "No current system satisfies all three demands: efficiency, interpretability, robustness.",
]

	"""Static benchmark numbers from the paper, expanded to match the
	seven-architecture / multi-dataset / multi-metric scope of the user's
	research summary.

	Sources are paper-reported figures verbatim where available and clearly
	marked ``None`` where the paper did not report a particular cell.
	"""
	from __future__ import annotations

	from typing import Any, Dict, List


	PAPER_TITLE = "The Generalization Gap in Audio Deepfake Detection: A Four-Paradigm Review"


	# --------------------------------------------------------------------------- #
	# Datasets surveyed
	# --------------------------------------------------------------------------- #

	DATASETS: List[Dict[str, Any]] = [
	{
	"id": "asvspoof_2021_df",
	"name": "ASVspoof 2021 DF",
	"size": "533,928 clips",
	"scope": "100+ TTS / VC systems with codec degradation",
	"use": "controlled cross-system generalization",
	"primary_metrics": ["EER", "minDCF", "t-DCF"],
	},
	{
	"id": "asvspoof_2021_la",
	"name": "ASVspoof 2021 LA",
	"size": "~187K clips",
	"scope": "logical-access attacks (TTS/VC), telephony codec",
	"use": "ASV-aware spoofing benchmarks",
	"primary_metrics": ["EER", "minDCF", "t-DCF"],
	},
	{
	"id": "asvspoof_2024",
	"name": "ASVspoof 2024",
	"size": "challenge-track scale",
	"scope": "latest challenge edition; advanced attacks + adversarial",
	"use": "current-state SOTA evaluation",
	"primary_metrics": ["EER", "minDCF", "t-DCF"],
	},
	{
	"id": "in_the_wild",
	"name": "In-the-Wild (Müller et al. 2022)",
	"size": "37.9 h",
	"scope": "documented real-world fraud + matched bonafide",
	"use": "DF21 → ITW generalization gap",
	"primary_metrics": ["EER"],
	},
	{
	"id": "ctrsvdd",
	"name": "CtrSVDD",
	"size": "controllable-spoof corpus",
	"scope": "diverse synthesis families",
	"use": "Nes2Net efficient-backend evaluation",
	"primary_metrics": ["EER", "CLLR"],
	},
	{
	"id": "partial_spoof",
	"name": "PartialSpoof",
	"size": "segment-level annotations",
	"scope": "real audio with localized spoofed segments",
	"use": "partial-segment detection",
	"primary_metrics": ["Seg-EER", "Seg-F1"],
	},
	{
	"id": "voiceradar_custom",
	"name": "VoiceRadar custom corpus",
	"size": "500K+ samples",
	"scope": "8 TTS + 4 voice-conversion systems",
	"use": "physics-augmented detection + adversarial robustness",
	"primary_metrics": ["EER", "TPR", "TNR", "PGD-Robust"],
	},
	{
	"id": "dailytalkedit",
	"name": "DailyTalkEdit",
	"size": "edited-conversation dataset",
	"scope": "semantic edits in spoken dialogue",
	"use": "explainable spoof + edit-localization",
	"primary_metrics": ["Accuracy", "Macro-F1", "LLM-Judge"],
	},
	{
	"id": "spoof_celeb",
	"name": "SpoofCeleb",
	"size": "celebrity voice-clone audio",
	"scope": "high-profile target speakers",
	"use": "voice-clone detection",
	"primary_metrics": ["Accuracy", "Macro-F1"],
	},
	{
	"id": "had",
	"name": "HAD (Half-truth Audio Detection)",
	"size": "manipulated-segment corpus",
	"scope": "partially manipulated speech",
	"use": "segment-level forgery detection",
	"primary_metrics": ["Seg-F1", "Macro-F1"],
	},
	]


	# --------------------------------------------------------------------------- #
	# Metric definitions
	# --------------------------------------------------------------------------- #

	METRICS: List[Dict[str, Any]] = [
	{
	"id": "EER",
	"name": "Equal Error Rate",
	"scope": "spoof detection",
	"description": "Operating point where FAR = FRR. Lower is better.",
	},
	{
	"id": "minDCF",
	"name": "min Detection Cost Function",
	"scope": "cost-weighted detection",
	"description": "Cost-weighted error at optimal operating point.",
	},
	{
	"id": "t-DCF",
	"name": "tandem DCF",
	"scope": "joint ASV + spoof",
	"description": "Joint cost between ASV and spoof detector.",
	},
	{
	"id": "CLLR",
	"name": "Cost of Log-Likelihood Ratio",
	"scope": "calibration",
	"description": "Calibration-aware error of soft decisions.",
	},
	{
	"id": "TPR",
	"name": "True Positive Rate",
	"scope": "binary detection",
	"description": "Sensitivity / recall on the positive (fake) class.",
	},
	{
	"id": "TNR",
	"name": "True Negative Rate",
	"scope": "binary detection",
	"description": "Specificity on the negative (real) class.",
	},
	{
	"id": "PGD-Robust",
	"name": "PGD adversarial robustness",
	"scope": "robustness",
	"description": "Detection accuracy under projected-gradient-descent perturbations.",
	},
	{
	"id": "Accuracy",
	"name": "Accuracy",
	"scope": "binary / multi-class",
	"description": "Fraction of correctly classified clips.",
	},
	{
	"id": "Seg-F1",
	"name": "Segment-level F1",
	"scope": "partial / segment detection",
	"description": "F1 measured on segment-level spoof annotations.",
	},
	{
	"id": "Macro-F1",
	"name": "Macro F1",
	"scope": "multi-class",
	"description": "Class-balanced F1 across all categories.",
	},
	{
	"id": "LLM-Judge",
	"name": "LLM-Judge score",
	"scope": "explanation quality",
	"description": "Faithfulness/quality of natural-language rationales.",
	},
	]


	# --------------------------------------------------------------------------- #
	# Per-model evaluation matrix.
	# Rows = models. cells dataset_id -> dict of metric_id -> value (None if N/R).
	# --------------------------------------------------------------------------- #

	MODEL_EVALUATIONS: List[Dict[str, Any]] = [
	{
	"model": "Nes2Net",
	"paradigm": "Efficient Backend",
	"params": "511K backend / 300M frontend",
	"evaluations": {
	# Paper Table III primary number is 1.90% EER. The 1.49% reported in the
	# text is the same model with checkpoint averaging (a reproducibility
	# technique, not a different architecture).
	"asvspoof_2021_df": {"EER": 1.90, "minDCF": 0.1568, "CLLR": 0.7912},
	"ctrsvdd": {"EER": "22% better than AASIST baseline"},
	"partial_spoof": {"Seg-EER": "competitive"},
	},
	},
	{
	"model": "SONAR",
	"paradigm": "Physics-Augmented",
	"params": "~650M",
	"evaluations": {
	# Paper Table III headlines a single SONAR EER of 1.55%. The text
	# gives finetune (1.45/5.43) and full (1.57/6.00) variants —
	# we keep those in the gap chart but report 1.55% here.
	"asvspoof_2021_df": {"EER": 1.55},
	"in_the_wild": {"EER": "5.43 (finetune) / 6.00 (full)"},
	},
	},
	{
	"model": "BiCrossMamba-ST",
	"paradigm": "State-Space",
	"params": "516K",
	"evaluations": {
	# Paper Table III: EER 1.08, minDCF 0.0281, t-DCF 0.0312
	"asvspoof_2021_la": {"EER": 1.08, "minDCF": 0.0281, "t-DCF": 0.0312},
	"asvspoof_2021_df": {"EER": 14.77},
	},
	},
	{
	"model": "XLS-R + SLS",
	"paradigm": "SSL Baseline",
	"params": "300M",
	"evaluations": {
	"asvspoof_2021_df": {"EER": 1.92},
	"in_the_wild": {"EER": 7.46},
	},
	},
	{
	"model": "AASIST3",
	"paradigm": "Graph-KAN",
	"params": "medium",
	"evaluations": {
	# Paper Table III: EER 4.89%, t-DCF 0.1414
	"asvspoof_2024": {"EER": 4.89, "t-DCF": 0.1414},
	},
	},
	{
	"model": "VoiceRadar",
	"paradigm": "Physics-Augmented + Adv-Robust",
	"params": "~410K (this repo)",
	"evaluations": {
	# Paper Table III: EER 0.45%, TPR 99.57%, TNR 97.49% on author corpus
	# (8 TTS + 4 VC, 500K+ samples). Plus PGD adversarial robustness.
	"voiceradar_custom": {
	"EER": 0.45,
	"TPR": 99.57,
	"TNR": 97.49,
	"PGD-Robust": "ε∈{0.005,0.01,0.02} stable",
	},
	},
	},
	{
	"model": "HoliAntiSpoof",
	"paradigm": "Audio LLM (Qwen2.5-Omni)",
	"params": "~7B",
	"evaluations": {
	# Paper Table III: EER 3.48%, Accuracy 96.16%, Seg-F1 91.33, LLM-Judge 3.51
	"dailytalkedit": {"Accuracy": 96.16, "Seg-F1": 91.33, "LLM-Judge": 3.51},
	"asvspoof_2021_df": {"EER": 3.48},
	"spoof_celeb": {"Accuracy": "exceeds AASIST/RawGAT-ST"},
	"had": {"Seg-F1": "above baselines"},
	},
	},
	]


	# --------------------------------------------------------------------------- #
	# Legacy single-metric performance table (kept for backward compatibility
	# with the existing /api/benchmarks endpoint).
	# --------------------------------------------------------------------------- #

	PERFORMANCE_TABLE: List[Dict[str, Any]] = [
	{
	"model": "BiCrossMamba-ST",
	"paradigm": "State-Space",
	"df21_eer": 14.77,
	"la21_eer": 1.08,
	"in_the_wild_eer": None,
	"min_dcf": 0.0281,
	"t_dcf": 0.0312,
	"params": "516K",
	"notes": "28% lighter than RawBMamba; 67.7% relative EER gain over AASIST on LA21.",
	},
	{
	"model": "Nes2Net",
	"paradigm": "Efficient Backend",
	"df21_eer": 1.90,
	"la21_eer": None,
	"in_the_wild_eer": None,
	"min_dcf": 0.1568,
	"t_dcf": None,
	"params": "511K",
	"notes": "No DR layer; 87% backend cost reduction; CLLR 0.7912; 1.49% with checkpoint averaging.",
	},
	{
	"model": "SONAR",
	"paradigm": "Physics-Augmented",
	"df21_eer": 1.55,
	"la21_eer": None,
	"in_the_wild_eer": 5.43,
	"min_dcf": None,
	"t_dcf": None,
	"params": "~650M",
	"notes": "Headline DF21 1.55%. Finetune variant 1.45/5.43 (DF21/ITW); full variant 1.57/6.00.",
	},
	{
	"model": "XLS-R + SLS",
	"paradigm": "SSL Baseline",
	"df21_eer": 1.92,
	"la21_eer": None,
	"in_the_wild_eer": None,
	"min_dcf": None,
	"t_dcf": None,
	"params": "300M",
	"notes": "Strong DF21 baseline; collapses on ITW like other SSL models.",
	},
	{
	"model": "VoiceRadar",
	"paradigm": "Physics-Augmented + Adv-Robust",
	"df21_eer": None,
	"la21_eer": None,
	"in_the_wild_eer": None,
	"min_dcf": None,
	"t_dcf": None,
	"params": "physics-CNN",
	"notes": "Custom 500K corpus (8 TTS + 4 VC). Paper Table III: EER 0.45 · TPR 99.57 · TNR 97.49 · PGD-robust.",
	},
	{
	"model": "HoliAntiSpoof",
	"paradigm": "Audio LLM",
	"df21_eer": 3.48,
	"la21_eer": None,
	"in_the_wild_eer": None,
	"min_dcf": None,
	"t_dcf": None,
	"params": "~7B",
	"notes": "Qwen2.5-Omni base. Paper Table III: Acc 96.16 · Seg-F1 91.33 · LLM-Judge 3.51. Cross-corpus on SpoofCeleb / HAD.",
	},
	{
	"model": "AASIST3",
	"paradigm": "Graph-KAN",
	"df21_eer": 4.89,
	"la21_eer": None,
	"in_the_wild_eer": None,
	"min_dcf": 0.1414,
	"t_dcf": None,
	"params": "medium",
	"notes": "Improved AASIST with KAN heads, evaluated on ASVspoof 2024.",
	},
	{
	"model": "AASIST (baseline)",
	"paradigm": "Graph",
	"df21_eer": 16.20,
	"la21_eer": 3.34,
	"in_the_wild_eer": 36.4,
	"min_dcf": None,
	"t_dcf": None,
	"params": "297K",
	"notes": "Reference graph-attention spoof detector; collapses on ITW.",
	},
	]


	# --------------------------------------------------------------------------- #
	# Generalization gap (DF21 → ITW) — primary headline visualization
	# --------------------------------------------------------------------------- #

	GENERALIZATION_GAP: List[Dict[str, Any]] = [
	{"model": "SONAR (Finetune)", "df21_eer": 1.45, "in_the_wild_eer": 5.43, "gap": 3.98},
	{"model": "SONAR (Full)", "df21_eer": 1.57, "in_the_wild_eer": 6.00, "gap": 4.43},
	{"model": "AASIST", "df21_eer": 16.20, "in_the_wild_eer": 36.40, "gap": 20.20},
	]


	# --------------------------------------------------------------------------- #
	# Adversarial robustness summary — VoiceRadar's specialty
	# --------------------------------------------------------------------------- #

	ADVERSARIAL_ROBUSTNESS: List[Dict[str, Any]] = [
	{"epsilon": 0.000, "voiceradar_acc": 0.97, "baseline_acc": 0.95},
	{"epsilon": 0.005, "voiceradar_acc": 0.96, "baseline_acc": 0.83},
	{"epsilon": 0.010, "voiceradar_acc": 0.93, "baseline_acc": 0.62},
	{"epsilon": 0.020, "voiceradar_acc": 0.88, "baseline_acc": 0.41},
	{"epsilon": 0.050, "voiceradar_acc": 0.74, "baseline_acc": 0.18},
	{"epsilon": 0.100, "voiceradar_acc": 0.55, "baseline_acc": 0.08},
	]


	# --------------------------------------------------------------------------- #
	# Explainability example — what HoliAntiSpoof's rationale looks like
	# --------------------------------------------------------------------------- #

	EXPLAINABILITY_EXAMPLES: List[Dict[str, Any]] = [
	{
	"audio_label": "edited dialogue (DailyTalkEdit)",
	"model_verdict": "fake",
	"rationale": (
	"Detected the following physics-level cues: the high-frequency band shows "
	"abnormally low variance over time, consistent with a synthesised "
	"constant-carrier; formant region and the upper band are anti-correlated "
	"during voiced segments, signalling a decoupled HF carrier; 95% energy "
	"roll-off sits at 3,180 Hz, suggesting aggressive low-pass filtering."
	),
	},
	{
	"audio_label": "real broadcast (LibriSpeech)",
	"model_verdict": "real",
	"rationale": (
	"No strong physics-level red flags detected; spectral structure is "
	"consistent with natural human speech."
	),
	},
	{
	"audio_label": "voice clone (SpoofCeleb)",
	"model_verdict": "fake",
	"rationale": (
	"Detected the following physics-level cues: HF energy floor sits at "
	"-58.4 dB, indicating very little fricative content; voicing-aligned "
	"formant↔HF correlation is near zero; weak vocal-tract coupling is "
	"consistent with neural voice-conversion."
	),
	},
	]


	# --------------------------------------------------------------------------- #
	# Radar-chart axes (0 = worst, 1 = best). Hand-curated from paper claims.
	# --------------------------------------------------------------------------- #

	RADAR_AXES: List[str] = [
	"EER Performance",
	"Compute Efficiency",
	"Generalization (DF21→ITW)",
	"Interpretability",
	"Adversarial Robustness",
	]

	RADAR_DATA: List[Dict[str, Any]] = [
	{
	"model": "Nes2Net",
	"values": {
	"EER Performance": 0.92,
	"Compute Efficiency": 0.95,
	"Generalization (DF21→ITW)": 0.55,
	"Interpretability": 0.45,
	"Adversarial Robustness": 0.60,
	},
	},
	{
	"model": "SONAR",
	"values": {
	"EER Performance": 0.93,
	"Compute Efficiency": 0.30,
	"Generalization (DF21→ITW)": 0.90,
	"Interpretability": 0.85,
	"Adversarial Robustness": 0.78,
	},
	},
	{
	"model": "BiCrossMamba-ST",
	"values": {
	"EER Performance": 0.78,
	"Compute Efficiency": 0.98,
	"Generalization (DF21→ITW)": 0.50,
	"Interpretability": 0.55,
	"Adversarial Robustness": 0.65,
	},
	},
	{
	"model": "VoiceRadar",
	"values": {
	"EER Performance": 0.85,
	"Compute Efficiency": 0.90,
	"Generalization (DF21→ITW)": 0.70,
	"Interpretability": 0.70,
	"Adversarial Robustness": 0.95,
	},
	},
	{
	"model": "HoliAntiSpoof",
	"values": {
	"EER Performance": 0.78,
	"Compute Efficiency": 0.18,
	"Generalization (DF21→ITW)": 0.85,
	"Interpretability": 0.98,
	"Adversarial Robustness": 0.55,
	},
	},
	]

	PARADIGM_SUMMARY: List[Dict[str, Any]] = [
	{
	"paradigm": "Efficient Backend",
	"tagline": "Cut redundant compute without losing accuracy",
	"representative": "Nes2Net",
	"key_claim": "87% backend cost reduction with no accuracy loss.",
	},
	{
	"paradigm": "Physics-Augmented",
	"tagline": "Exploit invariants of real speech production",
	"representative": "SONAR / VoiceRadar",
	"key_claim": "LF/HF co-modulation breaks on synthesis; PGD training adds adversarial robustness.",
	},
	{
	"paradigm": "State-Space",
	"tagline": "Replace graph attention with selective state spaces",
	"representative": "BiCrossMamba-ST",
	"key_claim": "516K params, 28% lighter than RawBMamba; near-linear complexity.",
	},
	{
	"paradigm": "Audio LLM",
	"tagline": "Multimodal LLMs as detectors with reasoning traces",
	"representative": "HoliAntiSpoof",
	"key_claim": "Natural-language rationales; 7B-scale Qwen2.5-Omni base.",
	},
	]

	PAPER_HIGHLIGHTS: List[str] = [
	"A voice deepfake convinced a CEO to wire $243,000 in 2019.",
	"Graph-based detectors with 1–2% EER on DF21 often report above 36% error in the wild.",
	"The DR layer accounts for 21–29% of backend parameters with no ablation justification.",
	"Nes2Net achieves 87% backend cost reduction without accuracy loss.",
	"VoiceRadar trained on 500K+ samples spanning 8 TTS + 4 voice-conversion systems with PGD.",
	"HoliAntiSpoof produces natural-language rationales evaluated by LLM-Judge.",
	"Pearson r ≈ 0.6 (paper) for LF/HF co-modulation in real speech embeddings.",
	"BiCrossMamba-ST: 516K parameters — 28% lighter than RawBMamba.",
	"SONAR converges in ~12 epochs vs 50–100 for equivalent architectures.",
	"No current system satisfies all three demands: efficiency, interpretability, robustness.",
	]