Multimodel_setup / scripts /verify_v2_reference_parity.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
15 kB
"""
Phase 7 V2 Reference Parity Verification Script
Verifies numerical agreement between official upstream references and corrected adapters
across deterministic 30 samples (10 Real, 10 Midjourney, 10 ADM) and ViT regression parity.
Checks:
- Decoded RGB pixels
- Input tensors
- Raw logits
- Fake probabilities
- Final binary predictions
Requires numerical agreement within floating-point tolerance (< 1e-5).
"""
import os
import sys
import csv
import json
import time
import cv2
import numpy as np
import torch
from torchvision import transforms
from PIL import Image
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv")
OUT_JSON = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase7_v2_reference_preprocessing", "parity_verification_audit.json")
# 1. Deterministic samples
def get_deterministic_samples():
with open(MANIFEST_PATH, "r", encoding="utf-8") as f:
rows = list(csv.DictReader(f))
reals = [r for r in rows if r["label"] == "REAL"][:10]
mjs = [r for r in rows if r["generator_family"] == "midjourney"][:10]
adms = [r for r in rows if r["generator_family"] == "adm"][:10]
return reals + mjs + adms
def verify_convnext_parity():
print("\n" + "=" * 70)
print("VERIFYING CONVNEXT DRCT REFERENCE PARITY (30 SAMPLES)")
print("=" * 70)
# Import reference DRCT
sys.path.insert(0, os.path.join(PROJECT_ROOT, "UniversalFakeDetect", "models"))
sys.path.insert(0, os.path.join(PROJECT_ROOT, "temp_DRCT_reference"))
from network.models import get_models
from scripts.parity_audit.audit_convnext_parity import drct_val_transform_is_crop
from src.convnext_adapter import ConvNeXtDetectorAdapter
samples = get_deterministic_samples()
ckpt_path = os.path.join(PROJECT_ROOT, "models", "chen2024_convnext", "16_acc0.9993.pth")
# Load official model
print("Loading Official DRCT model...")
official_net = get_models(model_name="convnext_base_in22k", num_classes=2, pretrained=False, embedding_size=1024)
ckpt = torch.load(ckpt_path, map_location="cpu")
official_net.load_state_dict(ckpt, strict=True)
official_net.eval()
# Load adapter
print("Loading ConvNeXtDetectorAdapter...")
adapter = ConvNeXtDetectorAdapter(checkpoint_path=ckpt_path, device="cpu", lazy_load=False)
max_pixel_diff = 0
max_tensor_diff = 0.0
max_logit_diff = 0.0
max_prob_diff = 0.0
pred_matches = 0
records = []
for idx, sample in enumerate(samples, 1):
sample_id = sample["sample_id"]
rel_path = sample["relative_path"]
abs_path = os.path.join(PROJECT_ROOT, rel_path)
gt = sample["label"]
gen = sample["generator_family"]
# 1. Decoded RGB comparison
img_bgr = cv2.imread(abs_path)
img_rgb_cv2 = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
with Image.open(abs_path) as opened:
img_rgb_pil = np.array(opened.convert("RGB"))
pixel_diff = int(np.max(np.abs(img_rgb_cv2.astype(np.int32) - img_rgb_pil.astype(np.int32))))
max_pixel_diff = max(max_pixel_diff, pixel_diff)
# 2. Input tensor comparison
t_official = drct_val_transform_is_crop(img_rgb_cv2, size=224)
t_adapter = adapter.transform(img_rgb_pil).unsqueeze(0)
tensor_diff = float(torch.max(torch.abs(t_official - t_adapter)).item())
max_tensor_diff = max(max_tensor_diff, tensor_diff)
# 3. Model outputs
with torch.no_grad():
out_official = official_net(t_official)
prob_official = torch.softmax(out_official, dim=1).squeeze(0)
fake_p_official = float(prob_official[1].item())
logit_official = float((out_official[0, 1] - out_official[0, 0]).item())
pred_official = "FAKE" if fake_p_official >= 0.50 else "REAL"
res_adapter = adapter.predict(abs_path)
fake_p_adapter = res_adapter.fake_probability
logit_adapter = res_adapter.raw_logit
pred_adapter = res_adapter.prediction
logit_diff = abs(logit_official - logit_adapter)
prob_diff = abs(fake_p_official - fake_p_adapter)
max_logit_diff = max(max_logit_diff, logit_diff)
max_prob_diff = max(max_prob_diff, prob_diff)
match = (pred_official == pred_adapter)
if match:
pred_matches += 1
records.append({
"sample_id": sample_id,
"generator": gen,
"gt": gt,
"pixel_diff": pixel_diff,
"tensor_diff": round(tensor_diff, 8),
"official_logit": round(logit_official, 6),
"adapter_logit": round(logit_adapter, 6),
"logit_diff": round(logit_diff, 8),
"official_prob": round(fake_p_official, 6),
"adapter_prob": round(fake_p_adapter, 6),
"prob_diff": round(prob_diff, 8),
"official_pred": pred_official,
"adapter_pred": pred_adapter,
"match": match,
})
print(f" [{idx:02d}/30] {sample_id} ({gen:10s}): TensorDiff={tensor_diff:.8f}, ProbDiff={prob_diff:.8f}, "
f"OffProb={fake_p_official:.4f}, AdProb={fake_p_adapter:.4f} [{'PASS' if match else 'FAIL'}]")
print("\nConvNeXt DRCT Parity Results:")
print(f" Max Pixel Diff (cv2 vs PIL): {max_pixel_diff}")
print(f" Max Input Tensor Diff: {max_tensor_diff:.10f}")
print(f" Max Raw Logit Diff: {max_logit_diff:.10f}")
print(f" Max Fake Probability Diff: {max_prob_diff:.10f}")
print(f" Binary Prediction Matches: {pred_matches} / 30 ({pred_matches / 30 * 100:.1f}%)")
assert max_tensor_diff < 1e-6, f"ConvNeXt tensor diff too high: {max_tensor_diff}"
assert max_logit_diff < 1e-4, f"ConvNeXt logit diff too high: {max_logit_diff}"
assert max_prob_diff < 1e-4, f"ConvNeXt prob diff too high: {max_prob_diff}"
assert pred_matches == 30, f"ConvNeXt prediction mismatch: {pred_matches}/30"
del official_net
adapter.unload_model()
del adapter
return {
"model": "ConvNeXt (DRCT / RAID)",
"status": "PASSED",
"max_pixel_diff": max_pixel_diff,
"max_tensor_diff": max_tensor_diff,
"max_logit_diff": max_logit_diff,
"max_prob_diff": max_prob_diff,
"pred_matches": pred_matches,
"total_samples": 30,
"records": records,
}
def verify_univfd_parity():
print("\n" + "=" * 70)
print("VERIFYING UNIVFD REFERENCE PARITY (30 SAMPLES)")
print("=" * 70)
ufd_dir = os.path.join(PROJECT_ROOT, "UniversalFakeDetect")
sys.path.insert(0, ufd_dir)
sys.path.insert(0, os.path.join(ufd_dir, "models"))
from models import get_model
from src.univfd_adapter import UnivFDAdapter
samples = get_deterministic_samples()
ckpt_path = os.path.join(ufd_dir, "pretrained_weights", "fc_weights.pth")
print("Loading Official UnivFD model...")
official_model = get_model("CLIP:ViT-L/14")
official_model.fc.load_state_dict(torch.load(ckpt_path, map_location="cpu"))
official_model.eval()
clip_mean = [0.48145466, 0.4578275, 0.40821073]
clip_std = [0.26862954, 0.26130258, 0.27577711]
transform_official = transforms.Compose([
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=clip_mean, std=clip_std),
])
print("Loading UnivFDAdapter...")
adapter = UnivFDAdapter(device="cpu", lazy_load=False)
max_tensor_diff = 0.0
max_logit_diff = 0.0
max_prob_diff = 0.0
pred_matches = 0
records = []
for idx, sample in enumerate(samples, 1):
sample_id = sample["sample_id"]
rel_path = sample["relative_path"]
abs_path = os.path.join(PROJECT_ROOT, rel_path)
gt = sample["label"]
gen = sample["generator_family"]
with Image.open(abs_path) as opened:
img_rgb_pil = opened.convert("RGB")
# 1. Input tensor comparison
t_official = transform_official(img_rgb_pil).unsqueeze(0)
t_adapter = adapter.preprocess(img_rgb_pil).unsqueeze(0)
tensor_diff = float(torch.max(torch.abs(t_official - t_adapter)).item())
max_tensor_diff = max(max_tensor_diff, tensor_diff)
# 2. Model outputs
with torch.no_grad():
feat_official = official_model.model.encode_image(t_official)
logit_official = float(official_model.fc(feat_official.float()).item())
prob_official = float(torch.sigmoid(torch.tensor(logit_official)).item())
pred_official = "FAKE" if prob_official >= 0.50 else "REAL"
res_adapter = adapter.predict(abs_path)
fake_p_adapter = res_adapter.fake_probability
logit_adapter = res_adapter.raw_logit
pred_adapter = res_adapter.prediction
logit_diff = abs(logit_official - logit_adapter)
prob_diff = abs(prob_official - fake_p_adapter)
max_logit_diff = max(max_logit_diff, logit_diff)
max_prob_diff = max(max_prob_diff, prob_diff)
match = (pred_official == pred_adapter)
if match:
pred_matches += 1
records.append({
"sample_id": sample_id,
"generator": gen,
"gt": gt,
"tensor_diff": round(tensor_diff, 8),
"official_logit": round(logit_official, 6),
"adapter_logit": round(logit_adapter, 6),
"logit_diff": round(logit_diff, 8),
"official_prob": round(prob_official, 6),
"adapter_prob": round(fake_p_adapter, 6),
"prob_diff": round(prob_diff, 8),
"official_pred": pred_official,
"adapter_pred": pred_adapter,
"match": match,
})
print(f" [{idx:02d}/30] {sample_id} ({gen:10s}): TensorDiff={tensor_diff:.8f}, ProbDiff={prob_diff:.8f}, "
f"OffProb={prob_official:.4f}, AdProb={fake_p_adapter:.4f} [{'PASS' if match else 'FAIL'}]")
print("\nUnivFD Parity Results:")
print(f" Max Input Tensor Diff: {max_tensor_diff:.10f}")
print(f" Max Raw Logit Diff: {max_logit_diff:.10f}")
print(f" Max Fake Probability Diff: {max_prob_diff:.10f}")
print(f" Binary Prediction Matches: {pred_matches} / 30 ({pred_matches / 30 * 100:.1f}%)")
assert max_tensor_diff < 1e-6, f"UnivFD tensor diff too high: {max_tensor_diff}"
assert max_logit_diff < 1e-4, f"UnivFD logit diff too high: {max_logit_diff}"
assert max_prob_diff < 1e-4, f"UnivFD prob diff too high: {max_prob_diff}"
assert pred_matches == 30, f"UnivFD prediction mismatch: {pred_matches}/30"
del official_model
adapter.unload_model()
del adapter
return {
"model": "UnivFD (CLIP ViT-L/14)",
"status": "PASSED",
"max_tensor_diff": max_tensor_diff,
"max_logit_diff": max_logit_diff,
"max_prob_diff": max_prob_diff,
"pred_matches": pred_matches,
"total_samples": 30,
"records": records,
}
def verify_vit_regression_parity():
print("\n" + "=" * 70)
print("VERIFYING VIT REGRESSION PARITY (10 SAMPLES)")
print("=" * 70)
from transformers import AutoModelForImageClassification, AutoImageProcessor
from src.vit_adapter import ViTAdapter
samples = get_deterministic_samples()[:10] # 10 samples
model_dir = os.path.join(PROJECT_ROOT, "models", "deepfake_vs_real_image_detection")
print("Loading HuggingFace ViT model & processor...")
processor = AutoImageProcessor.from_pretrained(model_dir)
hf_model = AutoModelForImageClassification.from_pretrained(model_dir)
hf_model.eval()
print("Loading ViTAdapter...")
adapter = ViTAdapter(device="cpu", lazy_load=False)
max_tensor_diff = 0.0
max_prob_diff = 0.0
pred_matches = 0
records = []
for idx, sample in enumerate(samples, 1):
sample_id = sample["sample_id"]
rel_path = sample["relative_path"]
abs_path = os.path.join(PROJECT_ROOT, rel_path)
gt = sample["label"]
with Image.open(abs_path) as opened:
img_rgb_pil = opened.convert("RGB")
inputs = processor(images=img_rgb_pil, return_tensors="pt")
with torch.no_grad():
outputs = hf_model(**inputs)
logits = outputs.logits
probs = torch.softmax(logits, dim=1).squeeze(0)
fake_prob_hf = float(probs[1].item())
pred_hf = "FAKE" if fake_prob_hf >= 0.50 else "REAL"
res_adapter = adapter.predict(abs_path)
fake_prob_adapter = res_adapter.fake_probability
pred_adapter = res_adapter.prediction
prob_diff = abs(fake_prob_hf - fake_prob_adapter)
max_prob_diff = max(max_prob_diff, prob_diff)
match = (pred_hf == pred_adapter)
if match:
pred_matches += 1
records.append({
"sample_id": sample_id,
"gt": gt,
"hf_prob": round(fake_prob_hf, 6),
"adapter_prob": round(fake_prob_adapter, 6),
"prob_diff": round(prob_diff, 8),
"hf_pred": pred_hf,
"adapter_pred": pred_adapter,
"match": match,
})
print(f" [{idx:02d}/10] {sample_id}: ProbDiff={prob_diff:.8f}, HF={fake_prob_hf:.4f}, Adapter={fake_prob_adapter:.4f} [{'PASS' if match else 'FAIL'}]")
print("\nViT Regression Parity Results:")
print(f" Max Fake Probability Diff: {max_prob_diff:.10f}")
print(f" Binary Prediction Matches: {pred_matches} / 10 ({pred_matches / 10 * 100:.1f}%)")
assert max_prob_diff < 1e-4, f"ViT prob diff too high: {max_prob_diff}"
assert pred_matches == 10, f"ViT prediction mismatch: {pred_matches}/10"
del hf_model
adapter.unload_model()
del adapter
return {
"model": "ViT (dima806)",
"status": "PASSED",
"max_prob_diff": max_prob_diff,
"pred_matches": pred_matches,
"total_samples": 10,
"records": records,
}
def main():
t_start = time.time()
convnext_audit = verify_convnext_parity()
univfd_audit = verify_univfd_parity()
vit_audit = verify_vit_regression_parity()
audit_summary = {
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
"overall_status": "ALL_PARITY_CHECKS_PASSED",
"duration_sec": round(time.time() - t_start, 2),
"convnext": convnext_audit,
"univfd": univfd_audit,
"vit": vit_audit,
}
os.makedirs(os.path.dirname(OUT_JSON), exist_ok=True)
with open(OUT_JSON, "w", encoding="utf-8") as f:
json.dump(audit_summary, f, indent=2)
print("\n" + "=" * 70)
print("ALL REFERENCE PARITY CHECKS COMPLETED SUCCESSFULLY")
print(f"Audit log saved to: {OUT_JSON}")
print("=" * 70)
if __name__ == "__main__":
main()