Download evaluate.py from AI-D3S/AI_Driven_Deepfake_Detection_System: direct link, hf CLI and curl.
- Browser
- Download file 9.74 kB
-
https://huggingface.co/spaces/AI-D3S/AI_Driven_Deepfake_Detection_System/resolve/main/evaluate.py
- Command line
-
hf download hf://spaces/AI-D3S/AI_Driven_Deepfake_Detection_System/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/AI-D3S/AI_Driven_Deepfake_Detection_System/resolve/main/evaluate.py
9.74 kB
| import os | |
| import time | |
| import torch | |
| import numpy as np | |
| import torch.nn.functional as F | |
| import matplotlib.pyplot as plt | |
| from torch.utils.data import DataLoader | |
| from torchvision import datasets, transforms | |
| from transformers import ViTForImageClassification, ViTConfig | |
| from sklearn.metrics import ( | |
| accuracy_score, | |
| precision_score, | |
| recall_score, | |
| f1_score, | |
| roc_auc_score, | |
| confusion_matrix, | |
| roc_curve | |
| ) | |
| from openpyxl import Workbook | |
| def main(): | |
| # ---------------------------------- | |
| # PERFORMANCE TUNING | |
| # ---------------------------------- | |
| torch.backends.cudnn.benchmark = True | |
| torch.set_num_threads(8) | |
| # ---------------------------------- | |
| # Device (SAFE) | |
| # ---------------------------------- | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| use_amp = device.type == "cuda" | |
| if use_amp: | |
| print(f"π Using device: cuda ({torch.cuda.get_device_name(0)})") | |
| else: | |
| print("π Using device: cpu") | |
| # ---------------------------------- | |
| # Paths | |
| # ---------------------------------- | |
| base_data_dir = "data" | |
| # π΄ UPDATED MODEL PATH (NEW MODEL) | |
| model_path = "model/vit_face_final_best.pth" | |
| datasets_config = { | |
| "FF++": f"{base_data_dir}/ff++/test", | |
| "Celeb-DF": f"{base_data_dir}/celeb-df/test", | |
| "DFDC": f"{base_data_dir}/dfdc/test", | |
| } | |
| os.makedirs("outputs", exist_ok=True) | |
| # ---------------------------------- | |
| # Transforms | |
| # ---------------------------------- | |
| test_tfms = transforms.Compose([ | |
| transforms.Resize(256), | |
| transforms.CenterCrop(224), | |
| transforms.ToTensor(), | |
| transforms.Normalize( | |
| mean=[0.485, 0.456, 0.406], | |
| std=[0.229, 0.224, 0.225] | |
| ) | |
| ]) | |
| # ---------------------------------- | |
| # Model | |
| # ---------------------------------- | |
| config = ViTConfig.from_pretrained( | |
| "google/vit-base-patch16-224", | |
| num_labels=2 | |
| ) | |
| model = ViTForImageClassification.from_pretrained( | |
| "google/vit-base-patch16-224", | |
| config=config, | |
| ignore_mismatched_sizes=True | |
| ) | |
| model.load_state_dict(torch.load(model_path, map_location=device)) | |
| model.to(device) | |
| model.eval() | |
| print("β Model loaded successfully") | |
| # ---------------------------------- | |
| # Evaluation | |
| # ---------------------------------- | |
| all_results = {} | |
| print("\nπ§ͺ Running Cross-Dataset Evaluation...\n") | |
| for ds_name, test_dir in datasets_config.items(): | |
| if not os.path.exists(test_dir): | |
| print(f"β οΈ {ds_name}: path not found β skipping") | |
| continue | |
| test_ds = datasets.ImageFolder(test_dir, transform=test_tfms) | |
| test_dl = DataLoader( | |
| test_ds, | |
| batch_size=32, | |
| shuffle=False, | |
| num_workers=2, # SAFE (inside main) | |
| pin_memory=True | |
| ) | |
| class_to_idx = test_ds.class_to_idx | |
| real_idx = class_to_idx["real"] | |
| fake_idx = class_to_idx["fake"] | |
| print(f"π {ds_name}: {len(test_ds)} samples | {class_to_idx}") | |
| y_true, y_pred, y_probs = [], [], [] | |
| total_images = 0 | |
| total_time = 0.0 | |
| with torch.no_grad(): | |
| for imgs, labels in test_dl: | |
| imgs = imgs.to(device, non_blocking=True) | |
| labels = labels.to(device, non_blocking=True) | |
| start = time.time() | |
| with torch.amp.autocast(device_type="cuda", enabled=use_amp): | |
| logits = model(imgs).logits | |
| end = time.time() | |
| probs = F.softmax(logits, dim=1)[:, fake_idx] | |
| preds = torch.argmax(logits, dim=1) | |
| total_time += (end - start) | |
| total_images += imgs.size(0) | |
| y_true.extend(labels.cpu().numpy()) | |
| y_pred.extend(preds.cpu().numpy()) | |
| y_probs.extend(probs.cpu().numpy()) | |
| # ---------------- Metrics ---------------- | |
| acc = accuracy_score(y_true, y_pred) | |
| prec = precision_score(y_true, y_pred, zero_division=0) | |
| rec = recall_score(y_true, y_pred, zero_division=0) | |
| f1 = f1_score(y_true, y_pred, zero_division=0) | |
| try: | |
| auc = roc_auc_score( | |
| (np.array(y_true) == fake_idx).astype(int), | |
| y_probs | |
| ) | |
| except ValueError: | |
| auc = float("nan") | |
| avg_time_ms = (total_time / total_images) * 1000 | |
| fps = total_images / total_time | |
| all_results[ds_name] = { | |
| "acc": acc, | |
| "prec": prec, | |
| "rec": rec, | |
| "f1": f1, | |
| "auc": auc, | |
| "time": avg_time_ms, | |
| "fps": fps, | |
| } | |
| print( | |
| f"π― {ds_name} | Acc: {acc:.4f} | F1: {f1:.4f} | " | |
| f"AUC: {auc:.4f} | {avg_time_ms:.2f} ms/img | {fps:.2f} FPS" | |
| ) | |
| # ---------------- Confusion Matrix ---------------- | |
| cm = confusion_matrix(y_true, y_pred, labels=[real_idx, fake_idx]) | |
| plt.figure(figsize=(5, 4)) | |
| plt.imshow(cm) | |
| plt.title(f"{ds_name} - Confusion Matrix") | |
| plt.xlabel("Predicted") | |
| plt.ylabel("True") | |
| plt.xticks([0, 1], ["Real", "Fake"]) | |
| plt.yticks([0, 1], ["Real", "Fake"]) | |
| for i in range(2): | |
| for j in range(2): | |
| plt.text(j, i, cm[i, j], ha="center", va="center") | |
| plt.tight_layout() | |
| plt.savefig(f"outputs/cm_{ds_name}.png") | |
| plt.close() | |
| # ---------------- ROC Curve ---------------- | |
| fpr, tpr, _ = roc_curve(y_true, y_probs, pos_label=fake_idx) | |
| plt.figure(figsize=(5, 4)) | |
| plt.plot(fpr, tpr, label=f"AUC = {auc:.4f}") | |
| plt.plot([0, 1], [0, 1], linestyle="--") | |
| plt.xlabel("False Positive Rate") | |
| plt.ylabel("True Positive Rate") | |
| plt.title(f"{ds_name} - ROC Curve") | |
| plt.legend(loc="lower right") | |
| plt.tight_layout() | |
| plt.savefig(f"outputs/roc_{ds_name}.png") | |
| plt.close() | |
| # ---------------------------------- | |
| # EXPORT (EXCEL + TEXT) | |
| # ---------------------------------- | |
| avg_metrics = { | |
| "acc": np.mean([m["acc"] for m in all_results.values()]), | |
| "prec": np.mean([m["prec"] for m in all_results.values()]), | |
| "rec": np.mean([m["rec"] for m in all_results.values()]), | |
| "f1": np.mean([m["f1"] for m in all_results.values()]), | |
| "auc": np.nanmean([m["auc"] for m in all_results.values()]), | |
| "time": np.mean([m["time"] for m in all_results.values()]), | |
| "fps": np.mean([m["fps"] for m in all_results.values()]), | |
| } | |
| all_results["AVERAGE"] = avg_metrics | |
| wb = Workbook() | |
| ws = wb.active | |
| ws.title = "Evaluation Results" | |
| ws.append(["Dataset", "Accuracy", "Precision", "Recall", "F1", "AUC", "ms/img", "FPS"]) | |
| for ds, m in all_results.items(): | |
| ws.append([ | |
| ds, | |
| round(m["acc"], 4), | |
| round(m["prec"], 4), | |
| round(m["rec"], 4), | |
| round(m["f1"], 4), | |
| round(m["auc"], 4), | |
| round(m["time"], 2), | |
| round(m["fps"], 2), | |
| ]) | |
| wb.save("outputs/evaluation_results.xlsx") | |
| with open("outputs/summary.txt", "w") as f: | |
| for ds, m in all_results.items(): | |
| f.write(f"Dataset: {ds}\n") | |
| for k, v in m.items(): | |
| f.write(f" {k.upper():<6}: {v}\n") | |
| f.write("-" * 45 + "\n") | |
| print("β Evaluation complete (new model evaluated successfully)") | |
| # ---------------------------------- | |
| # COMBINED ROC CURVE (IEEE-FRIENDLY) | |
| # ---------------------------------- | |
| plt.figure(figsize=(5, 4)) | |
| for ds_name in datasets_config.keys(): | |
| if ds_name not in all_results: | |
| continue | |
| # Reload dataset to recompute ROC cleanly | |
| test_dir = datasets_config[ds_name] | |
| test_ds = datasets.ImageFolder(test_dir, transform=test_tfms) | |
| class_to_idx = test_ds.class_to_idx | |
| fake_idx = class_to_idx["fake"] | |
| test_dl = DataLoader( | |
| test_ds, | |
| batch_size=32, | |
| shuffle=False, | |
| num_workers=2, | |
| pin_memory=True | |
| ) | |
| y_true, y_probs = [], [] | |
| with torch.no_grad(): | |
| for imgs, labels in test_dl: | |
| imgs = imgs.to(device, non_blocking=True) | |
| labels = labels.to(device, non_blocking=True) | |
| with torch.amp.autocast(device_type="cuda", enabled=use_amp): | |
| logits = model(imgs).logits | |
| probs = F.softmax(logits, dim=1)[:, fake_idx] | |
| y_true.extend((labels == fake_idx).cpu().numpy()) | |
| y_probs.extend(probs.cpu().numpy()) | |
| fpr, tpr, _ = roc_curve(y_true, y_probs) | |
| auc_val = roc_auc_score(y_true, y_probs) | |
| # Line styles for IEEE (print-safe) | |
| if ds_name == "FF++": | |
| style = "-" | |
| elif ds_name == "Celeb-DF": | |
| style = "--" | |
| else: # DFDC | |
| style = "-." | |
| plt.plot( | |
| fpr, | |
| tpr, | |
| linestyle=style, | |
| linewidth=2, | |
| label=f"{ds_name} (AUC = {auc_val:.4f})" | |
| ) | |
| # Random baseline | |
| plt.plot([0, 1], [0, 1], linestyle=":", linewidth=1) | |
| plt.xlabel("False Positive Rate") | |
| plt.ylabel("True Positive Rate") | |
| plt.legend(loc="lower right") | |
| plt.tight_layout() | |
| # Save IEEE-ready figure | |
| plt.savefig("outputs/roc_combined.png", dpi=300, bbox_inches="tight") | |
| plt.savefig("outputs/roc_combined.pdf", dpi=300, bbox_inches="tight") | |
| plt.close() | |
| if __name__ == "__main__": | |
| main() | |