Spaces:
Sleeping
Sleeping
Download src/evaluation/evaluate_benchmark.py from quangdat05/AIChamcong: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/spaces/quangdat05/AIChamcong/resolve/main/src/evaluation/evaluate_benchmark.py
- Command line
-
hf download hf://spaces/quangdat05/AIChamcong/src/evaluation/evaluate_benchmark.py
-
curl -L -o evaluate_benchmark.py https://huggingface.co/spaces/quangdat05/AIChamcong/resolve/main/src/evaluation/evaluate_benchmark.py
12.6 kB
| """ | |
| Script đánh giá hiệu năng mô hình nhận diện khuôn mặt trên các tập benchmark chuẩn (LFW, CALFW, CPLFW, AgeDB-30). | |
| Hỗ trợ cả mô hình đề xuất (Custom ResNet-50) và mô hình cơ sở (InsightFace buffalo_l). | |
| Sử dụng: | |
| # Đánh giá mô hình đề xuất (Custom) trên tập LFW | |
| python -m src.evaluation.evaluate_benchmark --model-type custom --checkpoint checkpoints/arcface_vggface2_warmup.pth --dataset lfw | |
| # Đánh giá mô hình đề xuất (Custom) trên tất cả các tập | |
| python -m src.evaluation.evaluate_benchmark --model-type custom --checkpoint checkpoints/arcface_vggface2_warmup.pth --dataset all | |
| # Đánh giá mô hình InsightFace (Baseline) trên tập LFW | |
| python -m src.evaluation.evaluate_benchmark --model-type baseline --dataset lfw | |
| """ | |
| import os | |
| import csv | |
| import argparse | |
| import numpy as np | |
| import cv2 | |
| import torch | |
| import matplotlib.pyplot as plt | |
| from PIL import Image | |
| from tqdm import tqdm | |
| from sklearn.metrics import roc_auc_score, roc_curve, confusion_matrix, ConfusionMatrixDisplay | |
| from src import config | |
| from src.models.face_recognition_model import FaceRecognitionModel | |
| from src.utils.transforms import get_val_transform | |
| def cosine_similarity(emb1, emb2): | |
| # Chuẩn hóa L2 trước khi tính dot product để có Cosine Similarity chuẩn | |
| emb1 = emb1 / np.linalg.norm(emb1) | |
| emb2 = emb2 / np.linalg.norm(emb2) | |
| return float(np.dot(emb1, emb2)) | |
| def calculate_best_threshold(labels, scores): | |
| best_acc = 0.0 | |
| best_threshold = 0.0 | |
| thresholds = np.linspace(-1, 1, 2001) | |
| labels = np.array(labels) | |
| scores = np.array(scores) | |
| for threshold in thresholds: | |
| preds = scores >= threshold | |
| acc = np.mean(preds == labels) | |
| if acc > best_acc: | |
| best_acc = acc | |
| best_threshold = threshold | |
| return best_threshold, best_acc | |
| def calculate_eer(labels, scores): | |
| labels = np.array(labels) | |
| scores = np.array(scores) | |
| fpr, tpr, thresholds = roc_curve(labels, scores) | |
| fnr = 1 - tpr | |
| idx = np.nanargmin(np.abs(fpr - fnr)) | |
| eer = (fpr[idx] + fnr[idx]) / 2 | |
| eer_threshold = thresholds[idx] | |
| return eer, eer_threshold | |
| class BenchmarkEvaluator: | |
| def __init__(self, model_type="custom", checkpoint_path=None, ctx_id=-1): | |
| self.model_type = model_type | |
| self.checkpoint_path = checkpoint_path | |
| self.cache = {} | |
| if self.model_type == "custom": | |
| print(f"[INFO] Initializing Custom Model (ResNet-50)...") | |
| self.model = FaceRecognitionModel(checkpoint_path=checkpoint_path) | |
| self.transform = get_val_transform() | |
| else: | |
| print(f"[INFO] Initializing Baseline Model (InsightFace buffalo_l)...") | |
| # Tải model từ InsightFace ONNX | |
| from insightface.model_zoo import get_model | |
| onnx_path = r"C:\Users\Admin\.insightface\models\buffalo_l\w600k_r50.onnx" | |
| if not os.path.exists(onnx_path): | |
| raise FileNotFoundError(f"InsightFace ONNX model not found at: {onnx_path}") | |
| self.rec_model = get_model(onnx_path, providers=["CPUExecutionProvider"]) | |
| self.rec_model.prepare(ctx_id=ctx_id) | |
| def load_image_embedding(self, image_path): | |
| if image_path in self.cache: | |
| return self.cache[image_path] | |
| if self.model_type == "custom": | |
| image = Image.open(image_path).convert("RGB") | |
| image = self.transform(image) | |
| # Trích xuất đặc trưng | |
| embedding = self.model.get_embedding(image) | |
| embedding = embedding.squeeze(0).numpy() | |
| else: | |
| image = cv2.imread(image_path) | |
| if image is None: | |
| raise RuntimeError(f"Cannot read image: {image_path}") | |
| if image.shape[:2] != (112, 112): | |
| image = cv2.resize(image, (112, 112)) | |
| embedding = self.rec_model.get_feat(image) | |
| embedding = embedding.flatten().astype(np.float32) | |
| # L2 Normalize | |
| embedding = embedding / np.linalg.norm(embedding) | |
| self.cache[image_path] = embedding | |
| return embedding | |
| def read_pairs(self, root_dir, pairs_file): | |
| pairs = [] | |
| with open(pairs_file, "r", encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| parts = line.split() | |
| if len(parts) != 3: | |
| continue | |
| label = int(parts[0]) | |
| img1 = parts[1] | |
| img2 = parts[2] | |
| img1_path = os.path.join(root_dir, img1) | |
| img2_path = os.path.join(root_dir, img2) | |
| pairs.append((img1_path, img2_path, label)) | |
| return pairs | |
| def evaluate_dataset(self, root_dir, pairs_file, output_csv): | |
| pairs = self.read_pairs(root_dir, pairs_file) | |
| labels = [] | |
| scores = [] | |
| missing = 0 | |
| failed = 0 | |
| dataset_name = os.path.basename(pairs_file).split("_")[0].upper() | |
| desc = f"Evaluating {dataset_name} ({self.model_type})" | |
| for img1, img2, label in tqdm(pairs, desc=desc): | |
| if not os.path.exists(img1) or not os.path.exists(img2): | |
| missing += 1 | |
| continue | |
| try: | |
| emb1 = self.load_image_embedding(img1) | |
| emb2 = self.load_image_embedding(img2) | |
| score = cosine_similarity(emb1, emb2) | |
| labels.append(label) | |
| scores.append(score) | |
| except Exception as e: | |
| failed += 1 | |
| print(f"[WARNING] Skip pair: {img1} | {img2} | Error: {e}") | |
| if len(labels) == 0: | |
| raise RuntimeError(f"No valid pairs for {dataset_name}.") | |
| labels_np = np.array(labels) | |
| scores_np = np.array(scores) | |
| # Tính toán các chỉ số | |
| threshold, accuracy = calculate_best_threshold(labels_np, scores_np) | |
| auc_score = roc_auc_score(labels_np, scores_np) | |
| eer, eer_threshold = calculate_eer(labels_np, scores_np) | |
| preds = scores_np >= threshold | |
| tp = np.sum((preds == 1) & (labels_np == 1)) | |
| tn = np.sum((preds == 0) & (labels_np == 0)) | |
| fp = np.sum((preds == 1) & (labels_np == 0)) | |
| fn = np.sum((preds == 0) & (labels_np == 1)) | |
| far = fp / max(1, fp + tn) | |
| frr = fn / max(1, fn + tp) | |
| # Lưu CSV kết quả | |
| os.makedirs(os.path.dirname(output_csv), exist_ok=True) | |
| with open(output_csv, "w", newline="", encoding="utf-8") as f: | |
| writer = csv.writer(f) | |
| writer.writerow([ | |
| "total_pairs", "valid_pairs", "missing_pairs", "failed_pairs", | |
| "accuracy", "auc", "eer", "best_threshold", "eer_threshold", | |
| "far", "frr", "tp", "tn", "fp", "fn" | |
| ]) | |
| writer.writerow([ | |
| len(pairs), len(labels), missing, failed, | |
| accuracy, auc_score, eer, threshold, eer_threshold, | |
| far, frr, tp, tn, fp, fn | |
| ]) | |
| # Vẽ Ma trận nhầm lẫn và lưu biểu đồ | |
| cm = confusion_matrix(labels_np, preds, labels=[0, 1]) | |
| disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=["Different", "Same"]) | |
| disp.plot(cmap="Blues", values_format="d") | |
| plt.title(f"{dataset_name} Confusion Matrix ({self.model_type.upper()})") | |
| plt.tight_layout() | |
| base_name = os.path.splitext(os.path.basename(output_csv))[0] | |
| cm_path = os.path.join(os.path.dirname(output_csv), f"{base_name}_confusion_matrix.png") | |
| plt.savefig(cm_path, dpi=300) | |
| plt.close() | |
| # Vẽ đường cong ROC và lưu biểu đồ | |
| fpr, tpr, _ = roc_curve(labels_np, scores_np) | |
| plt.figure() | |
| plt.plot(fpr, tpr, color="darkorange", lw=2, label=f"ROC curve (area = {auc_score:.4f})") | |
| plt.plot([0, 1], [0, 1], color="navy", lw=2, linestyle="--") | |
| plt.xlim([0.0, 1.0]) | |
| plt.ylim([0.0, 1.05]) | |
| plt.xlabel("False Positive Rate") | |
| plt.ylabel("True Positive Rate") | |
| plt.title(f"{dataset_name} ROC Curve ({self.model_type.upper()})") | |
| plt.legend(loc="lower right") | |
| plt.tight_layout() | |
| roc_path = os.path.join(os.path.dirname(output_csv), f"{base_name}_roc_curve.png") | |
| plt.savefig(roc_path, dpi=300) | |
| plt.close() | |
| return { | |
| "dataset": dataset_name, | |
| "total": len(pairs), | |
| "valid": len(labels), | |
| "accuracy": accuracy, | |
| "auc": auc_score, | |
| "eer": eer, | |
| "best_threshold": threshold, | |
| "far": far, | |
| "frr": frr, | |
| "tp": tp, | |
| "tn": tn, | |
| "fp": fp, | |
| "fn": fn, | |
| "cm_path": cm_path, | |
| "roc_path": roc_path | |
| } | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Evaluate model on LFW, CALFW, CPLFW, AgeDB-30") | |
| parser.add_argument( | |
| "--model-type", | |
| choices=["custom", "baseline"], | |
| default="custom", | |
| help="Model type: custom (ResNet-50) or baseline (InsightFace buffalo_l)" | |
| ) | |
| parser.add_argument( | |
| "--checkpoint", | |
| type=str, | |
| default=os.path.join(config.CHECKPOINT_DIR, "arcface_vggface2_warmup.pth"), | |
| help="Path to custom model checkpoint" | |
| ) | |
| parser.add_argument( | |
| "--dataset", | |
| choices=["lfw", "calfw", "cplfw", "agedb30", "all"], | |
| default="all", | |
| help="Dataset name" | |
| ) | |
| parser.add_argument( | |
| "--output-dir", | |
| type=str, | |
| default="outputs/evaluation", | |
| help="Output directory" | |
| ) | |
| parser.add_argument( | |
| "--ctx-id", | |
| type=int, | |
| default=-1, | |
| help="GPU ID (-1 for CPU)" | |
| ) | |
| args = parser.parse_args() | |
| # Định nghĩa cấu hình các tập dữ liệu | |
| val_root = "dataset/benchmark/val" | |
| datasets_config = { | |
| "lfw": { | |
| "root": val_root, | |
| "pairs": os.path.join(val_root, "lfw_ann.txt"), | |
| "csv": "lfw_eval.csv" if args.model_type == "custom" else "lfw_buffalo_l_eval.csv" | |
| }, | |
| "calfw": { | |
| "root": val_root, | |
| "pairs": os.path.join(val_root, "calfw_ann.txt"), | |
| "csv": "calfw_eval.csv" if args.model_type == "custom" else "calfw_buffalo_l_eval.csv" | |
| }, | |
| "cplfw": { | |
| "root": val_root, | |
| "pairs": os.path.join(val_root, "cplfw_ann.txt"), | |
| "csv": "cplfw_eval.csv" if args.model_type == "custom" else "cplfw_buffalo_l_eval.csv" | |
| }, | |
| "agedb30": { | |
| "root": val_root, | |
| "pairs": os.path.join(val_root, "agedb_30_ann.txt"), | |
| "csv": "agedb30_eval.csv" if args.model_type == "custom" else "agedb30_buffalo_l_eval.csv" | |
| } | |
| } | |
| # Lọc danh sách tập dữ liệu chạy thực tế | |
| target_keys = [args.dataset] if args.dataset != "all" else ["lfw", "calfw", "cplfw", "agedb30"] | |
| # Khởi tạo Evaluator | |
| evaluator = BenchmarkEvaluator( | |
| model_type=args.model_type, | |
| checkpoint_path=args.checkpoint if args.model_type == "custom" else None, | |
| ctx_id=args.ctx_id | |
| ) | |
| results = [] | |
| for key in target_keys: | |
| cfg = datasets_config[key] | |
| csv_path = os.path.join(args.output_dir, cfg["csv"]) | |
| print("\n" + "="*80) | |
| print(f"Evaluating: {key.upper()}...") | |
| print(f" Images root: {cfg['root']}") | |
| print(f" Pairs file: {cfg['pairs']}") | |
| print(f" Output CSV: {csv_path}") | |
| print("="*80) | |
| res = evaluator.evaluate_dataset( | |
| root_dir=cfg["root"], | |
| pairs_file=cfg["pairs"], | |
| output_csv=csv_path | |
| ) | |
| results.append(res) | |
| # In ra bảng tổng kết kết quả đánh giá | |
| print("\n" + "="*90) | |
| print(f" EVALUATION SUMMARY TABLE ({args.model_type.upper()})") | |
| print("="*90) | |
| print(f"{'Dataset':<12} | {'Pairs':<6} | {'Accuracy':<10} | {'AUC':<8} | {'EER':<8} | {'Threshold':<10} | {'FAR':<8} | {'FRR':<8}") | |
| print("-"*90) | |
| for r in results: | |
| print(f"{r['dataset']:<12} | {r['valid']:<6} | {r['accuracy']*100:>8.2f}% | {r['auc']:>8.4f} | {r['eer']*100:>6.2f}% | {r['best_threshold']:>10.4f} | {r['far']*100:>6.2f}% | {r['frr']*100:>6.2f}%") | |
| print("="*90) | |
| print(f"All reports saved to: {args.output_dir}") | |
| if __name__ == "__main__": | |
| main() | |