Blame for evaluate.py

"""
evaluate.py — Batch evaluation for I-ILIA-014 MIR Project
Group 7 (odd-numbered) → Table 1 queries (classes 0, 2, 4, 6, 8)
Database: even-prefix images (filter % 2 == 0)

Generates:
  - results/table3_indexing_performance.csv  (indexing time, descriptor size, search time)
  - results/table4_query_metrics.csv         (R, P, AP, R-Prec @50/@100 + mAP)
"""

import os
import sys
import time
import csv
import numpy as np

PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
sys.path.insert(0, PROJECT_ROOT)

import functions as f

DATASET_PATH     = os.path.join(PROJECT_ROOT, "dataset_voitures", "dataset")
DESCRIPTORS_PATH = os.path.join(PROJECT_ROOT, "descriptors")

# ── 15 mandatory queries — TABLE 1 (Group 7 = odd-numbered) ────────────────
# Classes: 0=BMW, 2=Volkswagen, 4=Opel, 6=Hyundai, 8=Ford
QUERIES = [
    ("0_1_BMW_X3_207.jpg",                "0"),
    ("0_0_BMW_Serie3Berline_74.jpg",      "0"),
    ("0_2_BMW_i8_299.jpg",                "0"),

    ("2_0_Volkswagen_Touareg_2822.jpg",   "2"),
    ("2_4_Volkswagen_Polo_3463.jpg",      "2"),
    ("2_9_Volkswagen_T-Roc_4209.jpg",     "2"),

    ("4_2_Opel_vivarofourgon_5999.jpg",   "4"),
    ("4_4_Opel_Insignatourer_6353.jpg",   "4"),
    ("4_9_Opel_zafiralife_6887.jpg",      "4"),

    ("6_0_Hyundai_Nexo_8282.jpg",         "6"),
    ("6_3_Hyundai_i10_8837.jpg",          "6"),
    ("6_5_Hyundai_i30_9125.jpg",          "6"),

    ("8_1_Ford_Puma_11276.jpg",           "8"),
    ("8_5_Ford_Explorer_11897.jpg",       "8"),
    ("8_6_Ford_Focus_11951.jpg",          "8"),
]

# ── Top-3 descriptors (must include ≥1 CNN + ≥1 ViT per énoncé Note 1) ────
# Tuple: (display_name, folder_name, descriptor_fn, default_distance)
TOP_3_DESCRIPTORS = [
    ("ResNet50", "ResNet50", f.generateResNet50, "Euclidienne"),
    ("CLIP_ViT", "CLIP",     f.generateCLIP,     "Cosine Similarity"),
    ("HOG",      "HOG",      f.generateHOG,      "Euclidienne"),
]


def get_image_class(filename):
    """First digit of the filename is the class (brand).
    e.g. '0_1_BMW_X3_207.jpg' -> '0'  (BMW)
    The second digit is a model sub-index — NOT part of the class.
    """
    return os.path.basename(filename).split("_")[0]


def compute_all_metrics(ranked_filenames, query_class, total_relevant, k):
    """Returns (precision, recall, ap, r_precision) at cutoff k."""
    top_k = ranked_filenames[:k]
    relevant_found = sum(1 for fn in top_k if get_image_class(fn) == query_class)

    precision = relevant_found / k if k > 0 else 0.0
    recall    = relevant_found / total_relevant if total_relevant > 0 else 0.0

    # Average Precision calculation
    hits, sum_precisions = 0, 0.0
    for i, fn in enumerate(top_k, 1):
        if get_image_class(fn) == query_class:
            hits += 1
            sum_precisions += hits / i
            
    # Standard global AP — academically correct denominator (all relevant in corpus)
    ap = sum_precisions / total_relevant if total_relevant > 0 else 0.0

    # AP@K — valid IR metric; denominator capped at k, yields higher values
    ap_at_k = sum_precisions / min(total_relevant, k) if total_relevant > 0 else 0.0
    

    # R-Precision (precision at rank R, where R = total relevant)
    r = min(total_relevant, len(ranked_filenames))
    r_hits = sum(1 for fn in ranked_filenames[:r] if get_image_class(fn) == query_class)
    r_prec = r_hits / r if r > 0 else 0.0

    return precision, recall, ap, ap_at_k, r_prec


def load_descriptors(desc_folder, desc_name):
    """Load all even-prefix descriptor files (matches the indexed DB)."""
    desc_dir = os.path.join(DESCRIPTORS_PATH, desc_folder)
    if not os.path.isdir(desc_dir):
        print(f"[WARN] Descriptor folder not found: {desc_dir}")
        return []

    files = [
        ff for ff in os.listdir(desc_dir)
        if ff.endswith(".txt")
        and ff[0].isdigit()
        and int(ff[0]) % 2 == 0    # even-prefix → classes 0, 2, 4, 6, 8 (Table 1)
    ]

    data = []
    for fname in files:
        fpath = os.path.join(desc_dir, fname)
        try:
            feat = np.loadtxt(fpath)
            img_file = fname.replace(f"_{desc_folder}.txt", ".jpg")
            data.append((img_file, feat))
        except Exception as e:
            print(f"[WARN] could not load {fname}: {e}")
    return data


def run_query(query_img_path, query_class, desc_data, desc_func, distance_name):
    """Run one query: extract feature, compute distance to every DB entry, return ranked filenames."""
    img = cv2.imread(query_img_path)
    if img is None:
        print(f"[ERROR] could not read {query_img_path}")
        return [], 0

    query_feat = desc_func(img)
    total_relevant = sum(1 for img_f, _ in desc_data if get_image_class(img_f) == query_class)

    results = []
    for img_file, db_feat in desc_data:
        try:
            dist = f.distance_f(query_feat, db_feat, distance_name)
            results.append((img_file, dist))
        except Exception:
            continue

    # similarity metrics sort descending; distance metrics sort ascending
    higher_is_better = distance_name in {
        "Correlation", "Cosine Similarity", "Intersection", "Brute force", "Flann"
    }
    results.sort(key=lambda x: x[1], reverse=higher_is_better)

    return [r[0] for r in results], total_relevant


def benchmark_indexing():
    """Table 3: indexing time, descriptor size, avg search time."""
    rows = []
    sample_img_path = os.path.join(DATASET_PATH, QUERIES[0][0])
    sample_img = cv2.imread(sample_img_path)

    if sample_img is None:
        print(f"[FATAL] sample image not found: {sample_img_path}")
        print("Check your dataset path and filenames.")
        return

    for desc_name, desc_folder, desc_func, distance_name in TOP_3_DESCRIPTORS:
        desc_dir = os.path.join(DESCRIPTORS_PATH, desc_folder)

        # indexing time = 1 image feature extraction
        start = time.time()
        desc_func(sample_img)
        index_time = round(time.time() - start, 4)

        # descriptor folder size in MB
        total_size = 0
        if os.path.exists(desc_dir):
            for ff in os.listdir(desc_dir):
                total_size += os.path.getsize(os.path.join(desc_dir, ff))
        size_mb = round(total_size / (1024 * 1024), 2)

        # avg search time over 50 comparisons
        desc_data = load_descriptors(desc_folder, desc_name)
        if not desc_data:
            print(f"[WARN] no descriptors loaded for {desc_name} — skipping search timing")
            avg_search = 0.0
        else:
            sample_feat = desc_func(sample_img)
            n = min(50, len(desc_data))
            start = time.time()
            for img_file, db_feat in desc_data[:n]:
                f.distance_f(sample_feat, db_feat, distance_name)
            avg_search = round((time.time() - start) / n, 4)

        rows.append([desc_name, index_time, size_mb, avg_search])
        print(f"[Table 3] {desc_name}: index={index_time}s · size={size_mb}MB · search={avg_search}s")

    os.makedirs("results", exist_ok=True)
    with open("results/table3_indexing_performance.csv", "w", newline="", encoding="utf-8") as csvf:
        writer = csv.writer(csvf)
        writer.writerow(["Descriptor", "Indexing Time (s)", "Size (MB)", "Avg Search Time (s)"])
        writer.writerows(rows)
    print("Saved: results/table3_indexing_performance.csv")


def benchmark_queries():
    """Table 4: R, P, AP, R-Prec @50 and @100 + mAP for all 15 queries × top-3 descriptors."""
    os.makedirs("results", exist_ok=True)
    rows = []
    map_summary = []

    for desc_name, desc_folder, desc_func, distance_name in TOP_3_DESCRIPTORS:
        print(f"\n[Table 4] Running queries for descriptor: {desc_name}")
        desc_data = load_descriptors(desc_folder, desc_name)
        if not desc_data:
            print(f"[ERROR] no descriptors loaded for {desc_name} — skipping")
            continue

        ap_list_50, ap_list_100 = [], []
        ap_k_list_50, ap_k_list_100 = [], []

        for q_idx, (q_file, q_class) in enumerate(QUERIES, 1):
            q_path = os.path.join(DATASET_PATH, q_file)
            ranked, total_rel = run_query(q_path, q_class, desc_data, desc_func, distance_name)

            p50,  r50,  ap50,  ap_k50,  rp50  = compute_all_metrics(ranked, q_class, total_rel, 50)
            p100, r100, ap100, ap_k100, rp100 = compute_all_metrics(ranked, q_class, total_rel, 100)

            

            # Append metrics to lists
            ap_list_50.append(ap50)
            ap_list_100.append(ap100)
            ap_k_list_50.append(ap_k50)
            ap_k_list_100.append(ap_k100)
            

            rows.append([
                f"R{q_idx}", q_file, desc_name,
                round(r50   * 100, 1), round(p50   * 100, 1),
                round(ap50  * 100, 1), round(ap_k50  * 100, 1), round(rp50  * 100, 1),
                round(r100  * 100, 1), round(p100  * 100, 1),
                round(ap100 * 100, 1), round(ap_k100 * 100, 1), round(rp100 * 100, 1),
            ])
            print(f"  R{q_idx:>2} {q_file[:42]:42} P@50={p50*100:5.1f}% R@50={r50*100:5.1f}% AP@50={ap50*100:5.1f}%")

        map50      = round(np.mean(ap_list_50)    * 100, 1)
        map100     = round(np.mean(ap_list_100)   * 100, 1)
        map_k50    = round(np.mean(ap_k_list_50)  * 100, 1)
        map_k100   = round(np.mean(ap_k_list_100) * 100, 1)
        map_summary.append((desc_name, map50, map100, map_k50, map_k100))
        print(f"  → mAP@50={map50}%  mAP@K50={map_k50}%  |  mAP@100={map100}%  mAP@K100={map_k100}%")

    with open("results/table4_query_metrics.csv", "w", newline="", encoding="utf-8") as csvf:
        writer = csv.writer(csvf)
        writer.writerow([
            "Query", "Image", "Descriptor",
            "R@50", "P@50", "AP@50", "AP@K50", "R-Prec@50",
            "R@100", "P@100", "AP@100", "AP@K100", "R-Prec@100",
        ])
        writer.writerows(rows)
    print("\nSaved: results/table4_query_metrics.csv")

    print("\n" + "=" * 60)
    print("FINAL mAP SUMMARY (Group 7 → Table 1 queries)")
    print("=" * 60)
    print(f"{'Descriptor':<14} {'mAP@50':>10} {'mAP@100':>10} {'mAP@K50':>10} {'mAP@K100':>10}")
    print("-" * 50)
    for name, m50, m100, mk50, mk100 in map_summary:
        print(f"{name:<14} {m50:>9.1f}% {m100:>9.1f}% {mk50:>9.1f}% {mk100:>9.1f}%")
    
    # ── Unimodal rankings dump — consumed by plot_pr_curves.py ──────────
    import json as _json
    _dump = {}
    for desc_name, desc_folder, desc_func, distance_name in TOP_3_DESCRIPTORS:
        _dump[desc_name] = {"queries": []}
        desc_data = load_descriptors(desc_folder, desc_name)
        for q_file, q_class in QUERIES:
            q_path = os.path.join(DATASET_PATH, q_file)
            ranked, total_rel = run_query(q_path, q_class, desc_data, desc_func, distance_name)
            _dump[desc_name]["queries"].append({
                "query_id":   q_file,
                "n_relevant": total_rel,
                "ranked": [
                    {"rank": r + 1,
                    "filename": fn,
                    "relevant": get_image_class(fn) == q_class}
                    for r, fn in enumerate(ranked)
                ]
            })
    os.makedirs("results", exist_ok=True)
    _json_path = os.path.join("results", "unimodal_rankings.json")
    with open(_json_path, "w", encoding="utf-8") as _f:
        _json.dump(_dump, _f)
    print(f"Saved: {_json_path}")


if __name__ == "__main__":
    import cv2  # delayed so the script can show a help banner without the import
    print("=" * 60)
    print("TABLE 3 — Indexing Performance")
    print("=" * 60)
    benchmark_indexing()

    print("\n" + "=" * 60)
    print("TABLE 4 — Query Metrics")
    print("=" * 60)
    benchmark_queries()