scripts/evaluate_recommendations.py

r"""Evaluation harness for the supervisor recommendation system.

Computes ranking metrics (Hit@k, Precision@k, Recall@k, MRR, nDCG@k, MAP) over a
ground-truth set of (research_title -> actual_supervisor) pairs, in-process
(loads artifacts once, then loops recommend_top_k).

Ground-truth sources:
  --input <csv>            CSV with a title column and an actual-supervisor column.
                           Configure with --title-col / --actual-col.
  --from-publications      Build an eval set from data/publications.csv using
                           judul_publikasi as the query title and dosen as the
                           actual supervisor(s). This is a PROXY (publications are
                           not thesis supervisions) and is not leave-one-out, so
                           treat the numbers as an upper-bound sanity check.

Usage (run from backend/ with the project venv):
  ..\.venv\Scripts\python.exe scripts\evaluate_recommendations.py --from-publications --sample 300
  ..\.venv\Scripts\python.exe scripts\evaluate_recommendations.py --input eval.csv --title-col judul --actual-col dosen
"""

from __future__ import annotations

import argparse
import os
import sys
import time

import numpy as np
import pandas as pd

BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if BACKEND_DIR not in sys.path:
    sys.path.insert(0, BACKEND_DIR)

from services.artifacts import load_artifacts  # noqa: E402
from services.evaluation import (  # noqa: E402
    average_precision,
    hit_at_k,
    ndcg_at_k,
    normalize_name,
    precision_at_k,
    recall_at_k,
    reciprocal_rank,
    split_actual_names,
)
from services.recommender import recommend_top_k  # noqa: E402


def build_eval_set(args) -> pd.DataFrame:
    if args.from_publications:
        path = args.publications or os.path.join(BACKEND_DIR, "data", "publications.csv")
        df = pd.read_csv(path)
        title_col = "judul_publikasi" if "judul_publikasi" in df.columns else args.title_col
        actual_col = "dosen" if "dosen" in df.columns else args.actual_col
    else:
        if not args.input:
            raise SystemExit("Sediakan --input <csv> atau gunakan --from-publications")
        df = pd.read_csv(args.input)
        title_col, actual_col = args.title_col, args.actual_col

    for col in (title_col, actual_col):
        if col not in df.columns:
            raise SystemExit(f"Kolom '{col}' tidak ada. Kolom tersedia: {list(df.columns)}")

    out = df[[title_col, actual_col]].rename(columns={title_col: "title", actual_col: "actual"})
    out = out.dropna(subset=["title", "actual"])
    out = out[out["title"].astype(str).str.strip().str.len() >= 3]
    out = out[out["actual"].astype(str).str.strip().str.len() >= 1].reset_index(drop=True)

    if args.sample and args.sample < len(out):
        out = out.sample(n=args.sample, random_state=args.seed).reset_index(drop=True)
    return out


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--input", default="")
    parser.add_argument("--from-publications", action="store_true")
    parser.add_argument("--publications", default="")
    parser.add_argument("--title-col", default="title")
    parser.add_argument("--actual-col", default="actual")
    parser.add_argument("--ks", default="1,3,5,10")
    parser.add_argument("--sample", type=int, default=0)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--artifacts-dir", default=os.path.join(BACKEND_DIR, "models"))
    parser.add_argument("--data-dir", default=os.path.join(BACKEND_DIR, "data"))
    parser.add_argument("--output", default=os.path.join(BACKEND_DIR, "scripts", "eval_details.csv"))
    args = parser.parse_args()

    ks = sorted({int(x) for x in args.ks.split(",") if x.strip()})
    max_k = max(ks)

    eval_df = build_eval_set(args)
    print(f"Eval set: {len(eval_df)} item | k = {ks}")

    # Avoid the PKL/NOTEBOOK loader; use the default split-lang loader.
    os.environ.pop("ARTIFACTS_MODE", None)
    print("Loading artifacts...")
    store = load_artifacts(artifacts_dir=args.artifacts_dir, data_dir=args.data_dir)

    # Accumulators per-k
    agg = {k: {"hit": 0.0, "p": 0.0, "r": 0.0, "ndcg": 0.0, "ap": 0.0} for k in ks}
    mrr_sum = 0.0
    rows = []
    latencies = []
    n_empty = 0

    for i, row in eval_df.iterrows():
        title = str(row["title"])
        actual = set(split_actual_names(row["actual"]))

        t0 = time.perf_counter()
        rec = recommend_top_k(store=store, title=title, top_k=max_k)
        latencies.append((time.perf_counter() - t0) * 1000.0)

        ranked = [normalize_name(r["nama_dosen"]) for r in rec.get("recommendations", [])]
        if not ranked:
            n_empty += 1

        rr = reciprocal_rank(ranked, actual)
        mrr_sum += rr

        detail = {
            "title": title[:120],
            "actual": "; ".join(sorted(actual)),
            "predicted_top": "; ".join(ranked[:max_k]),
            "detected_lang": rec.get("detected_lang"),
            "closest_topic_id": rec.get("closest_topic_id"),
            "rr": round(rr, 4),
        }
        for k in ks:
            agg[k]["hit"] += hit_at_k(ranked, actual, k)
            agg[k]["p"] += precision_at_k(ranked, actual, k)
            agg[k]["r"] += recall_at_k(ranked, actual, k)
            agg[k]["ndcg"] += ndcg_at_k(ranked, actual, k)
            agg[k]["ap"] += average_precision(ranked, actual, k)
            detail[f"hit@{k}"] = int(hit_at_k(ranked, actual, k))
        rows.append(detail)

        if (i + 1) % 50 == 0:
            print(f"  {i + 1}/{len(eval_df)} processed...")

    n = len(eval_df)
    denom = float(n) if n else 1.0

    print("\n" + "=" * 78)
    print(f"HASIL EVALUASI  (n={n}, empty_recs={n_empty})")
    print("=" * 78)
    header = f"{'k':>3} | {'Hit@k':>7} | {'Prec@k':>7} | {'Recall@k':>8} | {'nDCG@k':>7} | {'MAP@k':>7}"
    print(header)
    print("-" * len(header))
    for k in ks:
        a = agg[k]
        print(f"{k:>3} | {a['hit']/denom:>7.4f} | {a['p']/denom:>7.4f} | "
              f"{a['r']/denom:>8.4f} | {a['ndcg']/denom:>7.4f} | {a['ap']/denom:>7.4f}")
    print("-" * len(header))
    print(f"MRR (global): {mrr_sum/denom:.4f}")

    if latencies:
        lat = np.array(latencies)
        print(f"Latensi /recommend (in-process): mean={lat.mean():.1f}ms  "
              f"p50={np.percentile(lat,50):.1f}ms  p95={np.percentile(lat,95):.1f}ms")

    details_df = pd.DataFrame(rows)
    details_df.to_csv(args.output, index=False, encoding="utf-8-sig")
    print(f"\nDetail per-item disimpan: {args.output}")


if __name__ == "__main__":
    main()