| r"""Evaluation harness for the supervisor recommendation system. |
| |
| Computes ranking metrics (Hit@k, Precision@k, Recall@k, MRR, nDCG@k, MAP) over a |
| ground-truth set of (research_title -> actual_supervisor) pairs, in-process |
| (loads artifacts once, then loops recommend_top_k). |
| |
| Ground-truth sources: |
| --input <csv> CSV with a title column and an actual-supervisor column. |
| Configure with --title-col / --actual-col. |
| --from-publications Build an eval set from data/publications.csv using |
| judul_publikasi as the query title and dosen as the |
| actual supervisor(s). This is a PROXY (publications are |
| not thesis supervisions) and is not leave-one-out, so |
| treat the numbers as an upper-bound sanity check. |
| |
| Usage (run from backend/ with the project venv): |
| ..\.venv\Scripts\python.exe scripts\evaluate_recommendations.py --from-publications --sample 300 |
| ..\.venv\Scripts\python.exe scripts\evaluate_recommendations.py --input eval.csv --title-col judul --actual-col dosen |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import os |
| import sys |
| import time |
|
|
| import numpy as np |
| import pandas as pd |
|
|
| BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| if BACKEND_DIR not in sys.path: |
| sys.path.insert(0, BACKEND_DIR) |
|
|
| from services.artifacts import load_artifacts |
| from services.evaluation import ( |
| average_precision, |
| hit_at_k, |
| ndcg_at_k, |
| normalize_name, |
| precision_at_k, |
| recall_at_k, |
| reciprocal_rank, |
| split_actual_names, |
| ) |
| from services.recommender import recommend_top_k |
|
|
|
|
| def build_eval_set(args) -> pd.DataFrame: |
| if args.from_publications: |
| path = args.publications or os.path.join(BACKEND_DIR, "data", "publications.csv") |
| df = pd.read_csv(path) |
| title_col = "judul_publikasi" if "judul_publikasi" in df.columns else args.title_col |
| actual_col = "dosen" if "dosen" in df.columns else args.actual_col |
| else: |
| if not args.input: |
| raise SystemExit("Sediakan --input <csv> atau gunakan --from-publications") |
| df = pd.read_csv(args.input) |
| title_col, actual_col = args.title_col, args.actual_col |
|
|
| for col in (title_col, actual_col): |
| if col not in df.columns: |
| raise SystemExit(f"Kolom '{col}' tidak ada. Kolom tersedia: {list(df.columns)}") |
|
|
| out = df[[title_col, actual_col]].rename(columns={title_col: "title", actual_col: "actual"}) |
| out = out.dropna(subset=["title", "actual"]) |
| out = out[out["title"].astype(str).str.strip().str.len() >= 3] |
| out = out[out["actual"].astype(str).str.strip().str.len() >= 1].reset_index(drop=True) |
|
|
| if args.sample and args.sample < len(out): |
| out = out.sample(n=args.sample, random_state=args.seed).reset_index(drop=True) |
| return out |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| parser.add_argument("--input", default="") |
| parser.add_argument("--from-publications", action="store_true") |
| parser.add_argument("--publications", default="") |
| parser.add_argument("--title-col", default="title") |
| parser.add_argument("--actual-col", default="actual") |
| parser.add_argument("--ks", default="1,3,5,10") |
| parser.add_argument("--sample", type=int, default=0) |
| parser.add_argument("--seed", type=int, default=42) |
| parser.add_argument("--artifacts-dir", default=os.path.join(BACKEND_DIR, "models")) |
| parser.add_argument("--data-dir", default=os.path.join(BACKEND_DIR, "data")) |
| parser.add_argument("--output", default=os.path.join(BACKEND_DIR, "scripts", "eval_details.csv")) |
| args = parser.parse_args() |
|
|
| ks = sorted({int(x) for x in args.ks.split(",") if x.strip()}) |
| max_k = max(ks) |
|
|
| eval_df = build_eval_set(args) |
| print(f"Eval set: {len(eval_df)} item | k = {ks}") |
|
|
| |
| os.environ.pop("ARTIFACTS_MODE", None) |
| print("Loading artifacts...") |
| store = load_artifacts(artifacts_dir=args.artifacts_dir, data_dir=args.data_dir) |
|
|
| |
| agg = {k: {"hit": 0.0, "p": 0.0, "r": 0.0, "ndcg": 0.0, "ap": 0.0} for k in ks} |
| mrr_sum = 0.0 |
| rows = [] |
| latencies = [] |
| n_empty = 0 |
|
|
| for i, row in eval_df.iterrows(): |
| title = str(row["title"]) |
| actual = set(split_actual_names(row["actual"])) |
|
|
| t0 = time.perf_counter() |
| rec = recommend_top_k(store=store, title=title, top_k=max_k) |
| latencies.append((time.perf_counter() - t0) * 1000.0) |
|
|
| ranked = [normalize_name(r["nama_dosen"]) for r in rec.get("recommendations", [])] |
| if not ranked: |
| n_empty += 1 |
|
|
| rr = reciprocal_rank(ranked, actual) |
| mrr_sum += rr |
|
|
| detail = { |
| "title": title[:120], |
| "actual": "; ".join(sorted(actual)), |
| "predicted_top": "; ".join(ranked[:max_k]), |
| "detected_lang": rec.get("detected_lang"), |
| "closest_topic_id": rec.get("closest_topic_id"), |
| "rr": round(rr, 4), |
| } |
| for k in ks: |
| agg[k]["hit"] += hit_at_k(ranked, actual, k) |
| agg[k]["p"] += precision_at_k(ranked, actual, k) |
| agg[k]["r"] += recall_at_k(ranked, actual, k) |
| agg[k]["ndcg"] += ndcg_at_k(ranked, actual, k) |
| agg[k]["ap"] += average_precision(ranked, actual, k) |
| detail[f"hit@{k}"] = int(hit_at_k(ranked, actual, k)) |
| rows.append(detail) |
|
|
| if (i + 1) % 50 == 0: |
| print(f" {i + 1}/{len(eval_df)} processed...") |
|
|
| n = len(eval_df) |
| denom = float(n) if n else 1.0 |
|
|
| print("\n" + "=" * 78) |
| print(f"HASIL EVALUASI (n={n}, empty_recs={n_empty})") |
| print("=" * 78) |
| header = f"{'k':>3} | {'Hit@k':>7} | {'Prec@k':>7} | {'Recall@k':>8} | {'nDCG@k':>7} | {'MAP@k':>7}" |
| print(header) |
| print("-" * len(header)) |
| for k in ks: |
| a = agg[k] |
| print(f"{k:>3} | {a['hit']/denom:>7.4f} | {a['p']/denom:>7.4f} | " |
| f"{a['r']/denom:>8.4f} | {a['ndcg']/denom:>7.4f} | {a['ap']/denom:>7.4f}") |
| print("-" * len(header)) |
| print(f"MRR (global): {mrr_sum/denom:.4f}") |
|
|
| if latencies: |
| lat = np.array(latencies) |
| print(f"Latensi /recommend (in-process): mean={lat.mean():.1f}ms " |
| f"p50={np.percentile(lat,50):.1f}ms p95={np.percentile(lat,95):.1f}ms") |
|
|
| details_df = pd.DataFrame(rows) |
| details_df.to_csv(args.output, index=False, encoding="utf-8-sig") |
| print(f"\nDetail per-item disimpan: {args.output}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
