Update rag.py

rag.py

@@ -34,10 +34,32 @@ USER_AGENT = os.getenv(
     "IndoHoaxDetector-EvidenceBot/1.0 (+kontak: isi-email-anda@domain.com)",
 )
 
+# Sumber evidence kedua: API resmi MAFINDO (https://mafindodocs.netlify.app/v2/),
+# dipakai berdampingan dengan scraping TurnBackHoax di atas. Nonaktif secara default.
+MAFINDO_API_ENABLED   = os.getenv("MAFINDO_API_ENABLED", "0") == "1"
+MAFINDO_API_KEY       = os.getenv("MAFINDO_API_KEY", "")
+MAFINDO_API_BASE_URL  = os.getenv(
+    "MAFINDO_API_BASE_URL", "https://yudistira.turnbackhoax.id/api"
+).rstrip("/")
+MAFINDO_API_TIMEOUT   = float(os.getenv("MAFINDO_API_TIMEOUT", "10"))
+# Jumlah kandidat yang diminta ke endpoint search MAFINDO sebelum di-ranking ulang
+# pakai cosine similarity yang sama dengan index scraping (biar sebanding).
+MAFINDO_API_LIMIT     = int(os.getenv("MAFINDO_API_LIMIT", "10"))
+# /antihoax/search method=content kemungkinan besar pencarian berbasis kata kunci
+# (bukan semantik), jadi query yang dikirim dipotong dulu supaya peluang match lebih baik.
+MAFINDO_API_QUERY_MAX_CHARS = int(os.getenv("MAFINDO_API_QUERY_MAX_CHARS", "150"))
+
+# Strategi menggabungkan 2 sumber evidence saat keduanya aktif:
+#   "fallback" (default): coba API dulu, kalau API nonaktif/gagal/0 hasil relevan -> pakai scraping.
+#   "union"             : selalu panggil dua-duanya, gabungkan & dedup hasilnya berdasarkan similarity.
+RAG_SOURCE_MODE = os.getenv("RAG_SOURCE_MODE", "fallback").strip().lower()
+
 
 _rag_ready: bool = False
 _last_fetch_ok: Optional[bool] = None
 _last_fetch_error: Optional[str] = None
+_mafindo_warned_no_key: bool = False
+_last_mafindo_error: Optional[str] = None
 
 
 # Kata pertama pada slug URL artikel (mis. /articles/123-salah-...) menentukan verdict.
@@ -291,27 +313,92 @@ def start_rag_background_update(
     threading.Thread(target=_loop, daemon=True, name="rag-updater").start()
 
 
-def retrieve_evidence(
-    query_text: str,
-    embed_fn: Callable[[List[str]], np.ndarray],
-    top_k: Optional[int] = None,
-    threshold: Optional[float] = None,
-) -> List[Dict[str, Any]]:
-    if not RAG_ENABLED or not _rag_ready or not query_text:
+def _fetch_evidence_from_api(query_text: str) -> Optional[List[Dict[str, Any]]]:
+    """
+    Ambil kandidat evidence dari API resmi MAFINDO (POST /antihoax/search, method=content).
+    Return None = API nonaktif/gagal total (sinyal ke pemanggil utk fallback ke scraping).
+    Return []   = API berhasil dipanggil tapi tidak ada artikel yang cocok.
+    """
+    global _mafindo_warned_no_key, _last_mafindo_error
+
+    if not MAFINDO_API_ENABLED:
+        return None
+
+    if not MAFINDO_API_KEY:
+        if not _mafindo_warned_no_key:
+            print("[WARN][RAG] MAFINDO_API_ENABLED=1 tapi MAFINDO_API_KEY kosong, API dilewati.")
+            _mafindo_warned_no_key = True
+        return None
+
+    query_value = str(query_text or "").strip()[:MAFINDO_API_QUERY_MAX_CHARS]
+    if not query_value:
         return []
-    top_k = top_k or RAG_TOP_K
-    threshold = RAG_SIM_THRESHOLD if threshold is None else threshold
 
+    try:
+        resp = requests.post(
+            f"{MAFINDO_API_BASE_URL}/antihoax/search",
+            data={
+                "key": MAFINDO_API_KEY,
+                "method": "content",
+                "value": query_value,
+                "limit": MAFINDO_API_LIMIT,
+            },
+            headers={"Accept": "application/json"},
+            timeout=MAFINDO_API_TIMEOUT,
+        )
+        if resp.status_code != 200:
+            _last_mafindo_error = f"HTTP {resp.status_code}"
+            print(f"[WARN][RAG] MAFINDO API status {resp.status_code}, lanjut tanpa hasil API.")
+            return None
+        payload = resp.json()
+    except Exception as e:
+        _last_mafindo_error = str(e)
+        print(f"[WARN][RAG] MAFINDO API gagal: {e}")
+        return None
+
+    if not isinstance(payload, list):
+        # Key salah / parameter salah biasanya direspons sbg dict/string, bukan list artikel.
+        _last_mafindo_error = f"Format tak terduga: {str(payload)[:150]}"
+        print(f"[WARN][RAG] MAFINDO API format tak terduga, dianggap 0 hasil: {str(payload)[:150]}")
+        return []
+
+    _last_mafindo_error = None
+    articles: List[Dict[str, Any]] = []
+    for item in payload:
+        if not isinstance(item, dict):
+            continue
+        news_id = str(item.get("id", "")).strip()
+        title = str(item.get("title", "")).strip()
+        if not news_id or not title:
+            continue
+        status = str(item.get("status", "")).strip()
+        verdict = "hoax" if status == "2" else "fakta" if status == "1" else "tidak diketahui"
+        url = str(item.get("references") or item.get("source_link") or "").strip()
+        excerpt_src = item.get("conclusion") or item.get("fact") or item.get("content") or ""
+        articles.append({
+            "id": f"mafindo-{news_id}",
+            "title": title,
+            "excerpt": str(excerpt_src).strip()[:300],
+            "url": url,
+            "date": str(item.get("tanggal", "")).strip(),
+            "verdict": verdict,
+            "source": "api",
+        })
+    return articles
+
+
+def _retrieve_from_local_index(
+    q_emb: np.ndarray,
+    top_k: int,
+    threshold: float,
+) -> List[Dict[str, Any]]:
+    """Cari evidence dari index lokal hasil scraping TurnBackHoax (embeddings di _store)."""
+    if not RAG_ENABLED or not _rag_ready:
+        return []
     with _store.lock:
         if _store.embeddings is None or not _store.metadata:
             return []
-        try:
-            q_emb = embed_fn([query_text])[0]
-        except Exception as e:
-            print(f"[WARN][RAG] Gagal encode query: {e}")
-            return []
-
-        sims = _store.embeddings @ np.asarray(q_emb, dtype="float32")
+        sims = _store.embeddings @ q_emb
         top_idx = np.argsort(-sims)[:top_k]
 
         results = []
@@ -327,10 +414,86 @@ def retrieve_evidence(
                 "date": meta.get("date", ""),
                 "similarity": round(score, 4),
                 "excerpt": meta.get("excerpt", ""),
+                "source": "scrape",
             })
         return results
 
 
+def _rank_candidates_by_similarity(
+    articles: List[Dict[str, Any]],
+    q_emb: np.ndarray,
+    embed_fn: Callable[[List[str]], np.ndarray],
+    threshold: float,
+) -> List[Dict[str, Any]]:
+    """Beri skor cosine similarity ke kandidat artikel (mis. dari API) memakai embed_fn
+    yang sama dengan index scraping, supaya keduanya sebanding & bisa di-threshold sama."""
+    if not articles:
+        return []
+    texts = [f"{a.get('title', '')}. {a.get('excerpt', '')}".strip() for a in articles]
+    try:
+        embs = np.asarray(embed_fn(texts), dtype="float32")
+    except Exception as e:
+        print(f"[WARN][RAG] Gagal encode kandidat evidence API: {e}")
+        return []
+
+    ranked = []
+    for art, emb in zip(articles, embs):
+        score = float(emb @ q_emb)
+        if score < threshold:
+            continue
+        ranked.append({
+            "title": art.get("title", ""),
+            "url": art.get("url", ""),
+            "verdict": art.get("verdict", "tidak diketahui"),
+            "date": art.get("date", ""),
+            "similarity": round(score, 4),
+            "excerpt": art.get("excerpt", ""),
+            "source": art.get("source", "api"),
+        })
+    return ranked
+
+
+def retrieve_evidence(
+    query_text: str,
+    embed_fn: Callable[[List[str]], np.ndarray],
+    top_k: Optional[int] = None,
+    threshold: Optional[float] = None,
+) -> List[Dict[str, Any]]:
+    if not query_text or (not RAG_ENABLED and not MAFINDO_API_ENABLED):
+        return []
+    top_k = top_k or RAG_TOP_K
+    threshold = RAG_SIM_THRESHOLD if threshold is None else threshold
+
+    try:
+        q_emb = np.asarray(embed_fn([query_text])[0], dtype="float32")
+    except Exception as e:
+        print(f"[WARN][RAG] Gagal encode query: {e}")
+        return []
+
+    api_ranked: Optional[List[Dict[str, Any]]] = None
+    if MAFINDO_API_ENABLED:
+        api_articles = _fetch_evidence_from_api(query_text)
+        if api_articles is not None:
+            api_ranked = _rank_candidates_by_similarity(api_articles, q_emb, embed_fn, threshold)
+
+    if RAG_SOURCE_MODE == "union":
+        scrape_ranked = _retrieve_from_local_index(q_emb, top_k, threshold)
+        combined = (api_ranked or []) + scrape_ranked
+        dedup: Dict[str, Dict[str, Any]] = {}
+        for item in combined:
+            key = (item.get("url") or item.get("title") or "").strip().lower()
+            if not key:
+                continue
+            if key not in dedup or item["similarity"] > dedup[key]["similarity"]:
+                dedup[key] = item
+        return sorted(dedup.values(), key=lambda x: -x["similarity"])[:top_k]
+
+    # Mode default "fallback": pakai hasil API kalau ada & relevan, kalau tidak coba scraping.
+    if api_ranked:
+        return sorted(api_ranked, key=lambda x: -x["similarity"])[:top_k]
+    return _retrieve_from_local_index(q_emb, top_k, threshold)
+
+
 def rag_status() -> Dict[str, Any]:
     return {
         "enabled": RAG_ENABLED,
@@ -339,4 +502,8 @@ def rag_status() -> Dict[str, Any]:
         "last_updated": _store.last_updated,
         "last_fetch_ok": _last_fetch_ok,
         "last_fetch_error": _last_fetch_error,
+        "source_mode": RAG_SOURCE_MODE,
+        "mafindo_api_enabled": MAFINDO_API_ENABLED,
+        "mafindo_api_ready": MAFINDO_API_ENABLED and bool(MAFINDO_API_KEY),
+        "last_mafindo_error": _last_mafindo_error,
     }