Upload checker/external/local_search.py with huggingface_hub

checker/external/local_search.py

@@ -163,7 +163,7 @@ def load_cached_embeddings(articles):
     return cache["embeddings"], cache["titles"].tolist()
 
 
-def find_related_articles(user_text, threshold=0.55, top_n=10):
+def find_related_articles(user_text, threshold=0.55, top_n=10, detected_lang="en"):
     """Find articles related to user's input using hybrid matching.
 
     Uses cached embeddings for speed — only the user input is encoded live.
@@ -172,14 +172,27 @@ def find_related_articles(user_text, threshold=0.55, top_n=10):
     1. Semantic similarity (multilingual MiniLM) — meaning-based
     2. Keyword overlap — catches proper nouns (names, places) across languages
 
-    Minimum semantic similarity floor: articles must have at least 40% semantic
+    Minimum semantic similarity floor: articles must have at least some semantic
     similarity regardless of keyword overlap, preventing keyword-inflated
     false matches (e.g., an article about '40 years' matching because of
     shared generic keywords like 'years' or 'president').
+
+    For non-English (Cebuano/Tagalog) articles, both the floor and combined
+    threshold are lowered because MiniLM cross-lingual similarity between an
+    English-translated query and a native-language title is naturally lower
+    than English↔English comparison.
     """
     # Minimum semantic similarity — regardless of keyword overlap, the article
     # must actually be ABOUT the same topic at a basic meaning level.
-    SEMANTIC_FLOOR = 0.40
+    # Lower floor for non-English: cross-lingual embedding similarity is
+    # naturally reduced when comparing an English translation against
+    # Cebuano/Tagalog titles stored in the DB.
+    if detected_lang not in ("en",):
+        SEMANTIC_FLOOR = 0.30   # was 0.40
+        effective_threshold = max(threshold - 0.15, 0.40)  # e.g. 0.55 → 0.40
+    else:
+        SEMANTIC_FLOOR = 0.40
+        effective_threshold = threshold
 
     model = get_model()
 
@@ -242,7 +255,7 @@ def find_related_articles(user_text, threshold=0.55, top_n=10):
 
         combined_score = (semantic_score * 0.6) + (keyword_score * 0.4)
 
-        if combined_score >= threshold:
+        if combined_score >= effective_threshold:
             results.append(
                 {
                     "title": article["title"],