Upload app.py

app.py

@@ -217,6 +217,7 @@ def ensure_db_ready():
             time.sleep(1)
             load_vectordb.clear()
             load_all_chunks.clear()
+            load_bm25.clear()
             st.rerun()
         except Exception as e:
             st.error(f"❌ فشل البناء: {e}")
@@ -285,8 +286,8 @@ def hybrid_search(query: str, k: int = TOP_K_SEARCH) -> tuple:
     # Category-Aware Search (Soft Boosting)
     qn_low = qn.lower()
     target_cat = None
-    if any(k in qn_low for k in ["مناظرة", "مناظرات", "concours"]): target_cat = "concours_ar" if detect_lang(qn)=="ar" else "concours_fr"
-    elif any(k in qn_low for k in ["تكوين", "تكوينات", "formation"]): target_cat = "formation_ar" if detect_lang(qn)=="ar" else "formation_fr"
+    if any(kw in qn_low for kw in ["مناظرة", "مناظرات", "concours"]): target_cat = "concours_ar" if detect_lang(qn)=="ar" else "concours_fr"
+    elif any(kw in qn_low for kw in ["تكوين", "تكوينات", "formation"]): target_cat = "formation_ar" if detect_lang(qn)=="ar" else "formation_fr"
 
     results = []
     for t in ranked:
@@ -310,21 +311,29 @@ def expand_query(q: str):
 
 # Ordered: more specific triggers first (substring match on question)
 PAGE_URL_TRIGGERS = [
-    (PAGE_URLS["ar_concours_superieur"], ["concours-entree-cycle-superieur-ar", "المرحلة العليا", "cycle superieur", "cycle-superieur"]),
-    (PAGE_URLS["concours_superieur"], ["le-concours-dentree-au-cycle-superieur", "cycle-superieur"]),
-    (PAGE_URLS["ar_concours_a2"], ["entree-au-cycle-de-formation-des-cadres-moyens-ar", "cadres moyens", " أ2", " a2"]),
-    (PAGE_URLS["concours_a2"], ["cadres-moyens-de-la-sous-categorie-a2", "sous-categorie a2"]),
-    (PAGE_URLS["ar_concours_a3"], ["agents-categorie-a3-ar", " أ3", " a3"]),
-    (PAGE_URLS["concours_a3"], ["agents-de-la-sous-categorie-a3", "sous-categorie a3"]),
-    (PAGE_URLS["ar_concours_general"], ["informations-generales-ar"]),
-    (PAGE_URLS["concours_general"], ["informations-generales"]),
-    (PAGE_URLS["formation_continue"], ["formation-continue-a-distance", "formation continue", "تكوين مستمر"]),
+    # Arabic triggers — specific Arabic keywords only
+    (PAGE_URLS["ar_concours_superieur"], ["المرحلة العليا", "مناظرة عليا", "concours-entree-cycle-superieur-ar"]),
+    (PAGE_URLS["ar_concours_a2"],        ["أ2", "entree-au-cycle-de-formation-des-cadres-moyens-ar"]),
+    (PAGE_URLS["ar_concours_a3"],        ["أ3", "agents-categorie-a3-ar"]),
+    (PAGE_URLS["ar_concours_general"],   ["معلومات عامة", "informations-generales-ar"]),
+    # French triggers — specific French keywords only
+    (PAGE_URLS["concours_superieur"],    ["cycle supérieur", "cycle-superieur", "le-concours-dentree-au-cycle-superieur"]),
+    (PAGE_URLS["concours_a2"],           ["cadres-moyens-de-la-sous-categorie-a2", "sous-categorie a2"]),
+    (PAGE_URLS["concours_a3"],           ["agents-de-la-sous-categorie-a3", "sous-categorie a3"]),
+    (PAGE_URLS["concours_general"],      ["informations-generales"]),
+    (PAGE_URLS["formation_continue"],    ["formation-continue-a-distance", "formation continue", "تكوين مستمر"]),
 ]
 
 def get_best_url(question: str):
     q = question.lower()
+    lang = detect_lang(question)
     for url, keys in PAGE_URL_TRIGGERS:
         if any(k in q for k in keys):
+            # If question is Arabic, skip French-only URLs and vice versa
+            if lang == "ar" and "/fr/" in url:
+                continue
+            if lang == "fr" and "/ar/" in url:
+                continue
             return url
     return None
 
@@ -353,7 +362,7 @@ def rerank_results(query: str, results: list):
         raw = float(s)
         conf = sigmoid(raw)
         bonus = 0.05 if any(kw in r["content"] for kw in KWS) else 0.0
-        scored.append({**r, "raw_score": raw, "confidence": conf+bonus})
+        scored.append({**r, "raw_score": raw, "confidence": min(1.0, conf+bonus)})
     
     return sorted(scored, key=lambda x:x["confidence"], reverse=True)[:TOP_K_RERANK], t_ms
 
@@ -375,7 +384,9 @@ def build_conversation_summary(messages: list, max_chars: int = 1200) -> str:
     parts = []
     for m in messages[-8:]:
         role = "user" if m.get("role") == "user" else "assistant"
-        parts.append(f"{role}: {m['content'][:400]}")
+        # Strip HTML tags before summarizing
+        content = re.sub(r"<[^>]+>", "", m['content'])
+        parts.append(f"{role}: {content[:400]}")
     summary = "\n".join(parts)
     return summary[:max_chars]