Upload 2 files

app.py

@@ -277,22 +277,37 @@ def hybrid_search(query: str, k: int = TOP_K_SEARCH) -> tuple:
     brank   = {chunks[i]: j for j, i in enumerate(btop) if i < len(chunks)}
     tb      = time.time()-t1
 
-    texts  = set(vrank)|set(brank)
-    # إذا لا توجد نتائج على الإطلاق → أرجع vector results فقط
-    if not texts:
-        results = [{"content":d.page_content,"meta":d.metadata,"rrf_score":1.0,"snippet":d.page_content[:120].replace("\n"," ")+"..."} for d in vdocs]
-        return results, {"vector_ms":round(tv*1000),"bm25_ms":0}
-
-    fused  = {t: vw/(vrank.get(t,k+10)+RRF_K)+bw/(brank.get(t,k+10)+RRF_K) for t in texts}
-    ranked = sorted(fused, key=fused.get, reverse=True)[:k]
-
-    tmeta = {d.page_content: d.metadata for d in vdocs}
-    for i in btop:
-        if i < len(chunks) and chunks[i] not in tmeta:
-            tmeta[chunks[i]] = metas[i]
-
-    results = [{"content":t,"meta":tmeta.get(t,{}),"rrf_score":fused[t],"snippet":t[:120].replace("\n"," ")+"..."} for t in ranked]
-    return results, {"vector_ms":round(tv*1000),"bm25_ms":round(tb*1000)}
+    # 🎯 Category-Aware Search (Soft Boosting)
+    qn_low = qn.lower()
+    target_cat = None
+    if any(k in qn_low for k in ["مناظرة", "مناظرات", "concours"]):
+        target_cat = "concours_ar" if detect_lang(qn) == "ar" else "concours_fr"
+    elif any(k in qn_low for k in ["تكوين", "تكوينات", "formation"]):
+        target_cat = "formation_ar" if detect_lang(qn) == "ar" else "formation_fr"
+    elif any(k in qn_low for k in ["حوكمة", "gouvernance"]):
+        target_cat = "gouvernance"
+    elif any(k in qn_low for k in ["خبر", "أخبار", "actualites"]):
+        target_cat = "news"
+
+    results = []
+    for t in ranked:
+        meta = tmeta.get(t, {})
+        score = fused[t]
+        
+        # Soft boost targets
+        if target_cat and meta.get("category") == target_cat:
+            score *= 1.5
+            
+        results.append({
+            "content": t,
+            "meta": meta,
+            "rrf_score": score,
+            "snippet": t[:120].replace("\n", " ") + "..."
+        })
+    
+    # Re-sort by boosted score
+    results = sorted(results, key=lambda x: x["rrf_score"], reverse=True)
+    return results, {"vector_ms": round(tv * 1000), "bm25_ms": round(tb * 1000)}
 
 # ══════════════════════════════════════════════════════════
 # 🔎 QUERY EXPANSION
@@ -682,7 +697,7 @@ with st.sidebar:
         st.markdown("---")
     # DB status — آمن هنا خارج cache_resource
     chunks_count = len(load_all_chunks(vectordb)[0])
-    if chunks_count < 1500:
+    if chunks_count < 2200:
         st.warning("⏳ جاري بناء قاعدة البيانات لأول مرة على السيرفر... يرجى الانتظار دقيقة/دقيقتين...")
         try:
             # Auto-run build process seamlessly
@@ -694,8 +709,16 @@ with st.sidebar:
             ts = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=80, separators=["\n\n", "\n", ".", " "])
             all_chunks = []
             for page in pages:
+                # Use category from JSON or infer it
+                cat = page.get("category", "other")
                 for chunk in ts.split_text(page["content"]):
-                    all_chunks.append({"page_name": page["page_name"], "url": page["url"], "langue": page["langue"], "content": chunk})
+                    all_chunks.append({
+                        "page_name": page["page_name"], 
+                        "url": page["url"], 
+                        "langue": page["langue"], 
+                        "category": cat,
+                        "content": chunk
+                    })
             cclient = _chromadb.PersistentClient(path=CHROMA_PATH)
             try: cclient.delete_collection("ena_collection")
             except: pass
@@ -703,7 +726,15 @@ with st.sidebar:
             BATCH = 50
             for i in range(0, len(all_chunks), BATCH):
                 b = all_chunks[i:i+BATCH]
-                temp_db.add_texts(texts=[c["content"] for c in b], metadatas=[{"page_name": c["page_name"], "url": c["url"], "langue": c["langue"]} for c in b])
+                temp_db.add_texts(
+                    texts=[c["content"] for c in b], 
+                    metadatas=[{
+                        "page_name": c["page_name"], 
+                        "url": c["url"], 
+                        "langue": c["langue"],
+                        "category": c["category"]
+                    } for c in b]
+                )
             
             # Clear caches and rerun to load fresh data
             load_vectordb.clear()

scraper.py

@@ -14,14 +14,36 @@ import json, requests, time
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urlparse
 
-# ──────────────────────────────
-# 🌐 URLs البداية لكل لغة
-# ──────────────────────────────
 BASE_URLS = [
     "https://www.ena.tn/ar/",
     "https://www.ena.tn/fr/"
 ]
 
+# ──────────────────────────────
+# 🏷️ التصنيف الذكي والـ Blacklist
+# ──────────────────────────────
+CATEGORIES = {
+    "/concours/":    "concours_fr",
+    "/formation/":   "formation_fr",
+    "/gouvernance/": "gouvernance",
+    "/ar/concours":  "concours_ar",
+    "/ar/formation": "formation_ar",
+    "/ar/service":   "service_ar", # خدمات إدارية
+    "/actualites/":  "news",
+}
+
+BLACKLIST = [
+    "/login/", "/connexion/", "/reset-password/", "/mot-de-passe-oublie/",
+    "/my-account/", "/search/", "/cart/", "/checkout/",
+    "wp-login", "admin-ajax", "/feed/"
+]
+
+def get_category(url):
+    for pattern, cat in CATEGORIES.items():
+        if pattern in url.lower():
+            return cat
+    return "other"
+
 visited = set()
 all_data = []
 
@@ -48,7 +70,9 @@ def is_valid_url(url):
     parsed = urlparse(url)
     if "ena.tn" not in parsed.netloc:
         return False
-    if any(url.endswith(ext) for ext in [".pdf",".jpg",".png",".zip"]):
+    if any(url.endswith(ext) for ext in [".pdf",".jpg",".png",".zip",".docx",".xlsx"]):
+        return False
+    if any(b in url.lower() for b in BLACKLIST):
         return False
     return True
 
@@ -129,11 +153,13 @@ def crawl(url, depth=0, max_depth=3):
             content = clean_page(soup)
 
         if len(content) > 200:
+            category = get_category(url)
             all_data.append({
                 "page_name": url.split("/")[-2] if "/" in url else "page",
                 "url": url,
                 "source": "ena.tn",
                 "langue": "ar" if "/ar/" in url else "fr",
+                "category": category,
                 "content": content,
                 "chars": len(content)
             })