add keybert model in requirments.txt and pyproject.toml

pyproject.toml

@@ -14,8 +14,10 @@ dependencies = [
     "groq>=0.9.0",
     "greenlet==3.3.1",
     "httpx==0.26.0",
+    "keybert>=0.8.0",
     "openai-whisper==20250625",
     "passlib[bcrypt]==1.7.4",
+    "sentence-transformers>=3.0.0",
     "pydantic-core==2.41.5",
     "pydantic-settings==2.1.0",
     "pydantic[email]==2.12.5",

requirements.txt

@@ -39,8 +39,8 @@ firebase-admin==6.5.0
 dnspython
 
 # --- ML & Recommendations ---
-# keybert
-# sentence-transformers
+keybert>=0.8.0
+sentence-transformers>=3.0.0
 
 # --- Qwen Local Inference ---
 transformers>=4.37.0

src/recommendation/recommender.py

@@ -24,7 +24,7 @@ class RecommendationService:
     def __init__(self, api_key: Optional[str] = None):
         self.api_key = "AIzaSyA3erB-Lxd5SOoBOXaumOCVaEr3TcgYG60"
         self.youtube = build("youtube", "v3", developerKey=self.api_key)
-        self.groq_client = Groq(api_key="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")  # هنا
+        self.groq_client = Groq(api_key="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") 
 
     # ──────────────────────────────────────────────
     # Step 1: top 3 categories
@@ -36,17 +36,17 @@ class RecommendationService:
             cat = note.get("category")
             if not cat:
                 continue
-            cats = cat if isinstance(cat, list) else [cat]
-            for c in cats:
+            cats = cat if isinstance(cat, list) else [cat]  # sometimes category is a list(["AI","Technology"]), sometimes a string("AI")
+            for c in cats:  # loop through each category in the list
                 if c and c != "Uncategorized":
                     counter[c] += 1
 
         top = [cat for cat, _ in counter.most_common(top_n)]
-        logger.info(f"🏆 Top categories: {top}")
+        logger.info(f" Top categories: {top}")
         return top
 
     # ──────────────────────────────────────────────
-    # Step 2: keywords from latest note per category
+    # Step 2: keywords from latest 2notes per category
     # ──────────────────────────────────────────────
     def _latest_notes_per_category(
         self, notes: List[Dict], categories: List[str], top_n: int = 2
@@ -54,7 +54,7 @@ class RecommendationService:
         """
         return a dict mapping each category to its latest N notes, sorted by createdAt.
         """
-        buckets: Dict[str, List[Dict]] = {cat: [] for cat in categories}
+        buckets: Dict[str, List[Dict]] = {cat: [] for cat in categories} # buckets ={"AI": [note1,note2,..], "Technology": [], "Science": []}
 
         for note in notes:
             cat = note.get("category")
@@ -65,12 +65,17 @@ class RecommendationService:
 
         # sort each category's notes by createdAt and keep top N
         return {
-            cat: sorted(notes_list, key=lambda n: n.get("createdAt", 0), reverse=True)[:top_n]
+            cat: sorted(notes_list, 
+                        key=lambda n: n.get("createdAt", 0),  # sort by createdAt timestamp, default to 0 if missing
+                          reverse=True)[:top_n] # sort in descending order and take the top 2
             for cat, notes_list in buckets.items()
         }
-
+    
+    # ──────────────────────────────────────────────
+    # Step 3: extract keywords via grok(llama-3.3-70b-versatile)
+    # ──────────────────────────────────────────────
     async def _extract_keywords_with_claude(
-        self, notes: List[Dict], category: str  # ← List بدل Dict
+        self, notes: List[Dict], category: str  
     ) -> List[str]:
         
         # combine all relevant text fields from the notes into one string for context
@@ -88,7 +93,7 @@ class RecommendationService:
             f"extract 3 to 5 concise English keywords or short phrases that best "
             f"represent the core topic for a YouTube educational search. "
             f"Reply with ONLY a JSON array of strings, no explanation.\n\n"
-            f"Notes:\n{combined_content[:2000]}"  # ← زودي الحد شوية
+            f"Notes:\n{combined_content[:3000]}"  
         )
 
         try:
@@ -108,15 +113,15 @@ class RecommendationService:
             raw = re.sub(r"```json|```", "", raw).strip()
             keywords = json.loads(raw)
             if isinstance(keywords, list):
-                logger.info(f"🔑 Keywords for '{category}': {keywords}")
+                logger.info(f" Keywords for '{category}': {keywords}")
                 return [str(k) for k in keywords[:5]]
         except Exception as e:
-            logger.warning(f"⚠️ Claude keyword extraction failed for '{category}': {e}")
+            logger.warning(f" Grok keyword extraction failed for '{category}': {e}")
 
         return [category]  # fallback
 
     # ──────────────────────────────────────────────
-    # Step 3: build query & search YouTube
+    # Step 4: build query & search YouTube
     # ──────────────────────────────────────────────
     async def _build_search_query(
         self, category_keywords: Dict[str, List[str]]
@@ -129,7 +134,7 @@ class RecommendationService:
         for keywords in category_keywords.values():
             parts.extend(keywords[:2])
         query = " OR ".join(parts[:6])  # YouTube search works best under ~60 chars
-        logger.info(f"🔍 Final search query: {query}")
+        logger.info(f" Final search query: {query}")
         return query
 
     async def get_youtube_recommendations(
@@ -140,7 +145,7 @@ class RecommendationService:
             return []
 
         enhanced_query = f"{query} tutorial "
-        logger.info(f"🎬 Searching YouTube: {enhanced_query}")
+        logger.info(f" Searching YouTube: {enhanced_query}")
 
         try:
             loop = asyncio.get_event_loop()
@@ -176,11 +181,11 @@ class RecommendationService:
 
             random.shuffle(videos)
             result = videos[:limit]
-            logger.info(f"✅ Returning {len(result)} recommendations")
+            logger.info(f" Returning {len(result)} recommendations")
             return result
 
         except Exception as e:
-            logger.error(f"❌ YouTube search failed: {e}")
+            logger.error(f" YouTube search failed: {e}")
             return []
 
     # ──────────────────────────────────────────────
@@ -189,7 +194,7 @@ class RecommendationService:
     async def get_recommendations_for_user(
         self, db, user_id: str, limit: int = 5
     ) -> List[Dict]:
-        logger.info(f"📚 Fetching notes for user: {user_id}")
+        logger.info(f" Fetching notes for user: {user_id}")
 
         # ── Fetch notes ──────────────────────────
         try:
@@ -199,20 +204,20 @@ class RecommendationService:
                 .stream()
             )
             notes = [doc.to_dict() for doc in notes_docs]
-            logger.info(f"📝 Found {len(notes)} notes")
+            logger.info(f"Found {len(notes)} notes")
         except Exception as e:
-            logger.error(f"❌ Firebase fetch failed: {e}")
+            logger.error(f" Firebase fetch failed: {e}")
             notes = []
 
         if not notes:
-            logger.info("⚠️ No notes — falling back to general recommendations")
+            logger.info(" No notes — falling back to general recommendations")
             return await self.get_youtube_recommendations("educational tutorials", limit)
 
         # ── Step 1: top 3 categories ─────────────
         top_categories = self._get_top_categories(notes, top_n=3)
 
         if not top_categories:
-            logger.info("⚠️ No valid categories — falling back")
+            logger.info(" No valid categories — falling back")
             return await self.get_youtube_recommendations("educational tutorials", limit)
 
         # ── Step 2: keywords via Claude ──────────
@@ -232,7 +237,7 @@ class RecommendationService:
 
         category_keywords: Dict[str, List[str]] = {
             cat: kws
-            for cat, kws in zip(valid_categories, keyword_results)  # ✅ zip على نفس الليست
+            for cat, kws in zip(valid_categories, keyword_results) 
         }
         # ── Step 3: build query & recommend ──────
         all_videos = []
@@ -240,7 +245,7 @@ class RecommendationService:
         for category, keywords in category_keywords.items():
             query = " ".join(keywords[:3])
 
-            logger.info(f"🎯 Searching category: {category} | Query: {query}")
+            logger.info(f" Searching category: {category} | Query: {query}")
 
             videos = await self.get_youtube_recommendations(query, limit=2)