feat: implement similarity_engine with hybrid scoring and database-backed project comparison logic

src/similarity_model/similarity_engine.py

@@ -291,22 +291,67 @@ def find_similar_projects(
     )
 
     if not exclude_self:
+        # Check database table 'preprocess' directly to see if this project title already exists
+        db_match_found = False
+        db_project_id = None
+        db_features = []
+        db_abstract = ""
+        db_description = ""
+        matched_title = title
+        
+        try:
+            import json
+            from Data.database.sql_connector import engine
+            from sqlalchemy import text
+            
+            with engine.connect() as conn:
+                db_row = conn.execute(text("""
+                    SELECT id, project_title, features, abstract, description
+                    FROM preprocess
+                    WHERE LOWER(LTRIM(RTRIM(project_title))) = LOWER(:title)
+                """), {"title": title.strip()}).fetchone()
+                
+            if db_row:
+                db_match_found = True
+                db_project_id, matched_title, features_json, db_abstract, db_description = db_row
+                # Parse features
+                try:
+                    db_features = json.loads(features_json) if isinstance(features_json, str) else features_json
+                    if isinstance(db_features, str):
+                        db_features = json.loads(db_features)
+                except Exception:
+                    db_features = []
+                if not isinstance(db_features, list):
+                    db_features = []
+        except Exception as db_exc:
+            logger.warning(f"Direct database check of 'preprocess' table failed: {db_exc}")
+
+        # Check local/cached metadata dataframe as fallback
         existing_project = df[
             df["project_title"].str.lower().str.strip()
             ==
             title.lower().strip()
         ]
     
-        if len(existing_project) > 0:
-    
-            logger.info("Exact title match found — returning originality = 0")
-    
-            matched_row = existing_project.iloc[0]
-            stored_features = safe_feature_list(matched_row["features"])
-    
+        if len(existing_project) > 0 or db_match_found:
+            logger.info("Exact title match found in preprocess table — returning originality = 0")
+            
+            if len(existing_project) > 0:
+                matched_row = existing_project.iloc[0]
+                project_id = int(matched_row.name)
+                stored_features = safe_feature_list(matched_row["features"])
+                matched_title = matched_row["project_title"]
+                matched_abstract = matched_row.get("abstract", "")
+                matched_desc = matched_row.get("description", "")
+            else:
+                project_id = int(db_project_id)
+                stored_features = db_features
+                matched_abstract = db_abstract
+                matched_desc = db_description
+                
             return pd.DataFrame([{
-                "project_id":           int(matched_row.name),
-                "project_title":        matched_row["project_title"],
+                "project_id":           project_id,
+                "project_title":        matched_title,
                 "semantic_score":       1.0,
                 "feature_score":        1.0,
                 "coverage":             1.0,
@@ -319,7 +364,10 @@ def find_similar_projects(
                 "unique_query_features": [],
                 "unique_candidate_features": [],
                 "query_features_used":  stored_features,
-                "query_clean_text":     title
+                "query_clean_text":     title,
+                "candidate_features":   stored_features,
+                "abstract":             matched_abstract,
+                "description":          matched_desc
             }])
 
     logger.info("Extracting new features")