feat: implement modular hybrid similarity engine with semantic search and feature-based ranking components

src/similarity_model/feature_similarity.py

@@ -23,6 +23,8 @@ def safe_feature_list(features):
     Convert any feature input into clean List[str]
     """
     import numpy as np
+    import json
+    import ast
 
     if features is None:
         return []
@@ -37,15 +39,32 @@ def safe_feature_list(features):
         features = list(features)
 
     if isinstance(features, str):
+        features = features.strip()
+        parsed = None
+        # Try JSON parsing
         try:
-            import ast
-            parsed = ast.literal_eval(features)
-            if isinstance(parsed, list):
-                features = parsed
-            else:
-                features = [features]
+            parsed = json.loads(features)
+            if isinstance(parsed, str):
+                parsed = json.loads(parsed)
         except:
-            features = [features]
+            pass
+
+        # Try AST parsing as fallback
+        if not isinstance(parsed, list):
+            try:
+                parsed = ast.literal_eval(features)
+                if isinstance(parsed, str):
+                    parsed = ast.literal_eval(parsed)
+            except:
+                pass
+
+        if isinstance(parsed, list):
+            features = parsed
+        else:
+            if features:
+                features = [features]
+            else:
+                features = []
 
     if isinstance(features, list):
         cleaned = []

src/similarity_model/hybrid_ranker.py

@@ -90,7 +90,7 @@ def compute_hybrid_score(
     
     
     if feature_score == 0 or coverage == 0:
-        return round(clamp(0.60 * semantic_score), 4)
+        return round(clamp(0.85 * semantic_score), 4)
 
     
     
@@ -126,14 +126,7 @@ def compute_originality(
     """
     hybrid_score = clamp(hybrid_score)
 
-    if hybrid_score >= 0.80:
-        originality = 20.0 - ((hybrid_score - 0.80) / 0.20) * 20.0
-    elif hybrid_score >= 0.50:
-        originality = 50.0 - ((hybrid_score - 0.50) / 0.30) * 30.0
-    elif hybrid_score >= 0.30:
-        originality = 80.0 - ((hybrid_score - 0.30) / 0.20) * 30.0
-    else:
-        originality = 100.0 - (hybrid_score / 0.30) * 20.0
+    originality = 100.0 * (1.0 - hybrid_score)
 
     return round(max(0.0, min(100.0, originality)), 2)
 

src/similarity_model/semantic_search.py

@@ -106,9 +106,7 @@ def build_results(
         row = df.loc[idx]
 
         final_score = float(score)
-
         
-                
         if query_words:
 
             title_words = tokenize(row[TITLE_COL])

src/similarity_model/similarity_engine.py

@@ -230,7 +230,8 @@ def find_similar_projects(
     features: Optional[List[str]] = None,
     top_k: int = DEFAULT_TOP_K,
     search_pool: int = DEFAULT_SEARCH_POOL,
-    auto_extract_features: bool = True
+    auto_extract_features: bool = True,
+    exclude_self: bool = False
 ) -> pd.DataFrame:
     """
     Final Smart Pipeline
@@ -260,36 +261,37 @@ def find_similar_projects(
         "Preparing query..."
     )
 
-    existing_project = df[
-        df["project_title"].str.lower().str.strip()
-        ==
-        title.lower().strip()
-    ]
-
-    if len(existing_project) > 0:
-
-        logger.info("Exact title match found — returning originality = 0")
-
-        matched_row = existing_project.iloc[0]
-        stored_features = safe_feature_list(matched_row["features"])
-
-        return pd.DataFrame([{
-            "project_id":           int(matched_row.name),
-            "project_title":        matched_row["project_title"],
-            "semantic_score":       1.0,
-            "feature_score":        1.0,
-            "coverage":             1.0,
-            "hybrid_score":         1.0,
-            "originality_score":    0.0,
-            "confidence_score":     1.0,
-            "duplicate_risk":       "Very High",
-            "shared_features_count": len(stored_features),
-            "matched_features":     [{"feature_a": f, "feature_b": f, "score": 1.0} for f in stored_features],
-            "unique_query_features": [],
-            "unique_candidate_features": [],
-            "query_features_used":  stored_features,
-            "query_clean_text":     title
-        }])
+    if not exclude_self:
+        existing_project = df[
+            df["project_title"].str.lower().str.strip()
+            ==
+            title.lower().strip()
+        ]
+    
+        if len(existing_project) > 0:
+    
+            logger.info("Exact title match found — returning originality = 0")
+    
+            matched_row = existing_project.iloc[0]
+            stored_features = safe_feature_list(matched_row["features"])
+    
+            return pd.DataFrame([{
+                "project_id":           int(matched_row.name),
+                "project_title":        matched_row["project_title"],
+                "semantic_score":       1.0,
+                "feature_score":        1.0,
+                "coverage":             1.0,
+                "hybrid_score":         1.0,
+                "originality_score":    0.0,
+                "confidence_score":     1.0,
+                "duplicate_risk":       "Very High",
+                "shared_features_count": len(stored_features),
+                "matched_features":     [{"feature_a": f, "feature_b": f, "score": 1.0} for f in stored_features],
+                "unique_query_features": [],
+                "unique_candidate_features": [],
+                "query_features_used":  stored_features,
+                "query_clean_text":     title
+            }])
 
     logger.info("Extracting new features")
 
@@ -324,12 +326,17 @@ def find_similar_projects(
 
     semantic_results = search_by_text(
         query_text=query_text,
-        k=search_pool,
+        k=search_pool + (5 if exclude_self else 0),
         min_score=DEFAULT_MIN_SEMANTIC_SCORE
     )
 
     if "message" in semantic_results.columns:
         return semantic_results
+        
+    if exclude_self and title.strip():
+        semantic_results = semantic_results[
+            semantic_results["project_title"].str.lower().str.strip() != title.lower().strip()
+        ]