feat: implement similarity engine with embedding generation, hybrid ranking, and explainability modules

Data/database/sql_connector.py

@@ -53,7 +53,7 @@ def load_preprocessed_projects():
     try:
         query = """
         SELECT *
-        FROM PreProcessed_Projects
+        FROM preprocess
         """
 
         with engine.connect() as conn:

models/faiss_index.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:228ea7b7be11e828d482432107d6c3baab192ea7018906f35da2282841f5a628
-size 783405
+oid sha256:c18f77517ed17a25d9517bf89cd1393284ea02aa84261759cb305268cc744907
+size 770090

models/metadata.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:534c0c0a94e7082f80d28af3e3cc8228a3749078b95eb161cdc86b26b6afc293
-size 786625
+oid sha256:2fc6969f2e7119f7fbb5b1c4c7b6823f628e1a7d892d391c7ea3f965eeedbaee
+size 784587

models/project_embeddings.npy

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:481172fb4b1bcfa7886c170cae254bd7b4af7077265d330826f693fcfcdcb077
-size 783488
+oid sha256:1039fecbfb1791061f41d99cc727e93bd8bc0f43bc6272d5a1bb42c603859820
+size 768128

src/similarity_model/__init__.py

@@ -38,6 +38,11 @@ from .similarity_engine import (
     find_similar_projects
 )
 
+from .explainability import (
+    generate_explanation
+)
+
 from .sync_projects import (
     sync_projects
 )
+

src/similarity_model/embedding_engine.py

@@ -65,6 +65,7 @@ class ProjectEmbedder:
         ).tolist()
 
         embeddings = self.generate_embeddings(rich_texts)
+        self.embeddings = embeddings
         dim = embeddings.shape[1]
         
         base_index = faiss.IndexFlatIP(dim)
@@ -79,6 +80,8 @@ class ProjectEmbedder:
         path.mkdir(parents=True, exist_ok=True)
         faiss.write_index(self.index, str(path / "faiss_index.bin"))
         self.metadata.to_parquet(path / "metadata.parquet", index=False)
+        if hasattr(self, "embeddings"):
+            np.save(str(path / "project_embeddings.npy"), self.embeddings)
         logger.info(f"Artifacts saved to {folder}")
 
     def load_artifacts(self, folder: str = "models"):

src/similarity_model/evaluation.py

@@ -165,16 +165,22 @@ def final_status(
     )
 
     if final_score >= 0.90:
-        print("EXCELLENT ✅")
+        print("EXCELLENT [OK]")
 
     elif final_score >= 0.75:
-        print("VERY GOOD ✅")
+        print("VERY GOOD [OK]")
 
     elif final_score >= 0.60:
-        print("GOOD ⚠️")
+        print("GOOD [WARN]")
 
     else:
-        print("NEEDS IMPROVEMENT ❌")
+        print("NEEDS IMPROVEMENT [FAIL]")
 
     print("Overall Score:", round(final_score, 4))
     print("==============================\n")
+
+if __name__ == "__main__":
+    self_score = run_self_test()
+    query_score = run_real_queries()
+    final_status(self_score, query_score)
+

src/similarity_model/explainability.py

@@ -0,0 +1,44 @@
+def generate_explanation(originality_score: float, similarity_results, query_features) -> str:
+    explanation = []
+    explanation.append(f"Originality Score: {originality_score:.2f}%")
+    
+    if originality_score < 40.0:
+        explanation.append("Classification: LOW ORIGINALITY (High Duplicate Risk)\n")
+    elif originality_score < 75.0:
+        explanation.append("Classification: MEDIUM ORIGINALITY (Moderate Similarity)\n")
+    else:
+        explanation.append("Classification: HIGH ORIGINALITY (Novel Idea)\n")
+        
+    explanation.append("Top Similar Projects in Database:")
+    
+    # Loop over matched results
+    for idx, row in similarity_results.iterrows():
+        title = row.get("project_title", "Unknown Title")
+        score = row.get("hybrid_score", 0.0) * 100
+        explanation.append(f"- \"{title}\" (Similarity Score: {score:.1f}%)")
+        
+    explanation.append("\nConcept Matching Analysis:")
+    # Highlight matched features of the best candidate
+    if len(similarity_results) > 0:
+        best_row = similarity_results.iloc[0]
+        matched_feats = best_row.get("matched_features", [])
+        unique_feats = best_row.get("unique_query_features", [])
+        
+        explanation.append("Matched Concepts (Shared with database):")
+        if matched_feats:
+            for item in matched_feats:
+                if isinstance(item, dict) and 'feature_a' in item and 'feature_b' in item:
+                    explanation.append(f"  * \"{item['feature_a']}\" matched with \"{item['feature_b']}\"")
+                else:
+                    explanation.append(f"  * \"{item}\"")
+        else:
+            explanation.append("  * No shared concepts found.")
+            
+        explanation.append("\nUnique Concepts (Exclusive to this project):")
+        if unique_feats:
+            for item in unique_feats:
+                explanation.append(f"  * \"{item}\"")
+        else:
+            explanation.append("  * No unique concepts identified.")
+            
+    return "\n".join(explanation)

src/similarity_model/hybrid_ranker.py

@@ -40,29 +40,13 @@ def get_dynamic_weights(
     """
     Adaptive weights depending on feature richness.
     Returns (semantic_w, feature_w, coverage_w) — always sum to 1.0
-
-    NOTE: When coverage==0 (DB project has no stored features),
-    always fall back to the high-semantic-weight path so that
-    a rich multi-feature query is not penalised vs a sparse one.
     """
-
-    # No feature evidence at all — rely on semantic regardless of query richness
-    # Use 0.50 weight (not 0.70) to compensate for all-mpnet-base-v2's higher
-    # within-domain baseline scores; prevents same-domain/different-purpose
-    # projects from scoring too high when no features overlap.
-    if coverage == 0:
-        return 0.50, 0.30, 0.20
-
-    # Rich features + high overlap → trust features heavily
-    if feature_count >= 5 and coverage >= 0.60:
-        return 0.40, 0.45, 0.15
-
-    # Sparse query features
-    if feature_count <= 2:
-        return 0.70, 0.20, 0.10
-
-    # Balanced
-    return 0.55, 0.35, 0.10
+    if feature_count < 5:
+        # Sparse Feature Set configuration
+        return 0.75, 0.15, 0.10
+    else:
+        # Rich Feature Set configuration
+        return 0.50, 0.35, 0.15
 
 def compute_hybrid_score(
     semantic_score,
@@ -120,6 +104,40 @@ def compute_hybrid_score(
 
     return round(clamp(score), 4)
 
+import numpy as np
+import random
+
+_dynamic_baseline_similarity = None
+
+def get_baseline_similarity():
+    global _dynamic_baseline_similarity
+    if _dynamic_baseline_similarity is None:
+        try:
+            from src.similarity_model.semantic_search import load_embeddings
+            embeddings = load_embeddings()
+            n = len(embeddings)
+            if n > 1:
+                # Sample random pairs
+                num_samples = min(2000, n * (n - 1) // 2)
+                pairs = set()
+                while len(pairs) < num_samples:
+                    i = random.randint(0, n - 1)
+                    j = random.randint(0, n - 1)
+                    if i != j:
+                        pairs.add((min(i, j), max(i, j)))
+                
+                similarities = []
+                for i, j in pairs:
+                    # Dot product since vectors are normalized
+                    sim = float(np.dot(embeddings[i], embeddings[j]))
+                    similarities.append(sim)
+                _dynamic_baseline_similarity = float(np.median(similarities))
+            else:
+                _dynamic_baseline_similarity = 0.25
+        except Exception:
+            _dynamic_baseline_similarity = 0.25
+    return _dynamic_baseline_similarity
+
 def compute_originality(
     hybrid_score: float,
     unique_query_features: int,
@@ -127,21 +145,16 @@ def compute_originality(
 ) -> float:
     """
     Originality Score (0-100).
-    Base: (1 - hybrid_score) * 100.
-
-    Uniqueness bonus (+up to 10 pts) is only applied when the DB project
-    actually has stored features (total_query_features > 0 AND there were
-    real feature matches to compare against). While DB features are empty,
-    unique_query_features == total_query_features always, making the bonus
-    a constant +10 noise term — so we skip it until the parquet is rebuilt.
+    Base: (1 - calibrated_similarity) * 100.
     """
     hybrid_score = clamp(hybrid_score)
-
-    originality = 100.0 * (1.0 - hybrid_score)
+    baseline_sim = get_baseline_similarity()
+    
+    # Subtraction and Min-Max scaling
+    calibrated_similarity = max(0.0, (hybrid_score - baseline_sim) / (1.0 - baseline_sim))
+    originality = 100.0 * (1.0 - calibrated_similarity)
 
     # Only apply uniqueness bonus when feature comparison was meaningful
-    # (i.e. the candidate had stored features, so coverage > 0)
-    # Guarded by unique < total to avoid the always-1.0 ratio when DB is empty
     if total_query_features > 0 and unique_query_features < total_query_features:
         uniqueness_ratio = unique_query_features / total_query_features
         originality = min(100.0, originality + (uniqueness_ratio * 10.0))

src/similarity_model/preprocessing.py

@@ -74,19 +74,58 @@ def _get_yake_extractor():
     logger.info("Initializing YAKE NLP feature extractor")
     return yake.KeywordExtractor(lan="en", n=3, dedupLim=0.9, top=20, features=None)
 
+import json
+
+_feature_db_frequencies = None
+
+def load_feature_frequencies_cache():
+    global _feature_db_frequencies
+    if _feature_db_frequencies is None:
+        try:
+            from src.similarity_model.semantic_search import load_metadata
+            df = load_metadata()
+            from collections import Counter
+            counter = Counter()
+            total_docs = len(df)
+            if total_docs > 0:
+                for feats in df["features"]:
+                    if isinstance(feats, str):
+                        try:
+                            feats = json.loads(feats)
+                        except:
+                            feats = []
+                    if isinstance(feats, list):
+                        seen = set(str(f).strip().lower() for f in feats)
+                        for f in seen:
+                            if f:
+                                counter[f] += 1
+                _feature_db_frequencies = {k: v / total_docs for k, v in counter.items()}
+            else:
+                _feature_db_frequencies = {}
+        except Exception:
+            _feature_db_frequencies = {}
+    return _feature_db_frequencies
+
 def extract_features(text: str) -> list:
     """
     Extracts detailed, multi-word phrases generated purely by YAKE.
+    Filters out highly generic features appearing in > 15% of indexed projects.
     """
     matched = []
     try:
         kw_extractor = _get_yake_extractor()
         yake_results = kw_extractor.extract_keywords(text)
         
+        freq_cache = load_feature_frequencies_cache()
+        max_df_threshold = 0.15  # Filter if keyword appears in > 15% of database
+        
         for kw, score in yake_results:
             kw_clean = str(kw).strip().lower()
             if len(kw_clean.split()) > 1 and kw_clean not in matched:
-                matched.append(kw_clean)
+                # Apply IDF filter check
+                doc_freq = freq_cache.get(kw_clean, 0.0)
+                if doc_freq <= max_df_threshold:
+                    matched.append(kw_clean)
                 
     except Exception as e:
         logger.error(f"YAKE extraction failed: {e}")

src/similarity_model/similarity_engine.py

@@ -389,8 +389,12 @@ def find_similar_projects(
             "No strong similar projects found."
         }])
 
+    import numpy as np
+    from src.similarity_model.explainability import generate_explanation
+
     final_df = pd.DataFrame(rows)
 
+    # Sort results
     final_df = final_df.sort_values(
         by=[
             "hybrid_score",
@@ -398,15 +402,47 @@ def find_similar_projects(
             "semantic_score"
         ],
         ascending=False
-    ).head(top_k).reset_index(drop=True)
+    ).reset_index(drop=True)
+
+    # Decaying aggregation over Top-5
+    K_val = min(5, len(final_df))
+    if K_val > 0:
+        s1 = float(final_df.loc[0, "hybrid_score"])
+        density_penalty = 0.0
+        beta = 0.05
+        lam = 0.5
+        for i in range(1, K_val):
+            si = float(final_df.loc[i, "hybrid_score"])
+            density_penalty += np.exp(-lam * i) * si
+        
+        aggregated_score = min(1.0, s1 + beta * density_penalty)
+        
+        # Recalculate originality based on aggregated similarity score
+        top_row = final_df.iloc[0]
+        unique_q_feats = top_row.get("unique_query_features", [])
+        total_q_feats = len(query_project[FEATURE_COL])
+        
+        aggregated_originality = compute_originality(
+            hybrid_score=aggregated_score,
+            unique_query_features=len(unique_q_feats),
+            total_query_features=total_q_feats
+        )
+        if aggregated_score >= 0.90:
+            aggregated_originality = 0.0
+            
+        final_df.loc[0, "originality_score"] = aggregated_originality
+    else:
+        aggregated_score = 0.0
 
+    # Retain the top match
+    final_df = final_df.head(top_k).copy()
     
+    # Overwrite originality score for any project with >= 0.90 hybrid score to 0.0
     final_df.loc[
         final_df["hybrid_score"] >= 0.90,
         "originality_score"
     ] = 0.0
 
-    
     final_df["query_features_used"] = [
         final_features
     ] * len(final_df)
@@ -415,4 +451,11 @@ def find_similar_projects(
         query_text
     ] * len(final_df)
 
+    # Generate explanation
+    if len(final_df) > 0:
+        overall_orig = float(final_df.loc[0, "originality_score"])
+        explanation_text = generate_explanation(overall_orig, final_df, final_features)
+        final_df["explanation"] = explanation_text
+
     return final_df
+

src/similarity_model/sync_preprocess.py

@@ -0,0 +1,247 @@
+import json
+import logging
+import os
+import sys
+import pandas as pd
+import numpy as np
+from pathlib import Path
+from sklearn.metrics.pairwise import cosine_similarity
+
+# Ensure workspace root is in path
+sys.path.append(str(Path(__file__).resolve().parents[2]))
+
+from Data.database.sql_connector import engine
+from src.similarity_model.preprocessing import preprocess_dataset, normalize_text
+from src.similarity_model.semantic_search import load_model
+from src.similarity_model.feature_similarity import load_feature_model, compute_feature_similarity
+from src.similarity_model.hybrid_ranker import compute_hybrid_score, compute_originality
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+def run_sync_preprocess():
+    logger.info("Initializing Sync and Preprocess Service...")
+    
+    # 1. Verify DB Connection
+    try:
+        with engine.connect() as conn:
+            logger.info("Database connection verified successfully.")
+    except Exception as exc:
+        logger.error(f"Unable to connect to the SQL database. Error: {exc}")
+        sys.exit(1)
+
+    # 2. Fetch raw projects
+    logger.info("Pulling active records from 'Projects' table...")
+    projects_query = """
+    SELECT *
+    FROM Projects
+    WHERE Status IN (
+        'Completed',
+        'UnderReview',
+        'In_Progress'
+    )
+    """
+    with engine.connect() as conn:
+        raw_df = pd.read_sql(projects_query, conn)
+    logger.info(f"Loaded {len(raw_df)} active projects from 'Projects' table.")
+
+    # 3. Preprocess dataset
+    logger.info("Preprocessing dataset...")
+    processed_df = preprocess_dataset(raw_df)
+    logger.info(f"Total projects after preprocessing filters: {len(processed_df)}")
+
+    if len(processed_df) == 0:
+        logger.warning("No projects left after preprocessing. Exiting.")
+        return
+
+    # 4. Standarize / clean columns to match structure in the screenshot
+    cols_to_keep = [
+        "id",
+        "submittedat",
+        "project_title",
+        "studentnames",
+        "year",
+        "abstract",
+        "description",
+        "problemstatement",
+        "proposedsolution",
+        "objectives",
+        "full_content",
+        "clean_text",
+        "word_count",
+        "features"
+    ]
+
+    for col in cols_to_keep:
+        if col not in processed_df.columns:
+            processed_df[col] = ""
+
+    processed_df = processed_df[cols_to_keep]
+
+    processed_df = processed_df.rename(
+        columns={
+            "submittedat": "submitted_at",
+            "studentnames": "student_names",
+            "problemstatement": "problem_statement",
+            "proposedsolution": "proposed_solution"
+        }
+    )
+
+    # 5. Calculate Originality for each project
+    logger.info("Calculating originality score for each project...")
+    
+    # Load models
+    model = load_model()
+    feature_model = load_feature_model()
+
+    # Generate embeddings for all preprocessed projects
+    rich_texts = []
+    for idx, row in processed_df.iterrows():
+        title = str(row["project_title"]).strip()
+        abstract = str(row["abstract"]).strip()
+        description = str(row["description"]).strip()
+        feats = row["features"] if isinstance(row["features"], list) else []
+        
+        raw_text = f"{title}. {abstract}. {description}"
+        feature_text = " ".join(feats)
+        full_text = normalize_text(f"{raw_text}. {feature_text}")
+        rich_texts.append(full_text)
+
+    logger.info("Encoding projects to vector space...")
+    embeddings = model.encode(
+        rich_texts,
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+        show_progress_bar=True
+    ).astype("float32")
+
+    # Compute cosine similarity matrix
+    logger.info("Computing semantic similarity matrix...")
+    sim_matrix = cosine_similarity(embeddings, embeddings)
+
+    originality_scores = []
+
+    # Loop through each project to find closest match and compute originality
+    for i in range(len(processed_df)):
+        current_project = processed_df.iloc[i]
+        current_features = current_project["features"] if isinstance(current_project["features"], list) else []
+        
+        # Get semantic similarity scores against other projects
+        scores = sim_matrix[i].copy()
+        scores[i] = -1.0  # Exclude self-matching
+        
+        # Sort and pick top 50 semantic matches as candidates
+        top_indices = np.argsort(scores)[::-1][:50]
+        
+        max_hybrid_score = 0.0
+        best_candidate_features = []
+        best_candidate_idx = -1
+        
+        for idx in top_indices:
+            candidate_project = processed_df.iloc[idx]
+            sem_score = float(scores[idx])
+            
+            candidate_features = candidate_project["features"] if isinstance(candidate_project["features"], list) else []
+            
+            # Compute feature similarity
+            feat_result = compute_feature_similarity(
+                current_features,
+                candidate_features,
+                model=feature_model
+            )
+            
+            feature_score = feat_result["score"]
+            coverage = feat_result["coverage"]
+            
+            query_feature_count = len(current_features)
+            unique_query_count = len(feat_result["unique_a"])
+            
+            # Compute hybrid score
+            hybrid_score = compute_hybrid_score(
+                semantic_score=sem_score,
+                feature_score=feature_score,
+                coverage=coverage,
+                feature_count=query_feature_count,
+                unique_query_count=unique_query_count
+            )
+            
+            # Direct copy-paste detection
+            query_desc = (str(current_project.get("abstract", "")) + " " + str(current_project.get("description", ""))).strip()
+            candidate_desc = (str(candidate_project.get("abstract", "")) + " " + str(candidate_project.get("description", ""))).strip()
+            
+            words_q = set(normalize_text(query_desc).split())
+            words_c = set(normalize_text(candidate_desc).split())
+            
+            jaccard_overlap = 0.0
+            if words_q and words_c:
+                jaccard_overlap = len(words_q.intersection(words_c)) / len(words_q.union(words_c))
+                
+            if jaccard_overlap >= 0.60:
+                hybrid_score = 0.95
+                
+            if hybrid_score > max_hybrid_score:
+                max_hybrid_score = hybrid_score
+                best_candidate_features = candidate_features
+                best_candidate_idx = idx
+
+        # Calculate originality based on the worst-case (highest similarity) match
+        if best_candidate_idx != -1 and len(current_features) > 0:
+            # Recompute feature similarity for the best candidate to get unique_a count
+            feat_result = compute_feature_similarity(
+                current_features,
+                best_candidate_features,
+                model=feature_model
+            )
+            unique_query_count = len(feat_result["unique_a"])
+            orig_score = compute_originality(
+                hybrid_score=max_hybrid_score,
+                unique_query_features=unique_query_count,
+                total_query_features=len(current_features)
+            )
+        else:
+            orig_score = compute_originality(
+                hybrid_score=max_hybrid_score,
+                unique_query_features=0,
+                total_query_features=0
+            )
+            
+        originality_scores.append(orig_score)
+
+    processed_df["originality"] = originality_scores
+    logger.info("Finished calculating originality scores.")
+
+    # 6. Save locally to preprocessed.csv in Data/processed/
+    local_dir = Path("Data") / "processed"
+    local_dir.mkdir(parents=True, exist_ok=True)
+    local_path = local_dir / "preprocessed.csv"
+    
+    # Keep features as readable lists for the CSV file
+    csv_df = processed_df.copy()
+    csv_df.to_csv(local_path, index=False)
+    logger.info(f"Successfully saved preprocessed projects locally to: {local_path}")
+
+    # 7. Convert features list to JSON string for database push
+    db_df = processed_df.copy()
+    db_df["features"] = db_df["features"].apply(json.dumps)
+
+    # 8. Push to table 'preprocess' in database
+    logger.info("Uploading preprocessed records to database table 'preprocess'...")
+    try:
+        with engine.begin() as conn:
+            # Drop/replace table if exists to write the new preprocessed structure
+            db_df.to_sql(
+                "preprocess",
+                conn,
+                if_exists="replace",
+                index=False
+            )
+        logger.info("Successfully pushed all preprocessed projects to database table 'preprocess'.")
+    except Exception as exc:
+        logger.error(f"Failed to push table to database. Error: {exc}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    run_sync_preprocess()