Update: hybrid_ranker.py

src/similarity_model/hybrid_ranker.py

@@ -1,327 +1,327 @@
-import logging
-from typing import List, Dict, Any
-
-import pandas as pd
-
-from src.similarity_model import (
-    compute_feature_similarity,
-    load_feature_model,
-    safe_feature_list
-)
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s | %(levelname)s | %(message)s"
-)
-logger = logging.getLogger(__name__)
-
-TITLE_COL = "project_title"
-FEATURE_COL = "features"
-
-DEFAULT_SEMANTIC_WEIGHT = 0.65
-DEFAULT_FEATURE_WEIGHT = 0.35
-
-HIGH_FEATURE_WEIGHT = 0.45
-LOW_FEATURE_WEIGHT = 0.20
-
-BONUS_WEIGHT = 0.05
-MIN_HYBRID_SCORE = 0.05
-
-def clamp(value: float) -> float:
-    """
-    Keep score inside [0,1]
-    """
-    return max(0.0, min(1.0, float(value)))
-
-def get_dynamic_weights(
-    feature_count: int,
-    coverage: float
-):
-    """
-    Adaptive weights depending on feature richness.
-    Returns (semantic_w, feature_w, coverage_w) — always sum to 1.0
-
-    NOTE: When coverage==0 (DB project has no stored features),
-    always fall back to the high-semantic-weight path so that
-    a rich multi-feature query is not penalised vs a sparse one.
-    """
-
-    # No feature evidence at all — rely on semantic regardless of query richness
-    # Use 0.50 weight (not 0.70) to compensate for all-mpnet-base-v2's higher
-    # within-domain baseline scores; prevents same-domain/different-purpose
-    # projects from scoring too high when no features overlap.
-    if coverage == 0:
-        return 0.50, 0.30, 0.20
-
-    # Rich features + high overlap → trust features heavily
-    if feature_count >= 5 and coverage >= 0.60:
-        return 0.40, 0.45, 0.15
-
-    # Sparse query features
-    if feature_count <= 2:
-        return 0.70, 0.20, 0.10
-
-    # Balanced
-    return 0.55, 0.35, 0.10
-
-def compute_hybrid_score(
-    semantic_score,
-    feature_score,
-    coverage,
-    feature_count,
-    unique_query_count=0
-) -> float:
-
-    semantic_score = clamp(semantic_score)
-    feature_score  = clamp(feature_score)
-    coverage       = clamp(coverage)
-
-    shared_count = feature_count - unique_query_count
-
-    # Hard duplicate: most features shared, almost nothing unique
-    if shared_count >= 6 and unique_query_count <= 1:
-        return 0.95
-
-    # Near-total feature coverage with high similarity
-    if coverage >= 0.90 and feature_score >= 0.65:
-        return round(
-            clamp(
-                0.75 +
-                (0.15 * feature_score) +
-                (0.10 * semantic_score)
-            ),
-            4
-        )
-
-    # Get adaptive weights regardless of feature availability
-    semantic_w, feature_w, coverage_w = get_dynamic_weights(
-        feature_count, coverage
-    )
-
-    score = (
-        semantic_w * semantic_score +
-        feature_w  * feature_score +
-        coverage_w * coverage
-    )
-
-    # Only penalise low feature score when features actually exist to compare.
-    # Skipping this when feature_score==0 prevents an unfair 50% penalty
-    # on DB projects that have no stored features.
-    if feature_score > 0 and feature_score < 0.40:
-        penalty = 0.50 + (feature_score / 0.40) * 0.50
-        score *= penalty
-
-    return round(clamp(score), 4)
-
-def compute_originality(
-    hybrid_score: float,
-    unique_query_features: int,
-    total_query_features: int
-) -> float:
-    """
-    Originality Score (0-100).
-    Base: (1 - hybrid_score) * 100.
-
-    Uniqueness bonus (+up to 10 pts) is only applied when the DB project
-    actually has stored features (total_query_features > 0 AND there were
-    real feature matches to compare against). While DB features are empty,
-    unique_query_features == total_query_features always, making the bonus
-    a constant +10 noise term — so we skip it until the parquet is rebuilt.
-    """
-    hybrid_score = clamp(hybrid_score)
-
-    originality = 100.0 * (1.0 - hybrid_score)
-
-    # Only apply uniqueness bonus when feature comparison was meaningful
-    # (i.e. the candidate had stored features, so coverage > 0)
-    # Guarded by unique < total to avoid the always-1.0 ratio when DB is empty
-    if total_query_features > 0 and unique_query_features < total_query_features:
-        uniqueness_ratio = unique_query_features / total_query_features
-        originality = min(100.0, originality + (uniqueness_ratio * 10.0))
-
-    return round(max(0.0, min(100.0, originality)), 2)
-
-def compute_confidence(
-    semantic_score: float,
-    feature_score: float,
-    coverage: float
-) -> float:
-    """
-    Trust score for ranking result.
-    """
-
-    confidence = (
-        0.40 * clamp(semantic_score) +
-        0.40 * clamp(feature_score) +
-        0.20 * clamp(coverage)
-    )
-
-    return round(clamp(confidence), 4)
-
-def risk_label(score: float) -> str:
-    """
-    Duplicate risk label.
-    """
-
-    if score >= 0.90:
-        return "Very High"
-
-    if score >= 0.75:
-        return "High"
-
-    if score >= 0.55:
-        return "Medium"
-
-    if score >= 0.35:
-        return "Low"
-
-    return "Very Low"
-
-def compare_single_candidate(
-    query_row: Dict[str, Any],
-    candidate_row: Dict[str, Any],
-    semantic_score: float,
-    model=None
-) -> Dict[str, Any]:
-
-    if model is None:
-        model = load_feature_model()
-
-    query_features = safe_feature_list(
-        query_row.get(FEATURE_COL, [])
-    )
-
-    candidate_features = safe_feature_list(
-        candidate_row.get(FEATURE_COL, [])
-    )
-
-    feature_result = compute_feature_similarity(
-        query_features,
-        candidate_features,
-        model=model
-    )
-
-    feature_score = feature_result["score"]
-    coverage = feature_result["coverage"]
-
-    total_query_features = len(query_features)
-    unique_query_count = len(
-        feature_result["unique_a"]
-    )
-
-    base_similarity = compute_hybrid_score(
-        semantic_score=semantic_score,
-        feature_score=feature_score,
-        coverage=coverage,
-        feature_count=total_query_features,
-        unique_query_count=unique_query_count
-    )
-
-    originality_score = compute_originality(
-        hybrid_score=base_similarity,
-        unique_query_features=unique_query_count,
-        total_query_features=total_query_features
-    )
-
-    hybrid_score = base_similarity
-
-    confidence_score = compute_confidence(
-        semantic_score=semantic_score,
-        feature_score=feature_score,
-        coverage=coverage
-    )
-    print("=" * 50)
-    print("BASE SIMILARITY:", base_similarity)
-    print("ORIGINALITY:", originality_score)
-    print("FINAL SIMILARITY:", hybrid_score)
-    print("=" * 50)
-    return {
-        "project_title":
-            candidate_row.get(TITLE_COL, ""),
-
-        "semantic_score":
-            round(float(semantic_score), 4),
-
-        "feature_score":
-            feature_score,
-
-        "coverage":
-            coverage,
-
-        "hybrid_score":
-            hybrid_score,
-
-        "originality_score":
-            originality_score,
-
-        "confidence_score":
-            confidence_score,
-
-        "duplicate_risk":
-            risk_label(hybrid_score),
-
-        "shared_features_count":
-            feature_result["shared_count"],
-
-        "matches":
-            feature_result["matches"],
-
-        "unique_query_features":
-            feature_result["unique_a"],
-
-        "unique_candidate_features":
-            feature_result["unique_b"],
-
-        "candidate_features":
-            candidate_features
-    }
-
-def rank_candidates(
-    query_row,
-    candidate_rows,
-    semantic_scores: List[float],
-    model=None,
-    min_score: float = MIN_HYBRID_SCORE
-) -> pd.DataFrame:
-    """
-    Re-rank semantic retrieval results
-    using hybrid logic.
-    """
-
-    if model is None:
-        model = load_feature_model()
-
-    rows = []
-
-    for candidate_row, sem_score in zip(
-        candidate_rows,
-        semantic_scores
-    ):
-
-        result = compare_single_candidate(
-            query_row=query_row,
-            candidate_row=candidate_row,
-            semantic_score=float(sem_score),
-            model=model
-        )
-
-        if result["hybrid_score"] >= min_score:
-            rows.append(result)
-
-    if not rows:
-        return pd.DataFrame([{
-            "message": "No strong similar projects found."
-        }])
-
-    results = pd.DataFrame(rows)
-
-    results = results.sort_values(
-        by=[
-            "hybrid_score",
-            "confidence_score",
-            "semantic_score"
-        ],
-        ascending=False
-    ).reset_index(drop=True)
-
-    return results
+import logging
+from typing import List, Dict, Any
+
+import pandas as pd
+
+from src.similarity_model import (
+    compute_feature_similarity,
+    load_feature_model,
+    safe_feature_list
+)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+TITLE_COL = "project_title"
+FEATURE_COL = "features"
+
+DEFAULT_SEMANTIC_WEIGHT = 0.65
+DEFAULT_FEATURE_WEIGHT = 0.35
+
+HIGH_FEATURE_WEIGHT = 0.45
+LOW_FEATURE_WEIGHT = 0.20
+
+BONUS_WEIGHT = 0.05
+MIN_HYBRID_SCORE = 0.05
+
+def clamp(value: float) -> float:
+    """
+    Keep score inside [0,1]
+    """
+    return max(0.0, min(1.0, float(value)))
+
+def get_dynamic_weights(
+    feature_count: int,
+    coverage: float
+):
+    """
+    Adaptive weights depending on feature richness.
+    Returns (semantic_w, feature_w, coverage_w) — always sum to 1.0
+
+    NOTE: When coverage==0 (DB project has no stored features),
+    always fall back to the high-semantic-weight path so that
+    a rich multi-feature query is not penalised vs a sparse one.
+    """
+
+    # No feature evidence at all — rely on semantic regardless of query richness
+    # Use 0.50 weight (not 0.70) to compensate for all-mpnet-base-v2's higher
+    # within-domain baseline scores; prevents same-domain/different-purpose
+    # projects from scoring too high when no features overlap.
+    if coverage == 0:
+        return 0.50, 0.30, 0.20
+
+    # Rich features + high overlap → trust features heavily
+    if feature_count >= 5 and coverage >= 0.60:
+        return 0.40, 0.45, 0.15
+
+    # Sparse query features
+    if feature_count <= 2:
+        return 0.70, 0.20, 0.10
+
+    # Balanced
+    return 0.55, 0.35, 0.10
+
+def compute_hybrid_score(
+    semantic_score,
+    feature_score,
+    coverage,
+    feature_count,
+    unique_query_count=0
+) -> float:
+
+    semantic_score = clamp(semantic_score)
+    feature_score  = clamp(feature_score)
+    coverage       = clamp(coverage)
+
+    shared_count = feature_count - unique_query_count
+
+    # Hard duplicate: most features shared, almost nothing unique
+    if shared_count >= 6 and unique_query_count <= 1:
+        return 0.95
+
+    # Near-total feature coverage with high similarity
+    if coverage >= 0.90 and feature_score >= 0.65:
+        return round(
+            clamp(
+                0.75 +
+                (0.15 * feature_score) +
+                (0.10 * semantic_score)
+            ),
+            4
+        )
+
+    # Get adaptive weights regardless of feature availability
+    semantic_w, feature_w, coverage_w = get_dynamic_weights(
+        feature_count, coverage
+    )
+
+    score = (
+        semantic_w * semantic_score +
+        feature_w  * feature_score +
+        coverage_w * coverage
+    )
+
+    # Only penalise low feature score when features actually exist to compare.
+    # Skipping this when feature_score==0 prevents an unfair 50% penalty
+    # on DB projects that have no stored features.
+    if feature_score > 0 and feature_score < 0.40:
+        penalty = 0.50 + (feature_score / 0.40) * 0.50
+        score *= penalty
+
+    return round(clamp(score), 4)
+
+def compute_originality(
+    hybrid_score: float,
+    unique_query_features: int,
+    total_query_features: int
+) -> float:
+    """
+    Originality Score (0-100).
+    Base: (1 - hybrid_score) * 100.
+
+    Uniqueness bonus (+up to 10 pts) is only applied when the DB project
+    actually has stored features (total_query_features > 0 AND there were
+    real feature matches to compare against). While DB features are empty,
+    unique_query_features == total_query_features always, making the bonus
+    a constant +10 noise term — so we skip it until the parquet is rebuilt.
+    """
+    hybrid_score = clamp(hybrid_score)
+
+    originality = 100.0 * (1.0 - hybrid_score)
+
+    # Only apply uniqueness bonus when feature comparison was meaningful
+    # (i.e. the candidate had stored features, so coverage > 0)
+    # Guarded by unique < total to avoid the always-1.0 ratio when DB is empty
+    if total_query_features > 0 and unique_query_features < total_query_features:
+        uniqueness_ratio = unique_query_features / total_query_features
+        originality = min(100.0, originality + (uniqueness_ratio * 10.0))
+
+    return round(max(0.0, min(100.0, originality)), 2)
+
+def compute_confidence(
+    semantic_score: float,
+    feature_score: float,
+    coverage: float
+) -> float:
+    """
+    Trust score for ranking result.
+    """
+
+    confidence = (
+        0.40 * clamp(semantic_score) +
+        0.40 * clamp(feature_score) +
+        0.20 * clamp(coverage)
+    )
+
+    return round(clamp(confidence), 4)
+
+def risk_label(score: float) -> str:
+    """
+    Duplicate risk label.
+    """
+
+    if score >= 0.90:
+        return "Very High"
+
+    if score >= 0.75:
+        return "High"
+
+    if score >= 0.55:
+        return "Medium"
+
+    if score >= 0.35:
+        return "Low"
+
+    return "Very Low"
+
+def compare_single_candidate(
+    query_row: Dict[str, Any],
+    candidate_row: Dict[str, Any],
+    semantic_score: float,
+    model=None
+) -> Dict[str, Any]:
+
+    if model is None:
+        model = load_feature_model()
+
+    query_features = safe_feature_list(
+        query_row.get(FEATURE_COL, [])
+    )
+
+    candidate_features = safe_feature_list(
+        candidate_row.get(FEATURE_COL, [])
+    )
+
+    feature_result = compute_feature_similarity(
+        query_features,
+        candidate_features,
+        model=model
+    )
+
+    feature_score = feature_result["score"]
+    coverage = feature_result["coverage"]
+
+    total_query_features = len(query_features)
+    unique_query_count = len(
+        feature_result["unique_a"]
+    )
+
+    base_similarity = compute_hybrid_score(
+        semantic_score=semantic_score,
+        feature_score=feature_score,
+        coverage=coverage,
+        feature_count=total_query_features,
+        unique_query_count=unique_query_count
+    )
+
+    originality_score = compute_originality(
+        hybrid_score=base_similarity,
+        unique_query_features=unique_query_count,
+        total_query_features=total_query_features
+    )
+
+    hybrid_score = base_similarity
+
+    confidence_score = compute_confidence(
+        semantic_score=semantic_score,
+        feature_score=feature_score,
+        coverage=coverage
+    )
+    print("=" * 50)
+    print("BASE SIMILARITY:", base_similarity)
+    print("ORIGINALITY:", originality_score)
+    print("FINAL SIMILARITY:", hybrid_score)
+    print("=" * 50)
+    return {
+        "project_title":
+            candidate_row.get(TITLE_COL, ""),
+
+        "semantic_score":
+            round(float(semantic_score), 4),
+
+        "feature_score":
+            feature_score,
+
+        "coverage":
+            coverage,
+
+        "hybrid_score":
+            hybrid_score,
+
+        "originality_score":
+            originality_score,
+
+        "confidence_score":
+            confidence_score,
+
+        "duplicate_risk":
+            risk_label(hybrid_score),
+
+        "shared_features_count":
+            feature_result["shared_count"],
+
+        "matches":
+            feature_result["matches"],
+
+        "unique_query_features":
+            feature_result["unique_a"],
+
+        "unique_candidate_features":
+            feature_result["unique_b"],
+
+        "candidate_features":
+            candidate_features
+    }
+
+def rank_candidates(
+    query_row,
+    candidate_rows,
+    semantic_scores: List[float],
+    model=None,
+    min_score: float = MIN_HYBRID_SCORE
+) -> pd.DataFrame:
+    """
+    Re-rank semantic retrieval results
+    using hybrid logic.
+    """
+
+    if model is None:
+        model = load_feature_model()
+
+    rows = []
+
+    for candidate_row, sem_score in zip(
+        candidate_rows,
+        semantic_scores
+    ):
+
+        result = compare_single_candidate(
+            query_row=query_row,
+            candidate_row=candidate_row,
+            semantic_score=float(sem_score),
+            model=model
+        )
+
+        if result["hybrid_score"] >= min_score:
+            rows.append(result)
+
+    if not rows:
+        return pd.DataFrame([{
+            "message": "No strong similar projects found."
+        }])
+
+    results = pd.DataFrame(rows)
+
+    results = results.sort_values(
+        by=[
+            "hybrid_score",
+            "confidence_score",
+            "semantic_score"
+        ],
+        ascending=False
+    ).reset_index(drop=True)
+
+    return results