Spaces:

InnoTrack
/

Graduation_Project-v1.2

Runtime error

Graduation_Project-v1.2 / src /similarity_model /similarity_engine.py

feat: implement semantic search functionality and engine for project similarity analysis

eecb5f7 5 months ago

18.4 kB

import logging
from typing import Dict, Any, List, Optional
from functools import lru_cache

import pandas as pd

from src.similarity_model import (
    normalize_text,
    extract_features,
    load_model,
    load_faiss_index,
    load_metadata,
    search_by_text,
    load_feature_model,
    safe_feature_list,
    compute_feature_similarity,
    compute_hybrid_score,
    compute_originality,
    compute_confidence,
    risk_label
)

# ---------------------------------------------------------------------------
# Cross-encoder for paraphrase detection (lazy-loaded, cached)
# ---------------------------------------------------------------------------
@lru_cache(maxsize=1)
def _load_cross_encoder():
    from sentence_transformers import CrossEncoder
    logger.info("Loading cross-encoder: cross-encoder/stsb-distilroberta-base")
    return CrossEncoder("cross-encoder/stsb-distilroberta-base", max_length=512)

CROSS_ENCODER_THRESHOLD = 0.60   # minimum cross-score to trigger boost
CROSS_ENCODER_MAX_BOOST = 0.30   # maximum hybrid_score boost from cross-encoder
WORKFLOW_COVERAGE_THRESH = 0.50  # minimum coverage to trigger workflow penalty
WORKFLOW_FEATURE_THRESH  = 0.45  # minimum feature_score to trigger workflow penalty
WORKFLOW_MAX_BOOST       = 0.10  # maximum hybrid_score boost from workflow overlap

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s | %(levelname)s | %(message)s"
)
logger = logging.getLogger(__name__)

TITLE_COL = "project_title"
FEATURE_COL = "features"

DEFAULT_TOP_K = 5
DEFAULT_SEARCH_POOL = 50
DEFAULT_MIN_SEMANTIC_SCORE = 0.10
MAX_QUERY_FEATURES = 12

def build_raw_text(
    title: str = "",
    abstract: str = "",
    description: str = ""
) -> str:
    """
    Merge available text fields.
    """
    parts = [
        str(title).strip(),
        str(abstract).strip(),
        str(description).strip()
    ]

    return ". ".join(
        [p for p in parts if p]
    ).strip()

def merge_features(
    title: str = "",
    abstract: str = "",
    description: str = "",
    features: Optional[List[str]] = None,
    auto_extract: bool = True
) -> List[str]:
    """
    Use same extractor from preprocessing.py
    + merge manual features
    + remove duplicates
    """

    if features is None:
     features = []

    manual_features = safe_feature_list(features)

    raw_text = build_raw_text(
        title=title,
        abstract=abstract,
        description=description
    )

    auto_features = []

    if auto_extract:
        auto_features = extract_features(
            normalize_text(raw_text)
        )
    final = []
    seen = set()

    
    for feat in manual_features + auto_features:

        feat = str(feat).strip().lower()

        if feat and feat not in seen:
            seen.add(feat)
            final.append(feat)

    return final[:MAX_QUERY_FEATURES]

def build_query_project(
    title: str,
    abstract: str,
    description: str,
    features: List[str]
) -> Dict[str, Any]:

    return {
        TITLE_COL: str(title).strip(),
        "abstract": str(abstract).strip(),
        "description": str(description).strip(),
        FEATURE_COL: features
    }

def build_query_text(
    title: str,
    abstract: str,
    description: str,
    features: List[str]
) -> str:
    """
    Build semantic query text.
    """

    raw_text = build_raw_text(
        title=title,
        abstract=abstract,
        description=description
    )

    feature_text = " ".join(features)

    text = f"{raw_text}. {feature_text}"

    return normalize_text(text)

def compare_candidate(
    query_project: Dict[str, Any],
    candidate_row,
    semantic_score: float,
    feature_model
) -> Dict[str, Any]:

    candidate_features = safe_feature_list(
        candidate_row[FEATURE_COL]
    )

    feature_result = compute_feature_similarity(
        query_project[FEATURE_COL],
        candidate_features,
        model=feature_model
    )

    feature_score = feature_result["score"]
    coverage = feature_result["coverage"]

    query_feature_count = len(
        query_project[FEATURE_COL]
    )

    unique_query_count = len(
        feature_result["unique_a"]
    )

    base_similarity = compute_hybrid_score(
        semantic_score=semantic_score,
        feature_score=feature_score,
        coverage=coverage,
        feature_count=query_feature_count,
        unique_query_count=unique_query_count
    )

    # Calculate Jaccard word overlap on description + abstract to detect direct copy-pastes
    query_desc = (str(query_project.get("abstract", "")) + " " + str(query_project.get("description", ""))).strip()
    candidate_desc = (str(candidate_row.get("abstract", "")) + " " + str(candidate_row.get("description", ""))).strip()
    
    words_q = set(normalize_text(query_desc).split())
    words_c = set(normalize_text(candidate_desc).split())
    
    jaccard_overlap = 0.0
    if words_q and words_c:
        jaccard_overlap = len(words_q.intersection(words_c)) / len(words_q.union(words_c))

    if jaccard_overlap >= 0.60:
        hybrid_score = 0.95
    else:
        hybrid_score = base_similarity

    originality_score = compute_originality(
        hybrid_score=hybrid_score,
        unique_query_features=unique_query_count,
        total_query_features=query_feature_count
    )

    confidence_score = compute_confidence(
        semantic_score=semantic_score,
        feature_score=feature_score,
        coverage=coverage
    )

    return {
        "project_id": int(candidate_row.name),
        "project_title": candidate_row[TITLE_COL],

        "semantic_score":
            round(float(semantic_score), 4),

        "feature_score":
            feature_score,

        "coverage":
            coverage,

        "hybrid_score":
            hybrid_score,

        "originality_score":
            originality_score,

        "confidence_score":
            confidence_score,

        "duplicate_risk":
            risk_label(hybrid_score),

        "shared_features_count":
            feature_result["shared_count"],

        "matched_features":
            feature_result["matches"],

        "unique_query_features":
            feature_result["unique_a"],

        "unique_candidate_features":
            feature_result["unique_b"],
            
        "candidate_features":
            candidate_features
    }

def find_similar_projects(
    title: str = "",
    abstract: str = "",
    description: str = "",
    features: Optional[List[str]] = None,
    top_k: int = DEFAULT_TOP_K,
    search_pool: int = DEFAULT_SEARCH_POOL,
    auto_extract_features: bool = True,
    exclude_self: bool = False
) -> pd.DataFrame:
    """
    Final Smart Pipeline
    Supports:
    - title only
    - title + description
    - title + abstract
    - title + abstract + description
    - optional features
    """

    logger.info(
        "Loading models and artifacts..."
    )

    load_model()
    load_faiss_index()

    feature_model = load_feature_model()
    df = load_metadata()

    
    
    
    logger.info(
        "Preparing query..."
    )

    if not exclude_self:
        # Check local/cached metadata dataframe first (extremely fast, zero DB/network overhead)
        existing_project = df[
            df["project_title"].str.lower().str.strip()
            ==
            title.lower().strip()
        ]
        
        if len(existing_project) > 0:
            logger.info("Exact title match found in local metadata cache — returning originality = 0")
            matched_row = existing_project.iloc[0]
            project_id = int(matched_row.name)
            stored_features = safe_feature_list(matched_row["features"])
            matched_title = matched_row["project_title"]
            matched_abstract = matched_row.get("abstract", "")
            matched_desc = matched_row.get("description", "")
            
            return pd.DataFrame([{
                "project_id":           project_id,
                "project_title":        matched_title,
                "semantic_score":       1.0,
                "feature_score":        1.0,
                "coverage":             1.0,
                "hybrid_score":         1.0,
                "originality_score":    0.0,
                "confidence_score":     1.0,
                "duplicate_risk":       "Very High",
                "shared_features_count": len(stored_features),
                "matched_features":     [{"feature_a": f, "feature_b": f, "score": 1.0} for f in stored_features],
                "unique_query_features": [],
                "unique_candidate_features": [],
                "query_features_used":  stored_features,
                "query_clean_text":     title,
                "candidate_features":   stored_features,
                "abstract":             matched_abstract,
                "description":          matched_desc
            }])

        # If not found in cache, check database table 'preprocess' directly
        db_match_found = False
        db_project_id = None
        db_features = []
        db_abstract = ""
        db_description = ""
        matched_title = title
        
        try:
            import json
            from Data.database.sql_connector import engine
            from sqlalchemy import text
            
            with engine.connect() as conn:
                db_row = conn.execute(text("""
                    SELECT id, project_title, features, abstract, description
                    FROM preprocess
                    WHERE LOWER(LTRIM(RTRIM(project_title))) = LOWER(:title)
                """), {"title": title.strip()}).fetchone()
                
            if db_row:
                db_match_found = True
                db_project_id, matched_title, features_json, db_abstract, db_description = db_row
                # Parse features
                try:
                    db_features = json.loads(features_json) if isinstance(features_json, str) else features_json
                    if isinstance(db_features, str):
                        db_features = json.loads(db_features)
                except Exception:
                    db_features = []
                if not isinstance(db_features, list):
                    db_features = []
        except Exception as db_exc:
            logger.warning(f"Direct database check of 'preprocess' table failed: {db_exc}")
    
        if db_match_found:
            logger.info("Exact title match found in database 'preprocess' table — returning originality = 0")
            project_id = int(db_project_id)
            stored_features = db_features
            matched_abstract = db_abstract
            matched_desc = db_description
                
            return pd.DataFrame([{
                "project_id":           project_id,
                "project_title":        matched_title,
                "semantic_score":       1.0,
                "feature_score":        1.0,
                "coverage":             1.0,
                "hybrid_score":         1.0,
                "originality_score":    0.0,
                "confidence_score":     1.0,
                "duplicate_risk":       "Very High",
                "shared_features_count": len(stored_features),
                "matched_features":     [{"feature_a": f, "feature_b": f, "score": 1.0} for f in stored_features],
                "unique_query_features": [],
                "unique_candidate_features": [],
                "query_features_used":  stored_features,
                "query_clean_text":     title,
                "candidate_features":   stored_features,
                "abstract":             matched_abstract,
                "description":          matched_desc
            }])

    logger.info("Extracting new features")

    final_features = merge_features(
        title=title,
        abstract=abstract,
        description=description,
        features=features,
        auto_extract=auto_extract_features
    )

    query_project = build_query_project(
        title=title,
        abstract=abstract,
        description=description,
        features=final_features
    )

    query_text = build_query_text(
        title=title,
        abstract=abstract,
        description=description,
        features=final_features
    )

    
    
    
    logger.info(
        "Running semantic retrieval..."
    )

    semantic_results = search_by_text(
        query_text=query_text,
        k=search_pool + (5 if exclude_self else 0),
        min_score=DEFAULT_MIN_SEMANTIC_SCORE
    )

    if "message" in semantic_results.columns:
        return semantic_results
        
    if exclude_self and title.strip():
        semantic_results = semantic_results[
            semantic_results["project_title"].str.lower().str.strip() != title.lower().strip()
        ]

    
    
    
    logger.info(
        "Running hybrid ranking..."
    )

    rows = []

    for _, row in semantic_results.iterrows():

        candidate_id = int(
            row["project_id"]
        )

        semantic_score = float(
            row["score"]
        )

        candidate_row = df.loc[
            candidate_id
        ]

        result = compare_candidate(
            query_project=query_project,
            candidate_row=candidate_row,
            semantic_score=semantic_score,
            feature_model=feature_model
        )

        rows.append(result)

    if not rows:
        return pd.DataFrame([{
            "message":
            "No strong similar projects found."
        }])

    import numpy as np
    from src.similarity_model.explainability import generate_explanation

    final_df = pd.DataFrame(rows)

    # Sort results
    final_df = final_df.sort_values(
        by=[
            "hybrid_score",
            "confidence_score",
            "semantic_score"
        ],
        ascending=False
    ).reset_index(drop=True)

    # -----------------------------------------------------------------
    # CROSS-ENCODER RE-SCORING (top-1 candidate only)
    # -----------------------------------------------------------------
    if len(final_df) > 0:
        top_row = final_df.iloc[0]
        candidate_id = int(top_row["project_id"])
        candidate_row = df.loc[candidate_id]

        # Build full texts for cross-encoder comparison
        query_full = build_raw_text(
            title=title, abstract=abstract, description=description
        )
        candidate_full = build_raw_text(
            title=str(candidate_row.get(TITLE_COL, "")),
            abstract=str(candidate_row.get("abstract", "")),
            description=str(candidate_row.get("description", ""))
        )

        try:
            cross_encoder = _load_cross_encoder()
            cross_score = float(
                cross_encoder.predict([(query_full, candidate_full)])[0]
            )
            # stsb model already outputs [0, 1] — clamp for safety
            cross_score = max(0.0, min(1.0, cross_score))
            logger.info(
                f"Cross-encoder score (top-1): {cross_score:.4f}"
            )
        except Exception as exc:
            logger.warning(f"Cross-encoder failed, skipping: {exc}")
            cross_score = 0.0

        # Apply cross-encoder boost if threshold met
        if cross_score >= CROSS_ENCODER_THRESHOLD:
            boost = CROSS_ENCODER_MAX_BOOST * (
                (cross_score - CROSS_ENCODER_THRESHOLD)
                / (1.0 - CROSS_ENCODER_THRESHOLD)
            )
            original_hybrid = float(final_df.loc[0, "hybrid_score"])
            boosted_hybrid = min(1.0, original_hybrid + boost)
            final_df.loc[0, "hybrid_score"] = round(boosted_hybrid, 4)
            logger.info(
                f"Cross-encoder boost: {original_hybrid:.4f} -> {boosted_hybrid:.4f} "
                f"(+{boost:.4f})"
            )

        # -----------------------------------------------------------------
        # WORKFLOW OVERLAP PENALTY
        # -----------------------------------------------------------------
        top_coverage = float(top_row.get("coverage", 0.0))
        top_feat_score = float(top_row.get("feature_score", 0.0))

        if (top_coverage >= WORKFLOW_COVERAGE_THRESH
                and top_feat_score >= WORKFLOW_FEATURE_THRESH):
            workflow_boost = (
                WORKFLOW_MAX_BOOST * top_coverage * top_feat_score
            )
            current_hybrid = float(final_df.loc[0, "hybrid_score"])
            boosted_hybrid = min(1.0, current_hybrid + workflow_boost)
            final_df.loc[0, "hybrid_score"] = round(boosted_hybrid, 4)
            logger.info(
                f"Workflow overlap boost: {current_hybrid:.4f} -> "
                f"{boosted_hybrid:.4f} (+{workflow_boost:.4f})"
            )

    # -----------------------------------------------------------------
    # DECAYING AGGREGATION over Top-5
    # -----------------------------------------------------------------
    K_val = min(5, len(final_df))
    if K_val > 0:
        s1 = float(final_df.loc[0, "hybrid_score"])
        density_penalty = 0.0
        beta = 0.05
        lam = 0.5
        for i in range(1, K_val):
            si = float(final_df.loc[i, "hybrid_score"])
            density_penalty += np.exp(-lam * i) * si

        aggregated_score = min(1.0, s1 + beta * density_penalty)

        # Recalculate originality based on aggregated similarity score
        aggregated_originality = compute_originality(
            hybrid_score=aggregated_score
        )
        if aggregated_score >= 0.90:
            aggregated_originality = 0.0

        final_df.loc[0, "originality_score"] = aggregated_originality
    else:
        aggregated_score = 0.0

    # Retain the top match
    final_df = final_df.head(top_k).copy()
    
    # Overwrite originality score for any project with >= 0.90 hybrid score to 0.0
    final_df.loc[
        final_df["hybrid_score"] >= 0.90,
        "originality_score"
    ] = 0.0

    final_df["query_features_used"] = [
        final_features
    ] * len(final_df)

    final_df["query_clean_text"] = [
        query_text
    ] * len(final_df)

    # Generate explanation
    if len(final_df) > 0:
        overall_orig = float(final_df.loc[0, "originality_score"])
        explanation_text = generate_explanation(overall_orig, final_df, final_features)
        final_df["explanation"] = explanation_text

    return final_df

	import logging
	from typing import Dict, Any, List, Optional
	from functools import lru_cache

	import pandas as pd

	from src.similarity_model import (
	normalize_text,
	extract_features,
	load_model,
	load_faiss_index,
	load_metadata,
	search_by_text,
	load_feature_model,
	safe_feature_list,
	compute_feature_similarity,
	compute_hybrid_score,
	compute_originality,
	compute_confidence,
	risk_label
	)

	# ---------------------------------------------------------------------------
	# Cross-encoder for paraphrase detection (lazy-loaded, cached)
	# ---------------------------------------------------------------------------
	@lru_cache(maxsize=1)
	def _load_cross_encoder():
	from sentence_transformers import CrossEncoder
	logger.info("Loading cross-encoder: cross-encoder/stsb-distilroberta-base")
	return CrossEncoder("cross-encoder/stsb-distilroberta-base", max_length=512)

	CROSS_ENCODER_THRESHOLD = 0.60 # minimum cross-score to trigger boost
	CROSS_ENCODER_MAX_BOOST = 0.30 # maximum hybrid_score boost from cross-encoder
	WORKFLOW_COVERAGE_THRESH = 0.50 # minimum coverage to trigger workflow penalty
	WORKFLOW_FEATURE_THRESH = 0.45 # minimum feature_score to trigger workflow penalty
	WORKFLOW_MAX_BOOST = 0.10 # maximum hybrid_score boost from workflow overlap

	logging.basicConfig(
	level=logging.INFO,
	format="%(asctime)s \| %(levelname)s \| %(message)s"
	)
	logger = logging.getLogger(__name__)

	TITLE_COL = "project_title"
	FEATURE_COL = "features"

	DEFAULT_TOP_K = 5
	DEFAULT_SEARCH_POOL = 50
	DEFAULT_MIN_SEMANTIC_SCORE = 0.10
	MAX_QUERY_FEATURES = 12

	def build_raw_text(
	title: str = "",
	abstract: str = "",
	description: str = ""
	) -> str:
	"""
	Merge available text fields.
	"""
	parts = [
	str(title).strip(),
	str(abstract).strip(),
	str(description).strip()
	]

	return ". ".join(
	[p for p in parts if p]
	).strip()

	def merge_features(
	title: str = "",
	abstract: str = "",
	description: str = "",
	features: Optional[List[str]] = None,
	auto_extract: bool = True
	) -> List[str]:
	"""
	Use same extractor from preprocessing.py
	+ merge manual features
	+ remove duplicates
	"""

	if features is None:
	features = []

	manual_features = safe_feature_list(features)

	raw_text = build_raw_text(
	title=title,
	abstract=abstract,
	description=description
	)

	auto_features = []

	if auto_extract:
	auto_features = extract_features(
	normalize_text(raw_text)
	)
	final = []
	seen = set()


	for feat in manual_features + auto_features:

	feat = str(feat).strip().lower()

	if feat and feat not in seen:
	seen.add(feat)
	final.append(feat)

	return final[:MAX_QUERY_FEATURES]

	def build_query_project(
	title: str,
	abstract: str,
	description: str,
	features: List[str]
	) -> Dict[str, Any]:

	return {
	TITLE_COL: str(title).strip(),
	"abstract": str(abstract).strip(),
	"description": str(description).strip(),
	FEATURE_COL: features
	}

	def build_query_text(
	title: str,
	abstract: str,
	description: str,
	features: List[str]
	) -> str:
	"""
	Build semantic query text.
	"""

	raw_text = build_raw_text(
	title=title,
	abstract=abstract,
	description=description
	)

	feature_text = " ".join(features)

	text = f"{raw_text}. {feature_text}"

	return normalize_text(text)

	def compare_candidate(
	query_project: Dict[str, Any],
	candidate_row,
	semantic_score: float,
	feature_model
	) -> Dict[str, Any]:

	candidate_features = safe_feature_list(
	candidate_row[FEATURE_COL]
	)

	feature_result = compute_feature_similarity(
	query_project[FEATURE_COL],
	candidate_features,
	model=feature_model
	)

	feature_score = feature_result["score"]
	coverage = feature_result["coverage"]

	query_feature_count = len(
	query_project[FEATURE_COL]
	)

	unique_query_count = len(
	feature_result["unique_a"]
	)

	base_similarity = compute_hybrid_score(
	semantic_score=semantic_score,
	feature_score=feature_score,
	coverage=coverage,
	feature_count=query_feature_count,
	unique_query_count=unique_query_count
	)

	# Calculate Jaccard word overlap on description + abstract to detect direct copy-pastes
	query_desc = (str(query_project.get("abstract", "")) + " " + str(query_project.get("description", ""))).strip()
	candidate_desc = (str(candidate_row.get("abstract", "")) + " " + str(candidate_row.get("description", ""))).strip()

	words_q = set(normalize_text(query_desc).split())
	words_c = set(normalize_text(candidate_desc).split())

	jaccard_overlap = 0.0
	if words_q and words_c:
	jaccard_overlap = len(words_q.intersection(words_c)) / len(words_q.union(words_c))

	if jaccard_overlap >= 0.60:
	hybrid_score = 0.95
	else:
	hybrid_score = base_similarity

	originality_score = compute_originality(
	hybrid_score=hybrid_score,
	unique_query_features=unique_query_count,
	total_query_features=query_feature_count
	)

	confidence_score = compute_confidence(
	semantic_score=semantic_score,
	feature_score=feature_score,
	coverage=coverage
	)

	return {
	"project_id": int(candidate_row.name),
	"project_title": candidate_row[TITLE_COL],

	"semantic_score":
	round(float(semantic_score), 4),

	"feature_score":
	feature_score,

	"coverage":
	coverage,

	"hybrid_score":
	hybrid_score,

	"originality_score":
	originality_score,

	"confidence_score":
	confidence_score,

	"duplicate_risk":
	risk_label(hybrid_score),

	"shared_features_count":
	feature_result["shared_count"],

	"matched_features":
	feature_result["matches"],

	"unique_query_features":
	feature_result["unique_a"],

	"unique_candidate_features":
	feature_result["unique_b"],

	"candidate_features":
	candidate_features
	}

	def find_similar_projects(
	title: str = "",
	abstract: str = "",
	description: str = "",
	features: Optional[List[str]] = None,
	top_k: int = DEFAULT_TOP_K,
	search_pool: int = DEFAULT_SEARCH_POOL,
	auto_extract_features: bool = True,
	exclude_self: bool = False
	) -> pd.DataFrame:
	"""
	Final Smart Pipeline

	Supports:
	- title only
	- title + description
	- title + abstract
	- title + abstract + description
	- optional features
	"""

	logger.info(
	"Loading models and artifacts..."
	)

	load_model()
	load_faiss_index()

	feature_model = load_feature_model()
	df = load_metadata()




	logger.info(
	"Preparing query..."
	)

	if not exclude_self:
	# Check local/cached metadata dataframe first (extremely fast, zero DB/network overhead)
	existing_project = df[
	df["project_title"].str.lower().str.strip()
	==
	title.lower().strip()
	]

	if len(existing_project) > 0:
	logger.info("Exact title match found in local metadata cache — returning originality = 0")
	matched_row = existing_project.iloc[0]
	project_id = int(matched_row.name)
	stored_features = safe_feature_list(matched_row["features"])
	matched_title = matched_row["project_title"]
	matched_abstract = matched_row.get("abstract", "")
	matched_desc = matched_row.get("description", "")

	return pd.DataFrame([{
	"project_id": project_id,
	"project_title": matched_title,
	"semantic_score": 1.0,
	"feature_score": 1.0,
	"coverage": 1.0,
	"hybrid_score": 1.0,
	"originality_score": 0.0,
	"confidence_score": 1.0,
	"duplicate_risk": "Very High",
	"shared_features_count": len(stored_features),
	"matched_features": [{"feature_a": f, "feature_b": f, "score": 1.0} for f in stored_features],
	"unique_query_features": [],
	"unique_candidate_features": [],
	"query_features_used": stored_features,
	"query_clean_text": title,
	"candidate_features": stored_features,
	"abstract": matched_abstract,
	"description": matched_desc
	}])

	# If not found in cache, check database table 'preprocess' directly
	db_match_found = False
	db_project_id = None
	db_features = []
	db_abstract = ""
	db_description = ""
	matched_title = title

	try:
	import json
	from Data.database.sql_connector import engine
	from sqlalchemy import text

	with engine.connect() as conn:
	db_row = conn.execute(text("""
	SELECT id, project_title, features, abstract, description
	FROM preprocess
	WHERE LOWER(LTRIM(RTRIM(project_title))) = LOWER(:title)
	"""), {"title": title.strip()}).fetchone()

	if db_row:
	db_match_found = True
	db_project_id, matched_title, features_json, db_abstract, db_description = db_row
	# Parse features
	try:
	db_features = json.loads(features_json) if isinstance(features_json, str) else features_json
	if isinstance(db_features, str):
	db_features = json.loads(db_features)
	except Exception:
	db_features = []
	if not isinstance(db_features, list):
	db_features = []
	except Exception as db_exc:
	logger.warning(f"Direct database check of 'preprocess' table failed: {db_exc}")

	if db_match_found:
	logger.info("Exact title match found in database 'preprocess' table — returning originality = 0")
	project_id = int(db_project_id)
	stored_features = db_features
	matched_abstract = db_abstract
	matched_desc = db_description

	return pd.DataFrame([{
	"project_id": project_id,
	"project_title": matched_title,
	"semantic_score": 1.0,
	"feature_score": 1.0,
	"coverage": 1.0,
	"hybrid_score": 1.0,
	"originality_score": 0.0,
	"confidence_score": 1.0,
	"duplicate_risk": "Very High",
	"shared_features_count": len(stored_features),
	"matched_features": [{"feature_a": f, "feature_b": f, "score": 1.0} for f in stored_features],
	"unique_query_features": [],
	"unique_candidate_features": [],
	"query_features_used": stored_features,
	"query_clean_text": title,
	"candidate_features": stored_features,
	"abstract": matched_abstract,
	"description": matched_desc
	}])

	logger.info("Extracting new features")

	final_features = merge_features(
	title=title,
	abstract=abstract,
	description=description,
	features=features,
	auto_extract=auto_extract_features
	)

	query_project = build_query_project(
	title=title,
	abstract=abstract,
	description=description,
	features=final_features
	)

	query_text = build_query_text(
	title=title,
	abstract=abstract,
	description=description,
	features=final_features
	)




	logger.info(
	"Running semantic retrieval..."
	)

	semantic_results = search_by_text(
	query_text=query_text,
	k=search_pool + (5 if exclude_self else 0),
	min_score=DEFAULT_MIN_SEMANTIC_SCORE
	)

	if "message" in semantic_results.columns:
	return semantic_results

	if exclude_self and title.strip():
	semantic_results = semantic_results[
	semantic_results["project_title"].str.lower().str.strip() != title.lower().strip()
	]




	logger.info(
	"Running hybrid ranking..."
	)

	rows = []

	for _, row in semantic_results.iterrows():

	candidate_id = int(
	row["project_id"]
	)

	semantic_score = float(
	row["score"]
	)

	candidate_row = df.loc[
	candidate_id
	]

	result = compare_candidate(
	query_project=query_project,
	candidate_row=candidate_row,
	semantic_score=semantic_score,
	feature_model=feature_model
	)

	rows.append(result)

	if not rows:
	return pd.DataFrame([{
	"message":
	"No strong similar projects found."
	}])

	import numpy as np
	from src.similarity_model.explainability import generate_explanation

	final_df = pd.DataFrame(rows)

	# Sort results
	final_df = final_df.sort_values(
	by=[
	"hybrid_score",
	"confidence_score",
	"semantic_score"
	],
	ascending=False
	).reset_index(drop=True)

	# -----------------------------------------------------------------
	# CROSS-ENCODER RE-SCORING (top-1 candidate only)
	# -----------------------------------------------------------------
	if len(final_df) > 0:
	top_row = final_df.iloc[0]
	candidate_id = int(top_row["project_id"])
	candidate_row = df.loc[candidate_id]

	# Build full texts for cross-encoder comparison
	query_full = build_raw_text(
	title=title, abstract=abstract, description=description
	)
	candidate_full = build_raw_text(
	title=str(candidate_row.get(TITLE_COL, "")),
	abstract=str(candidate_row.get("abstract", "")),
	description=str(candidate_row.get("description", ""))
	)

	try:
	cross_encoder = _load_cross_encoder()
	cross_score = float(
	cross_encoder.predict([(query_full, candidate_full)])[0]
	)
	# stsb model already outputs [0, 1] — clamp for safety
	cross_score = max(0.0, min(1.0, cross_score))
	logger.info(
	f"Cross-encoder score (top-1): {cross_score:.4f}"
	)
	except Exception as exc:
	logger.warning(f"Cross-encoder failed, skipping: {exc}")
	cross_score = 0.0

	# Apply cross-encoder boost if threshold met
	if cross_score >= CROSS_ENCODER_THRESHOLD:
	boost = CROSS_ENCODER_MAX_BOOST * (
	(cross_score - CROSS_ENCODER_THRESHOLD)
	/ (1.0 - CROSS_ENCODER_THRESHOLD)
	)
	original_hybrid = float(final_df.loc[0, "hybrid_score"])
	boosted_hybrid = min(1.0, original_hybrid + boost)
	final_df.loc[0, "hybrid_score"] = round(boosted_hybrid, 4)
	logger.info(
	f"Cross-encoder boost: {original_hybrid:.4f} -> {boosted_hybrid:.4f} "
	f"(+{boost:.4f})"
	)

	# -----------------------------------------------------------------
	# WORKFLOW OVERLAP PENALTY
	# -----------------------------------------------------------------
	top_coverage = float(top_row.get("coverage", 0.0))
	top_feat_score = float(top_row.get("feature_score", 0.0))

	if (top_coverage >= WORKFLOW_COVERAGE_THRESH
	and top_feat_score >= WORKFLOW_FEATURE_THRESH):
	workflow_boost = (
	WORKFLOW_MAX_BOOST * top_coverage * top_feat_score
	)
	current_hybrid = float(final_df.loc[0, "hybrid_score"])
	boosted_hybrid = min(1.0, current_hybrid + workflow_boost)
	final_df.loc[0, "hybrid_score"] = round(boosted_hybrid, 4)
	logger.info(
	f"Workflow overlap boost: {current_hybrid:.4f} -> "
	f"{boosted_hybrid:.4f} (+{workflow_boost:.4f})"
	)

	# -----------------------------------------------------------------
	# DECAYING AGGREGATION over Top-5
	# -----------------------------------------------------------------
	K_val = min(5, len(final_df))
	if K_val > 0:
	s1 = float(final_df.loc[0, "hybrid_score"])
	density_penalty = 0.0
	beta = 0.05
	lam = 0.5
	for i in range(1, K_val):
	si = float(final_df.loc[i, "hybrid_score"])
	density_penalty += np.exp(-lam * i) * si

	aggregated_score = min(1.0, s1 + beta * density_penalty)

	# Recalculate originality based on aggregated similarity score
	aggregated_originality = compute_originality(
	hybrid_score=aggregated_score
	)
	if aggregated_score >= 0.90:
	aggregated_originality = 0.0

	final_df.loc[0, "originality_score"] = aggregated_originality
	else:
	aggregated_score = 0.0

	# Retain the top match
	final_df = final_df.head(top_k).copy()

	# Overwrite originality score for any project with >= 0.90 hybrid score to 0.0
	final_df.loc[
	final_df["hybrid_score"] >= 0.90,
	"originality_score"
	] = 0.0

	final_df["query_features_used"] = [
	final_features
	] * len(final_df)

	final_df["query_clean_text"] = [
	query_text
	] * len(final_df)

	# Generate explanation
	if len(final_df) > 0:
	overall_orig = float(final_df.loc[0, "originality_score"])
	explanation_text = generate_explanation(overall_orig, final_df, final_features)
	final_df["explanation"] = explanation_text

	return final_df