diff --git a/.env b/.env index 0cff7486498cec9857c67044cb3d0f31b7c02141..b8efe84c452cc104e7f1b7b66b25b2fa5391c2b3 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ -GEMINI_API_KEY=AIzaSyAkFsaN3BKoSQmRW4FzTahhZXbq-ldsDZ4 +GEMINI_API_KEY=AQ.Ab8RN6K-wFpuRR3mv7AUv7p6QqXf98Dlth0IRF3N42UIfjCEbw GEMINI_MODEL_NAME=gemini-2.5-flash \ No newline at end of file diff --git a/src/recommendation_engine/__init__.py b/src/recommendation_engine/__init__.py index 0d38487c879bd97987150154f59fc32301f3f326..a7774d0f8c42b6a25fa6ebbaea94fd4b6025d32a 100644 --- a/src/recommendation_engine/__init__.py +++ b/src/recommendation_engine/__init__.py @@ -1,2 +1,17 @@ -# __init__.py -# empty file is totally fine \ No newline at end of file +from .chatbot_engine import chatbot +from .command_handler import handle_command, is_command +from .context_builder import build_project_context, build_domain_context, build_architecture_hints +from .feature_generator import generate_features +from .full_project_generator import generate_full_project +from .idea_generator import generate_ideas +from .intent_classifier import detect_intent, classify_with_llm +from .llm_client import generate_text, generate_list, LLMProviderError +from .llm_router import analyze_user_input +from .local_llm import generate_local +from .memory_store import get_user_memory, save_user_memory, get_all_chats, switch_chat, clear_user_memory +from .novelty_checker import score_feature_novelty, is_feature_novel, is_idea_novel +from .prompt_builder import build_chat_prompt, build_feature_prompt, build_idea_prompt, build_description_prompt, build_full_project_prompt +from .response_formatter import format_response, format_full_project +from .semantic_intent_classifier import detect_intent_semantic +from .state_manager import update_state, reset_for_new_idea +from .validator import validate_generated_list, filter_items diff --git a/src/recommendation_engine/__pycache__/__init__.cpython-313.pyc b/src/recommendation_engine/__pycache__/__init__.cpython-313.pyc index 1859e0e78ebccdf929d7274735adc59f0cc5bc65..c495ac09a7308b724d241694f2cde8f6da45e3ee 100644 Binary files a/src/recommendation_engine/__pycache__/__init__.cpython-313.pyc and b/src/recommendation_engine/__pycache__/__init__.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/chatbot_engine.cpython-313.pyc b/src/recommendation_engine/__pycache__/chatbot_engine.cpython-313.pyc index 9d9c7110624236f50a92f9edef56c3507bbbc000..1cf1fdd1b0896d7f75048c28be9ef4f38bf25f6d 100644 Binary files a/src/recommendation_engine/__pycache__/chatbot_engine.cpython-313.pyc and b/src/recommendation_engine/__pycache__/chatbot_engine.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/command_handler.cpython-313.pyc b/src/recommendation_engine/__pycache__/command_handler.cpython-313.pyc index df8b92932fa823ccf628a0dbf30c98a0bf7d8289..0d5f21f798534fdb5d88b51211acf186e3e44e8f 100644 Binary files a/src/recommendation_engine/__pycache__/command_handler.cpython-313.pyc and b/src/recommendation_engine/__pycache__/command_handler.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/config.cpython-313.pyc b/src/recommendation_engine/__pycache__/config.cpython-313.pyc index 7173cf912387bfbb2e8485b1c77480875d7cd34e..bae54366649f59323b2eba9e2f8eff987a35bdcf 100644 Binary files a/src/recommendation_engine/__pycache__/config.cpython-313.pyc and b/src/recommendation_engine/__pycache__/config.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/context_builder.cpython-313.pyc b/src/recommendation_engine/__pycache__/context_builder.cpython-313.pyc index 547573ea2044eb53c68dee35d2a2c3c139471e88..678aa6c510705737b7feac3bd3744d3707c05732 100644 Binary files a/src/recommendation_engine/__pycache__/context_builder.cpython-313.pyc and b/src/recommendation_engine/__pycache__/context_builder.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/feature_generator.cpython-313.pyc b/src/recommendation_engine/__pycache__/feature_generator.cpython-313.pyc index ddea12785de6e5d303805ca13a16d8f4f8cea2b3..a972e258a621c82cb96922a032a7972ac886fece 100644 Binary files a/src/recommendation_engine/__pycache__/feature_generator.cpython-313.pyc and b/src/recommendation_engine/__pycache__/feature_generator.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/full_project_generator.cpython-313.pyc b/src/recommendation_engine/__pycache__/full_project_generator.cpython-313.pyc index 5283ca7cd6ffd6d5dac70fc31323e9a446cf400d..fe61adcbbbb7cf5d7de2bbbde4040d47c62e025c 100644 Binary files a/src/recommendation_engine/__pycache__/full_project_generator.cpython-313.pyc and b/src/recommendation_engine/__pycache__/full_project_generator.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/idea_generator.cpython-313.pyc b/src/recommendation_engine/__pycache__/idea_generator.cpython-313.pyc index 395fae23a48fb6be66db2804048ddf5c4a3e1f2b..3276deb730f8cb5617eea213bcebe25196a7b728 100644 Binary files a/src/recommendation_engine/__pycache__/idea_generator.cpython-313.pyc and b/src/recommendation_engine/__pycache__/idea_generator.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/llm_client.cpython-313.pyc b/src/recommendation_engine/__pycache__/llm_client.cpython-313.pyc index f2cbf736879c4f4bdbe700ce8d88902ee22bc47a..3380a41630dbc07aa37ea68144b8bd6762fb434f 100644 Binary files a/src/recommendation_engine/__pycache__/llm_client.cpython-313.pyc and b/src/recommendation_engine/__pycache__/llm_client.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/llm_router.cpython-313.pyc b/src/recommendation_engine/__pycache__/llm_router.cpython-313.pyc index 1cca8158ae9aa100f8c5c25b2f695c99dfef154c..5cbbe26893d748d1e22f011c39433afeeab4aef4 100644 Binary files a/src/recommendation_engine/__pycache__/llm_router.cpython-313.pyc and b/src/recommendation_engine/__pycache__/llm_router.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/memory_store.cpython-313.pyc b/src/recommendation_engine/__pycache__/memory_store.cpython-313.pyc index 2d03465742baf91fa1e5df5cb4c60c6a18b00809..b22ebc1745688273d257f5675410ccf0905ed1c5 100644 Binary files a/src/recommendation_engine/__pycache__/memory_store.cpython-313.pyc and b/src/recommendation_engine/__pycache__/memory_store.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/novelty_checker.cpython-313.pyc b/src/recommendation_engine/__pycache__/novelty_checker.cpython-313.pyc index 8f3e575564d949f5e5a0781cb31a2a6cb2354946..4cb0e84c0f24d2a1d8579e2ac85507b30aaf1436 100644 Binary files a/src/recommendation_engine/__pycache__/novelty_checker.cpython-313.pyc and b/src/recommendation_engine/__pycache__/novelty_checker.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/prompt_builder.cpython-313.pyc b/src/recommendation_engine/__pycache__/prompt_builder.cpython-313.pyc index 14cc871cf8ab48b15395aa0c9bc56ca23c3769f3..67c82bdf5d2563f8bced562aea0e5db4600f961e 100644 Binary files a/src/recommendation_engine/__pycache__/prompt_builder.cpython-313.pyc and b/src/recommendation_engine/__pycache__/prompt_builder.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/response_formatter.cpython-313.pyc b/src/recommendation_engine/__pycache__/response_formatter.cpython-313.pyc index 12580db22ef15aa5015f2cc1826e2e746c19c3f2..f8aecea5b3a1c113d14c48d9215b5fe3f4479572 100644 Binary files a/src/recommendation_engine/__pycache__/response_formatter.cpython-313.pyc and b/src/recommendation_engine/__pycache__/response_formatter.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/state_manager.cpython-313.pyc b/src/recommendation_engine/__pycache__/state_manager.cpython-313.pyc index d4b6831c6e06f8df57e7eebb5968b1b50c28b0df..62e4a08b61e9a1d657aa59f7098c44702b6e408d 100644 Binary files a/src/recommendation_engine/__pycache__/state_manager.cpython-313.pyc and b/src/recommendation_engine/__pycache__/state_manager.cpython-313.pyc differ diff --git a/src/recommendation_engine/__pycache__/validator.cpython-313.pyc b/src/recommendation_engine/__pycache__/validator.cpython-313.pyc index 3d5ee84eac1902b889754833eb75381f496df507..0e83f273573c55c2be6926b5988bd2f6980ce9b1 100644 Binary files a/src/recommendation_engine/__pycache__/validator.cpython-313.pyc and b/src/recommendation_engine/__pycache__/validator.cpython-313.pyc differ diff --git a/src/recommendation_engine/chatbot_engine.py b/src/recommendation_engine/chatbot_engine.py index d9c8ffd263232957d185d697d606559bc255619b..8b70a7988e121651cf11938fcbf3f384f83c70cb 100644 --- a/src/recommendation_engine/chatbot_engine.py +++ b/src/recommendation_engine/chatbot_engine.py @@ -1,5 +1,3 @@ -# src.recommendation_engine.chatbot_engine.py - from src.recommendation_engine.memory_store import ( get_user_memory, save_user_memory, @@ -28,17 +26,12 @@ from src.recommendation_engine.full_project_generator import ( import re - -# ===================================================== -# Helpers -# ===================================================== def extract_number(text: str, default=5): nums = re.findall(r"\d+", text) return min(int(nums[0]), 20) if nums else default - def is_weak_project_title(title: str) -> bool: if not title: @@ -48,7 +41,7 @@ def is_weak_project_title(title: str) -> bool: words = title.split() - # too short + if len(words) < 4: return True @@ -72,7 +65,6 @@ def is_weak_project_title(title: str) -> bool: return len(words) < 3 - def is_generic_project_reference(text: str) -> bool: text = text.strip().lower() @@ -103,17 +95,17 @@ def looks_like_real_project_title(title: str) -> bool: words = title.split() - # minimum words + if len(words) < 2: return False - # too repetitive + unique_ratio = len(set(words)) / len(words) if unique_ratio < 0.5: return False - # gibberish detection + nonsense_patterns = [ "asd", "qwe", @@ -125,10 +117,9 @@ def looks_like_real_project_title(title: str) -> bool: lowered = title.lower() - - # ========================= - # QUESTION / NORMAL CHAT FILTER - # ========================= + + + question_starts = ( "how ", "what ", @@ -150,10 +141,10 @@ def looks_like_real_project_title(title: str) -> bool: if p in lowered: return False - # at least one meaningful keyword + keywords = { - # AI / Systems + "management", "analysis", "detection", @@ -172,7 +163,7 @@ def looks_like_real_project_title(title: str) -> bool: "authentication", "optimization", - # domains + "healthcare", "fintech", "education", @@ -184,7 +175,7 @@ def looks_like_real_project_title(title: str) -> bool: "agriculture", "transport", - # technologies + "ai", "iot", "blockchain", @@ -193,7 +184,7 @@ def looks_like_real_project_title(title: str) -> bool: "vision", "embedded", - # project words + "system", "platform", "application", @@ -208,7 +199,6 @@ def looks_like_real_project_title(title: str) -> bool: return True - FOLLOWUP_WORDS = [ "another", "more", @@ -219,9 +209,6 @@ FOLLOWUP_WORDS = [ "another features" ] -# ===================================================== -# RESPONSE FINALIZER -# ===================================================== def finalize_response( user_input, response, @@ -249,19 +236,18 @@ def finalize_response( return response - def is_gibberish_text(text: str) -> bool: text = text.strip().lower() - # allow menu choices + if text in {"1", "2", "3"}: return False - # very short non-numeric random text + if len(text) < 3: - # allow short meaningful commands + allowed_short = { "hi", "hey", @@ -297,7 +283,7 @@ def is_gibberish_text(text: str) -> bool: words = text.split() - # repeated same words + if len(words) >= 3: unique_ratio = len(set(words)) / len(words) @@ -313,7 +299,7 @@ def is_project_related(text: str) -> bool: keywords = [ - # project words + "project", "system", "platform", @@ -323,7 +309,7 @@ def is_project_related(text: str) -> bool: "dashboard", "management", - # technical + "ai", "ml", "machine learning", @@ -338,7 +324,7 @@ def is_project_related(text: str) -> bool: "database", "api", - # actions + "generate", "feature", "features", @@ -349,7 +335,7 @@ def is_project_related(text: str) -> bool: "technologies", "architecture", - # domains + "healthcare", "education", "fintech", @@ -366,17 +352,13 @@ def is_project_related(text: str) -> bool: for keyword in keywords ) -# ===================================================== -# MAIN CHATBOT -# ===================================================== def chatbot(user_id: str, user_input: str): text = user_input.lower().strip() - - # ========================================= - # EXPLICIT IDEA REQUESTS - # ========================================= + + + explicit_idea_requests = [ "idea", "new idea", @@ -407,14 +389,14 @@ def chatbot(user_id: str, user_input: str): "🎯 Specify a domain (AI, healthcare, fintech...)" ) - # ================================================= - # ALLOW MENU NUMBERS - # ================================================= + + + if text in {"1", "2", "3"}: pass - # ================================================= - # GIBBERISH DETECTION - # ================================================= + + + if is_gibberish_text(text): return ( @@ -425,22 +407,22 @@ def chatbot(user_id: str, user_input: str): "- improve my project\n" "- suggest technologies for fintech app" ) - # ================================================= - # COMMANDS - # ================================================= + + + if is_command(user_input): return handle_command(user_input) - # ================================================= - # MEMORY - # ================================================= + + + memory = get_user_memory(user_id) history = memory.get("history", []) state = memory.get("state") or default_state() - # ensure keys + state.setdefault("menu_mode", False) state.setdefault("selected_option", None) state.setdefault("waiting_for_domain", False) @@ -453,9 +435,9 @@ def chatbot(user_id: str, user_input: str): state.setdefault("waiting_for_full_project_domain", False) state.setdefault("waiting_for_full_project_selection", False) - # ================================================= - # GREETING - # ================================================= + + + if text in ["","hi", "hello", "hey"]: save_user_memory(user_id, { @@ -477,9 +459,9 @@ def chatbot(user_id: str, user_input: str): "- improve my graduation project\n" "- suggest technologies for fintech app" ) - # ================================================= - # FULL PROJECT STARTER - # ================================================= + + + if any(x in text for x in [ "generate full project", @@ -491,12 +473,12 @@ def chatbot(user_id: str, user_input: str): ]): - # ============================================ - # IF PROJECT TITLE ALREADY EXISTS - # ============================================ + + + if state.get("project_title"): - # Generate features automatically if missing + if not state.get("features"): feature_result = generate_features( @@ -517,7 +499,7 @@ def chatbot(user_id: str, user_input: str): [] ) - # Generate full project + result = generate_full_project( title=state.get("project_title"), @@ -586,9 +568,9 @@ def chatbot(user_id: str, user_input: str): user_id ) - # ============================================ - # NO PROJECT TITLE → ASK FOR DOMAIN - # ============================================ + + + state["waiting_for_full_project_domain"] = True save_user_memory(user_id, { @@ -604,9 +586,9 @@ def chatbot(user_id: str, user_input: str): "- education\n" "- fintech" ) - # ================================================= - # MENU HANDLER - # ================================================= + + + if state.get("menu_mode"): if text in ["1", "2", "3"]: @@ -673,9 +655,9 @@ def chatbot(user_id: str, user_input: str): state["menu_mode"] = False - # ================================================= - # PROJECT ACTION HANDLER - # ================================================= + + + if state.get("waiting_for_project_action"): feature_words = [ @@ -706,9 +688,9 @@ def chatbot(user_id: str, user_input: str): "final project" ] - # ========================================= - # FEATURE GENERATION - # ========================================= + + + if any(w in text for w in feature_words): state["waiting_for_project_action"] = False @@ -720,9 +702,9 @@ def chatbot(user_id: str, user_input: str): return chatbot(user_id, "generate features") - # ========================================= - # PROJECT DISCUSSION - # ========================================= + + + elif any(w in text for w in chat_words): state["waiting_for_project_action"] = False @@ -740,9 +722,9 @@ def chatbot(user_id: str, user_input: str): "You can now discuss or improve the project." ) - # ========================================= - # FEATURE GENERATION - # ========================================= + + + if any(w in text for w in feature_words): state["waiting_for_project_action"] = False @@ -754,10 +736,9 @@ def chatbot(user_id: str, user_input: str): return chatbot(user_id, "generate features") - - # ========================================= - # PROJECT DISCUSSION - # ========================================= + + + elif any(w in text for w in chat_words): state["waiting_for_project_action"] = False @@ -775,15 +756,14 @@ def chatbot(user_id: str, user_input: str): "You can now discuss or improve the project." ) - - # ========================================= - # FULL PROJECT GENERATION - # ========================================= + + + elif any(w in text for w in full_project_words): state["waiting_for_project_action"] = False - # Generate features first if missing + if not state.get("features"): feature_result = generate_features( @@ -804,7 +784,7 @@ def chatbot(user_id: str, user_input: str): [] ) - # Generate full project + result = generate_full_project( title=state.get("project_title"), @@ -873,9 +853,9 @@ def chatbot(user_id: str, user_input: str): user_id ) - # ================================================= - # WEAK TITLE IMPROVEMENT HANDLER - # ================================================= + + + if state.get("weak_title_candidate"): improve_words = [ @@ -933,16 +913,16 @@ def chatbot(user_id: str, user_input: str): ) - # ================================================= - # WAITING FOR FEATURE TITLE - # ================================================= + + + if state.get("waiting_for_feature_title"): state["waiting_for_feature_title"] = False possible_title = user_input.strip() - # weak title + if is_weak_project_title(possible_title): state["weak_title_candidate"] = possible_title @@ -962,7 +942,7 @@ def chatbot(user_id: str, user_input: str): "2️⃣ Enter another title" ) - # valid title + state["project_title"] = possible_title save_user_memory(user_id, { @@ -972,11 +952,11 @@ def chatbot(user_id: str, user_input: str): return chatbot(user_id, "generate features") - # ================================================= - # DIRECT FEATURE TITLE EXTRACTION - # ================================================= + + + feature_patterns = [ - # standard + "generate features for", "generate feature for", "features for", @@ -984,36 +964,36 @@ def chatbot(user_id: str, user_input: str): "create features for", "suggest features for", - # casual + "give me features for", "show features for", "i need features for", "can you generate features for", "can you suggest features for", - # project related + "add features to", "improve features for", "feature ideas for", "smart features for", - # short forms + "features:", "project:", "idea:", - # typo tolerance + "gen features for", "generate faetures for", "fetures for", - # direct requests + "make features for", "build features for", "what features for", "best features for", - # ai chatbot style + "i have project", "my project is", "project title is", @@ -1041,9 +1021,9 @@ def chatbot(user_id: str, user_input: str): break - # ================================================= - # SIMPLE PROJECT TITLE DETECTION (BEFORE LLM) - # ================================================= + + + if analysis is None: blocked_starts = ( @@ -1078,7 +1058,7 @@ def chatbot(user_id: str, user_input: str): command_phrases = { - # idea commands + "new idea", "another idea", "generate idea", @@ -1087,18 +1067,18 @@ def chatbot(user_id: str, user_input: str): "more ideas", "idea", - # feature commands + "generate features", "generate feature", "more features", "features", - # full project + "generate full project", "full project", "complete project", - # generic commands + "another", "again", "more", @@ -1111,7 +1091,7 @@ def chatbot(user_id: str, user_input: str): clean_text = text.strip() - # detect short titles directly + words = clean_text.lower().split() if ( @@ -1133,9 +1113,9 @@ def chatbot(user_id: str, user_input: str): else: analysis = analyze_user_input(user_input, state) - # ================================================= - # IDEA SELECTION DETECTION - # ================================================= + + + existing_ideas = state.get("ideas", []) normalized_input = text.strip().lower() @@ -1146,7 +1126,7 @@ def chatbot(user_id: str, user_input: str): input_words = set(normalized_input.split()) - # ignore extremely short inputs + if len(input_words) < 2: continue @@ -1202,10 +1182,9 @@ Choose what you want: user_id ) - - # ===================================== - # NORMAL IDEA SELECTION - # ===================================== + + + state["project_title"] = idea state["waiting_for_project_action"] = True state["project_chat_mode"] = False @@ -1225,25 +1204,25 @@ Choose what you want: "- discuss project" ) - # ================================================= - # FOLLOW-UP CONTINUATION - # ================================================= + + + if any(w in text for w in FOLLOWUP_WORDS): - # continue ideas + if state.get("last_action") == "idea": analysis["intent"] = "idea" analysis["domain"] = state.get("domain") - # continue features + elif state.get("last_action") == "feature": analysis["intent"] = "feature" - # ================================================= - # DOMAIN FOLLOW-UP - # ================================================= + + + if state.get("waiting_for_domain"): detected = extract_domain(user_input) @@ -1268,9 +1247,9 @@ Choose what you want: user_project = analysis.get("project_title") - # ================================================= - # FULL PROJECT DOMAIN HANDLER - # ================================================= + + + if state.get("waiting_for_full_project_domain"): domain = extract_domain(user_input) @@ -1310,15 +1289,15 @@ Choose what you want: for idea in ideas ) ) - # ================================================= - # FALLBACK DOMAIN - # ================================================= + + + if not domain: domain = extract_domain(text) - # ================================================= - # DOMAIN-ONLY INPUT DETECTION - # ================================================= + + + full_project_commands = [ "fill all", "fill all columns", @@ -1340,9 +1319,9 @@ Choose what you want: and not any(cmd in text for cmd in full_project_commands) ): intent = "idea" - # ================================================= - # FALLBACK PROJECT TITLE DETECTION - # ================================================= + + + if ( not user_project and intent == "chat" @@ -1364,7 +1343,7 @@ Choose what you want: "more ideas", ] - # reject obvious commands/questions + blocked_starts = ( "how ", "what ", @@ -1411,16 +1390,16 @@ Choose what you want: ) ): - # treat short non-question text as project title + user_project = user_input.strip() - # ================================================= - # USER PROVIDED PROJECT - # ================================================= + + + if user_project: - # ============================================= - # WEAK TITLE VALIDATION - # ============================================= + + + if is_weak_project_title(user_project): state["weak_title_candidate"] = user_project @@ -1465,9 +1444,9 @@ Choose what you want: "- discuss project\n" "- generate full project" ) - # ================================================= - # FEATURE REQUEST WITHOUT TITLE - # ================================================= + + + feature_request_phrases = [ "generate features", "generate feature", @@ -1498,9 +1477,9 @@ Choose what you want: "- AI-powered smart library recommendation system\n" "- Smart hospital emergency response platform" ) - # ================================================= - # IDEA GENERATION - # ================================================= + + + if intent == "idea": if not domain: @@ -1541,14 +1520,14 @@ Choose what you want: user_id ) - # ================================================= - # FEATURE GENERATION - # ================================================= + + + elif intent == "feature": - # ========================================= - # DIRECT PROJECT TITLE FROM USER INPUT - # ========================================= + + + if analysis.get("project_title"): state["project_title"] = ( @@ -1579,8 +1558,8 @@ Choose what you want: [] ) - # IMPORTANT: - # Replace old features completely + + state["features"] = features state["last_action"] = "feature" @@ -1594,11 +1573,10 @@ Choose what you want: user_id ) - - # ================================================= - # FULL PROJECT GENERATION - # ================================================= + + + elif any(x in text for x in [ "fill all", @@ -1692,9 +1670,9 @@ Choose what you want: user_id ) - # ================================================= - # POSSIBLE NEW PROJECT DETECTION - # ================================================= + + + elif ( state.get("project_title") and len(text.split()) >= 3 @@ -1716,7 +1694,7 @@ Choose what you want: / max(1, len(new_input_words)) ) - # low overlap may indicate new project/topic + if ( overlap_ratio < 0.2 and analysis.get("intent") == "chat" @@ -1765,9 +1743,9 @@ User: state, user_id ) - # ================================================= - # NON PROJECT FILTER - # ================================================= + + + if not is_project_related(text): return ( @@ -1778,9 +1756,9 @@ User: "- suggest technologies for fintech app\n" "- improve my graduation project" ) - # ================================================= - # GENERAL CHAT - # ================================================= + + + else: system_prompt = build_chat_prompt(state) @@ -1817,5 +1795,3 @@ User: state, user_id ) - - \ No newline at end of file diff --git a/src/recommendation_engine/command_handler.py b/src/recommendation_engine/command_handler.py index 95f428a7bbdc64877cb168921fb0574942068e67..a0670e2ffa2c07b428df8cc004494db90c225fa8 100644 --- a/src/recommendation_engine/command_handler.py +++ b/src/recommendation_engine/command_handler.py @@ -1,11 +1,5 @@ -# src/recommendation_engine/command_handler.py - import re - -# ===================================================== -# Command Detection (Improved) -# ===================================================== def is_command(text: str) -> bool: """ Detect if input is a system/terminal command @@ -16,44 +10,40 @@ def is_command(text: str) -> bool: text = text.strip().lower() - # ========================================= - # Exact commands - # ========================================= + + + if text in {"exit", "quit", "clear"}: return True - # ========================================= - # Command-like patterns (regex) - # ========================================= + + + command_patterns = [ - r"^python\s+\S+\.py", # python file.py - r"^pip\s+install", # pip install - r"^npm\s+install", # npm install - r"^node\s+\S+", # node file.js - r"^cd\s+.+", # cd folder - r"^ls\b", # ls - r"^dir\b", # dir - r"^git\s+.+", # git command - r"^sudo\s+.+", # sudo command - r".+\.py\s*$", # ends with file.py (ONLY if standalone) + r"^python\s+\S+\.py", + r"^pip\s+install", + r"^npm\s+install", + r"^node\s+\S+", + r"^cd\s+.+", + r"^ls\b", + r"^dir\b", + r"^git\s+.+", + r"^sudo\s+.+", + r".+\.py\s*$", ] for pattern in command_patterns: if re.match(pattern, text): return True - # ========================================= - # Inline execution attempts - # ========================================= + + + if "run" in text and re.search(r"\.py\b", text): return True return False - -# ===================================================== -# Command Handling -# ===================================================== def handle_command(text: str) -> str: """ Return safe response for command-like inputs @@ -61,26 +51,26 @@ def handle_command(text: str) -> str: text = text.lower().strip() - # exit commands + if text in {"exit", "quit"}: return "👋 Session ended. Start a new chat anytime." - # python execution + if re.search(r"python\s+\S+\.py", text): return ( "⚠️ This looks like a code execution command.\n" "I only help with graduation project ideas and development." ) - # package installation + if "pip install" in text or "npm install" in text: return ( "⚠️ Installation commands are outside my scope.\n" "I can help you design your graduation project instead." ) - # generic fallback + return ( "⚠️ This looks like a system command.\n" "Please ask about graduation projects (ideas, features, or system design)." - ) \ No newline at end of file + ) diff --git a/src/recommendation_engine/config.py b/src/recommendation_engine/config.py index f3095e7c7b290109f74d4be913f86d83c9fd68ee..8342af90f8ae438400b1ca517454f640df42a70b 100644 --- a/src/recommendation_engine/config.py +++ b/src/recommendation_engine/config.py @@ -1,27 +1,16 @@ -# src/recommendation_engine/config.py - import os from pathlib import Path from dotenv import load_dotenv -# ===================================================== -# Load Environment Variables -# ===================================================== BASE_DIR = Path(__file__).resolve().parent.parent.parent ENV_PATH = BASE_DIR / ".env" if ENV_PATH.exists(): load_dotenv(ENV_PATH) -# ===================================================== -# App Mode -# ===================================================== ENV = os.getenv("ENV", "development") DEBUG_MODE = os.getenv("DEBUG_MODE", "true").lower() == "true" -# ===================================================== -# Gemini API Config -# ===================================================== GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "") MODEL_CANDIDATES = [ @@ -31,22 +20,17 @@ MODEL_CANDIDATES = [ os.getenv("QUALITY_MODEL", "gemini-2.5-pro"), ] -# remove duplicates safely _seen = set() MODEL_CANDIDATES = [ m for m in MODEL_CANDIDATES if m and not (m in _seen or _seen.add(m)) ] -# ===================================================== -# Generation Settings -# ===================================================== IDEA_TEMPERATURE = float(os.getenv("IDEA_TEMPERATURE", 0.9)) FEATURE_TEMPERATURE = float(os.getenv("FEATURE_TEMPERATURE", 0.6)) CHAT_TEMPERATURE = float(os.getenv("CHAT_TEMPERATURE", 0.7)) INTENT_TEMPERATURE = float(os.getenv("INTENT_TEMPERATURE", 0.0)) -# Max tokens IDEA_MAX_TOKENS = int(os.getenv("IDEA_MAX_TOKENS", 800)) FEATURE_MAX_TOKENS = int(os.getenv("FEATURE_MAX_TOKENS", 600)) CHAT_MAX_TOKENS = int(os.getenv("CHAT_MAX_TOKENS", 700)) @@ -55,23 +39,14 @@ INTENT_MAX_TOKENS = int(os.getenv("INTENT_MAX_TOKENS", 20)) TOP_P = float(os.getenv("TOP_P", 0.95)) TOP_K = int(os.getenv("TOP_K", 40)) -# ===================================================== -# Retry / Timeout -# ===================================================== REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", 30)) MAX_RETRIES = int(os.getenv("MAX_RETRIES", 3)) RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", 2)) -# ===================================================== -# Recommendation Defaults -# ===================================================== DEFAULT_FEATURE_COUNT = int(os.getenv("DEFAULT_FEATURE_COUNT", 5)) DEFAULT_IDEA_COUNT = int(os.getenv("DEFAULT_IDEA_COUNT", 5)) GENERATION_BATCH_SIZE = int(os.getenv("GENERATION_BATCH_SIZE", 10)) -# ===================================================== -# Duplicate Detection -# ===================================================== IDEA_DUPLICATE_THRESHOLD = float( os.getenv("IDEA_DUPLICATE_THRESHOLD", 0.82) ) @@ -80,48 +55,27 @@ FEATURE_DUPLICATE_THRESHOLD = float( os.getenv("FEATURE_DUPLICATE_THRESHOLD", 0.75) ) -# ===================================================== -# Idea Generator Control -# ===================================================== MAX_IDEA_RETRIES = int(os.getenv("MAX_IDEA_RETRIES", 3)) MAX_IDEA_LIMIT = int(os.getenv("MAX_IDEA_LIMIT", 20)) -# ===================================================== -# Similarity Engine -# ===================================================== SIMILARITY_TOP_K = int(os.getenv("SIMILARITY_TOP_K", 5)) -# ===================================================== -# State / Memory Control -# ===================================================== MAX_HISTORY = int(os.getenv("MAX_HISTORY", 20)) MAX_FEATURES = int(os.getenv("MAX_FEATURES", 20)) MAX_IDEAS = int(os.getenv("MAX_IDEAS", 10)) -# ===================================================== -# Semantic Routing -# ===================================================== ENABLE_SEMANTIC_INTENT = os.getenv( "ENABLE_SEMANTIC_INTENT", "true" ).lower() == "true" -# ===================================================== -# Logging -# ===================================================== ENABLE_LOGGING = os.getenv("ENABLE_LOGGING", "true").lower() == "true" LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") -# ===================================================== -# Safety Check (🔥 IMPORTANT) -# ===================================================== if not GEMINI_API_KEY: raise ValueError( "❌ GEMINI_API_KEY is missing. Please set it in .env file." ) -# ===================================================== -# Debug Print -# ===================================================== if DEBUG_MODE and ENV == "development": print("\n CONFIG LOADED:") print(f"ENV: {ENV}") @@ -129,4 +83,4 @@ if DEBUG_MODE and ENV == "development": print(f"MODELS: {MODEL_CANDIDATES}") print(f"MAX_RETRIES: {MAX_RETRIES}") print(f"IDEA_TEMP: {IDEA_TEMPERATURE}") - print("=================================\n") \ No newline at end of file + print("=================================\n") diff --git a/src/recommendation_engine/context_builder.py b/src/recommendation_engine/context_builder.py index 9bc9dd1d8336b49f1f671bc26000f4a73430cfcd..7b98200ba99c01fa7560b1c9923d37ebc5632e6e 100644 --- a/src/recommendation_engine/context_builder.py +++ b/src/recommendation_engine/context_builder.py @@ -1,5 +1,3 @@ -# src/recommendation_engine/context_builder.py - import logging import re @@ -22,10 +20,6 @@ from src.recommendation_engine.config import ( logger = logging.getLogger(__name__) - -# ===================================================== -# DOMAIN DEFINITIONS -# ===================================================== DOMAIN_KEYWORDS = { "artificial intelligence": [ "ai", @@ -104,10 +98,6 @@ DOMAIN_KEYWORDS = { ] } - -# ===================================================== -# Helpers -# ===================================================== def normalize(text: str) -> str: text = str(text).lower().strip() @@ -118,7 +108,6 @@ def normalize(text: str) -> str: return text - def clean_list( items: List[str], limit: int = 20 @@ -143,10 +132,6 @@ def clean_list( return final[:limit] - -# ===================================================== -# Multi-domain detection -# ===================================================== def detect_domains(text: str) -> List[str]: text = normalize(text) @@ -159,7 +144,7 @@ def detect_domains(text: str) -> List[str]: for w in words: - # phrase + if " " in w: if w in text: @@ -167,7 +152,7 @@ def detect_domains(text: str) -> List[str]: detected.append(domain) break - # word + else: if w in words_in_text: @@ -177,10 +162,6 @@ def detect_domains(text: str) -> List[str]: return clean_list(detected, limit=3) - -# ===================================================== -# Main domain extractor -# ===================================================== def extract_domain(text: str) -> str: if not text: @@ -188,20 +169,20 @@ def extract_domain(text: str) -> str: text = normalize(text) - # hard normalization + if text in ["ai", "ml"]: return "artificial intelligence" - # direct domain match + if text in DOMAIN_KEYWORDS: return text - # multi-domain detection + domains = detect_domains(text) if domains: - # prioritize non-general + for d in domains: if d != "general": @@ -209,7 +190,7 @@ def extract_domain(text: str) -> str: return domains[0] - # fuzzy typo matching + all_words = [] word_map = {} @@ -232,7 +213,7 @@ def extract_domain(text: str) -> str: if match: return word_map[match[0]] - # partial match + for domain, words in DOMAIN_KEYWORDS.items(): for w in words: @@ -243,10 +224,6 @@ def extract_domain(text: str) -> str: return "" - -# ===================================================== -# Similarity cache -# ===================================================== @lru_cache(maxsize=100) def cached_similarity( title: str, @@ -259,10 +236,6 @@ def cached_similarity( top_k=SIMILARITY_TOP_K ) - -# ===================================================== -# Common feature extraction -# ===================================================== def extract_common_features( results: pd.DataFrame ) -> List[str]: @@ -300,10 +273,6 @@ def extract_common_features( in counter.most_common(12) ] - -# ===================================================== -# Similar titles -# ===================================================== def extract_titles( results: pd.DataFrame ) -> List[str]: @@ -319,10 +288,6 @@ def extract_titles( return clean_list(titles, limit=10) - -# ===================================================== -# Architecture hints -# ===================================================== def build_architecture_hints( domains: List[str] ) -> List[str]: @@ -359,10 +324,6 @@ def build_architecture_hints( return clean_list(hints, limit=10) - -# ===================================================== -# Project Context -# ===================================================== def build_project_context( title: str, description: str, @@ -380,9 +341,9 @@ def build_project_context( f"{description}" ) - # ========================================= - # Domains - # ========================================= + + + domains = detect_domains(full_text) main_domain = ( @@ -391,9 +352,9 @@ def build_project_context( else "general" ) - # ========================================= - # Auto features - # ========================================= + + + auto_features = extract_features( full_text ) @@ -403,9 +364,9 @@ def build_project_context( MAX_FEATURES ) - # ========================================= - # Similarity search - # ========================================= + + + try: results = cached_similarity( @@ -421,9 +382,9 @@ def build_project_context( results = None - # ========================================= - # Empty fallback - # ========================================= + + + if ( not isinstance(results, pd.DataFrame) or len(results) == 0 @@ -443,9 +404,9 @@ def build_project_context( "context_strength": 0.0 } - # ========================================= - # Rich context - # ========================================= + + + similar_titles = extract_titles(results) common_features = extract_common_features( @@ -487,10 +448,6 @@ def build_project_context( "context_strength": round(context_strength, 4) } - -# ===================================================== -# Domain Context -# ===================================================== def build_domain_context( domain: str ) -> Dict[str, Any]: @@ -553,4 +510,4 @@ def build_domain_context( float(hybrid_scores.mean()), 4 ) - } \ No newline at end of file + } diff --git a/src/recommendation_engine/feature_generator.py b/src/recommendation_engine/feature_generator.py index 1177e5a2431bb9ff2bb7786fbbf528e150328e85..02bf738f2f59390470afe9c0f89ae48b3d19b8f0 100644 --- a/src/recommendation_engine/feature_generator.py +++ b/src/recommendation_engine/feature_generator.py @@ -1,5 +1,3 @@ -# src/recommendation_engine/feature_generator.py - import logging from typing import List, Dict, Any, Set @@ -20,20 +18,11 @@ logger = logging.getLogger(__name__) MAX_RETRIES = 5 -# 🔥 stronger duplicate control SIMILARITY_THRESHOLD_LOCAL = 0.82 - -# ===================================================== -# Helpers -# ===================================================== def normalize(text: str) -> str: return " ".join(str(text).strip().lower().split()) - -# ===================================================== -# Generic feature detector -# ===================================================== GENERIC_PATTERNS = [ "dashboard", "login", @@ -47,7 +36,6 @@ GENERIC_PATTERNS = [ "user management" ] - def is_generic_feature(text: str) -> bool: low = normalize(text) @@ -61,10 +49,6 @@ def is_generic_feature(text: str) -> bool: return False - -# ===================================================== -# Feature cleaner -# ===================================================== def clean_features(features: List[str]) -> List[str]: final = [] @@ -78,7 +62,7 @@ def clean_features(features: List[str]) -> List[str]: words = clean.split() - # 🔥 ideal feature size + if len(words) < 3 or len(words) > 10: continue @@ -89,10 +73,6 @@ def clean_features(features: List[str]) -> List[str]: return final - -# ===================================================== -# Semantic duplicate checker -# ===================================================== def is_duplicate_local(feature: str, existing: List[str]) -> bool: for old in existing: @@ -105,17 +85,13 @@ def is_duplicate_local(feature: str, existing: List[str]) -> bool: return False - -# ===================================================== -# Better contextual fallback -# ===================================================== def fallback_features(title: str) -> List[str]: title = (title or "").lower() - # ========================================= - # Healthcare - # ========================================= + + + if any(k in title for k in ["health", "hospital", "medical", "clinic"]): return [ @@ -126,9 +102,9 @@ def fallback_features(title: str) -> List[str]: "Predictive patient risk analysis" ] - # ========================================= - # Education - # ========================================= + + + if any(k in title for k in ["education", "learning", "student", "school"]): return [ @@ -139,9 +115,9 @@ def fallback_features(title: str) -> List[str]: "Personalized study path generation" ] - # ========================================= - # Security - # ========================================= + + + if any(k in title for k in ["security", "cyber", "threat"]): return [ @@ -152,9 +128,9 @@ def fallback_features(title: str) -> List[str]: "Risk prediction analytics" ] - # ========================================= - # Default - # ========================================= + + + return [ "Real-time intelligent monitoring", "Predictive analytics engine", @@ -163,10 +139,6 @@ def fallback_features(title: str) -> List[str]: "Dynamic performance optimization" ] - -# ===================================================== -# Main Generator -# ===================================================== def generate_features( title: str, description: str, @@ -183,9 +155,9 @@ def generate_features( logger.info(f"Starting feature generation | title={title}") - # ========================================= - # Build Context - # ========================================= + + + context = build_project_context( title=title, description=description, @@ -210,9 +182,9 @@ def generate_features( attempts = 0 - # ========================================= - # Generation Loop - # ========================================= + + + while len(final_features) < top_k and attempts < MAX_RETRIES: attempts += 1 @@ -224,18 +196,18 @@ def generate_features( GENERATION_BATCH_SIZE ) - # ===================================== - # Build Prompt - # ===================================== + + + prompt = build_feature_prompt( context=context, count=generation_count, previous_features=previous_generated_features ) - # ===================================== - # LLM Generation - # ===================================== + + + raw_text = generate_text( prompt, task="feature" @@ -245,9 +217,9 @@ def generate_features( logger.warning("Empty feature response") continue - # ===================================== - # Validation - # ===================================== + + + generated = validate_generated_list( text=raw_text, top_k=generation_count @@ -257,9 +229,9 @@ def generate_features( logger.info(f"Generated {len(generated)} candidate features") - # ===================================== - # Filtering - # ===================================== + + + for feat in generated: norm = normalize(feat) @@ -267,7 +239,7 @@ def generate_features( if not norm: continue - # exact duplicate + if ( norm in final_norm_set or norm in existing_norm @@ -275,11 +247,11 @@ def generate_features( ): continue - # semantic novelty + if not is_feature_novel(feat, existing_features): continue - # local semantic duplicate + if is_duplicate_local(feat, final_features): continue @@ -291,9 +263,9 @@ def generate_features( if len(final_features) >= top_k: break - # ========================================= - # Fallback - # ========================================= + + + if len(final_features) < top_k: logger.warning("Using fallback features") @@ -323,4 +295,4 @@ def generate_features( "recommended_features": final_features, "originality_score": context.get("originality_score", 1.0), "similar_projects": context.get("similar_titles", []) - } \ No newline at end of file + } diff --git a/src/recommendation_engine/full_project_generator.py b/src/recommendation_engine/full_project_generator.py index c2911c9ee707d98cfbf9663a2a53dc06370ee243..cf3fd716c10a8ffd4275c4dab4eba2e37cc4c809 100644 --- a/src/recommendation_engine/full_project_generator.py +++ b/src/recommendation_engine/full_project_generator.py @@ -5,10 +5,6 @@ from src.recommendation_engine.prompt_builder import ( import re - -# ===================================================== -# SECTION EXTRACTOR -# ===================================================== def extract_section(text, section_name): text = text.strip() @@ -48,10 +44,6 @@ def extract_section(text, section_name): return text[start:end].strip() - -# ===================================================== -# BULLET PARSER -# ===================================================== def parse_bullets(text): lines = text.splitlines() @@ -76,10 +68,6 @@ def parse_bullets(text): return final - -# ===================================================== -# FULL PROJECT GENERATOR -# ===================================================== def generate_full_project( title, features, @@ -103,9 +91,9 @@ def generate_full_project( result = { - # ============================================= - # BASIC - # ============================================= + + + "project_title": title, "category": @@ -117,9 +105,9 @@ def generate_full_project( "description": extract_section(raw, "DESCRIPTION"), - # ============================================= - # LISTS - # ============================================= + + + "technologies": parse_bullets( extract_section(raw, "TECHNOLOGIES") @@ -140,9 +128,9 @@ def generate_full_project( extract_section(raw, "FUTURE_WORK") ), - # ============================================= - # TEXT FIELDS - # ============================================= + + + "problem_statement": extract_section( raw, @@ -168,9 +156,9 @@ def generate_full_project( ) } - # ================================================= - # FALLBACKS - # ================================================= + + + if not result.get("category"): result["category"] = "General AI System" @@ -224,4 +212,4 @@ def generate_full_project( f"automation, monitoring, and predictive analysis." ) - return result \ No newline at end of file + return result diff --git a/src/recommendation_engine/idea_generator.py b/src/recommendation_engine/idea_generator.py index b97cf64830c0b187f86a72e27d328af7e3c99f75..53a59f53e639e696f3d6acd5a3d46198af0ccba5 100644 --- a/src/recommendation_engine/idea_generator.py +++ b/src/recommendation_engine/idea_generator.py @@ -1,5 +1,3 @@ -# src/recommendation_engine/idea_generator.py - import logging from typing import Dict, Any, List, Set @@ -16,16 +14,11 @@ logger = logging.getLogger(__name__) MAX_RETRIES = 5 SIMILARITY_THRESHOLD_LOCAL = 0.60 - -# ========================================= -# 🔥 LOAD DATASET TITLES (ONCE) -# ========================================= DATA_TITLES = set() def normalize_idea(text: str) -> str: return " ".join(text.lower().strip().split()) - def init_dataset_titles(): global DATA_TITLES @@ -39,24 +32,15 @@ def init_dataset_titles(): init_dataset_titles() - -# ========================================= -# HELPERS -# ========================================= def clean_ideas(ideas: List[str]) -> List[str]: return [i.strip() for i in ideas if 2 <= len(i.split()) <= 8] - def is_duplicate_local(idea: str, existing: Set[str]) -> bool: for old in existing: if compare_two_ideas(idea, old) >= SIMILARITY_THRESHOLD_LOCAL: return True return False - -# ========================================= -# FALLBACK -# ========================================= def fallback_by_domain(domain: str) -> List[str]: domain = (domain or "general").lower() @@ -93,10 +77,6 @@ def fallback_by_domain(domain: str) -> List[str]: "Predictive analytics engine" ]) - -# ========================================= -# MAIN GENERATOR -# ========================================= def generate_ideas( domain: str, top_k: int = DEFAULT_IDEA_COUNT, @@ -120,9 +100,9 @@ def generate_ideas( all_generated: List[str] = [] attempts = 0 - # ========================================= - # GENERATION LOOP - # ========================================= + + + while len(final_ideas) < top_k and attempts < MAX_RETRIES: attempts += 1 @@ -152,9 +132,9 @@ def generate_ideas( logger.info(f"Generated {len(generated)} ideas") all_generated.extend(generated) - # ========================================= - # FILTERING - # ========================================= + + + for idea in generated: normalized = normalize_idea(idea) @@ -162,15 +142,15 @@ def generate_ideas( if not normalized: continue - # ❌ dataset duplicate - # 🔥 dataset exact match + + if normalized in DATA_TITLES: logger.info(f"[SKIP DATASET EXACT] {idea}") continue - # 🔥 dataset semantic match + skip_dataset_similar = False - for data_title in list(DATA_TITLES)[:500]: # limit for speed + for data_title in list(DATA_TITLES)[:500]: if compare_two_ideas(idea, data_title) >= 0.85: logger.info(f"[SKIP DATASET SIMILAR] {idea}") skip_dataset_similar = True @@ -179,17 +159,16 @@ def generate_ideas( if skip_dataset_similar: continue - # ❌ duplicate inside current batch + if normalized in final_set: continue - # ❌ duplicate from previous session + if normalized in previous_norm: logger.info(f"[SKIP PREVIOUS] {idea}") continue - - # 🔥 semantic check with previous ideas + skip_similar_prev = False for old in previous_generated_ideas: if compare_two_ideas(idea, old) >= 0.85: @@ -200,12 +179,12 @@ def generate_ideas( if skip_similar_prev: continue - # ❌ semantic duplicate + if is_duplicate_local(idea, final_set): logger.info(f"[SKIP SIMILAR] {idea}") continue - # ✅ accept idea + logger.info(f"[NEW IDEA] {idea}") final_ideas.append(idea) @@ -214,9 +193,9 @@ def generate_ideas( if len(final_ideas) >= top_k: break - # ========================================= - # FALLBACK - # ========================================= + + + if len(final_ideas) < top_k: logger.warning("Using fallback ideas") @@ -240,4 +219,4 @@ def generate_ideas( "domain": domain, "generated_ideas": all_generated, "final_ideas": final_ideas - } \ No newline at end of file + } diff --git a/src/recommendation_engine/intent_classifier.py b/src/recommendation_engine/intent_classifier.py index 37bc23a034e42d5fa147b4543663561b115f9269..68e990615a7733bd47902f4a36e4a08679df7940 100644 --- a/src/recommendation_engine/intent_classifier.py +++ b/src/recommendation_engine/intent_classifier.py @@ -1,5 +1,3 @@ -# src/recommendation_engine/intent_classifier.py - import logging from typing import Dict @@ -7,10 +5,6 @@ from src.recommendation_engine.llm_client import generate_text logger = logging.getLogger(__name__) - -# ========================================= -# VALID INTENTS -# ========================================= VALID_INTENTS = { "idea", "feature", @@ -18,10 +12,6 @@ VALID_INTENTS = { "chat" } - -# ========================================= -# SAFE NORMALIZER -# ========================================= def normalize_intent(text: str) -> str: if not text: @@ -34,10 +24,6 @@ def normalize_intent(text: str) -> str: return "chat" - -# ========================================= -# LLM INTENT CLASSIFIER -# ========================================= def classify_with_llm(user_input: str, state: Dict) -> str: prompt = f""" @@ -76,10 +62,6 @@ Rules: logger.warning(f"[INTENT ERROR] {e}") return "chat" - -# ========================================= -# HYBRID DETECTION (FINAL 🔥) -# ========================================= def detect_intent(text: str, state: dict = None) -> str: if state is None: @@ -90,48 +72,47 @@ def detect_intent(text: str, state: dict = None) -> str: has_project = bool(state.get("project_title")) has_features = bool(state.get("features")) - # ========================================= - # HARD RULES (NO FLOW FORCING ❌) - # ========================================= + + + - # explicit idea requests + if any(x in text_clean for x in [ "idea", "project idea", "new idea", "another idea", "suggest" ]): return "idea" - # explicit feature requests + if "feature" in text_clean: return "feature" - - # full project + if any(x in text_clean for x in [ "full project", "complete", "all details" ]): return "full_project" - # ========================================= - # USER PROVIDED IDEA (🔥 important) - # ========================================= - # If user writes sentence like a project idea + + + + if not has_project and len(text_clean.split()) >= 3: return "feature" - # ========================================= - # LLM fallback - # ========================================= + + + intent = classify_with_llm(text, state) logger.info(f"[INTENT] {intent}") - # ========================================= - # SAFETY FIXES - # ========================================= + + + if intent == "feature" and not has_project: - return "feature" # user likely gave idea + return "feature" if intent == "description" and not has_project: return "idea" - return intent \ No newline at end of file + return intent diff --git a/src/recommendation_engine/llm_client.py b/src/recommendation_engine/llm_client.py index ed2d8c97bc645f2debcb90f35ddcf24eb9d95dc3..0b04de026be1557e814829780a5a6c8e3103606a 100644 --- a/src/recommendation_engine/llm_client.py +++ b/src/recommendation_engine/llm_client.py @@ -1,5 +1,3 @@ -# src/recommendation_engine/llm_client.py - import time import logging from typing import List @@ -26,9 +24,6 @@ from src.recommendation_engine.config import ( from src.recommendation_engine.validator import validate_generated_list -# ========================================= -# Logging -# ========================================= logger = logging.getLogger(__name__) if ENABLE_LOGGING: @@ -37,14 +32,12 @@ if ENABLE_LOGGING: format="%(asctime)s | %(levelname)s | %(message)s" ) - class LLMProviderError(Exception): def __init__(self, message: str, status_code: int = 503): super().__init__(message) self.message = message self.status_code = status_code - def classify_provider_error(error: Exception): text = str(error).lower() @@ -72,26 +65,19 @@ def classify_provider_error(error: Exception): return None -# ========================================= -# Init Gemini -# ========================================= client = genai.Client(api_key=GEMINI_API_KEY) - -# ========================================= -# Extract Text (Robust) -# ========================================= def extract_text(response) -> str: if not response: return "" - # direct + text = getattr(response, "text", None) if text: return text.strip() - # fallback + try: candidates = getattr(response, "candidates", []) if candidates: @@ -104,10 +90,6 @@ def extract_text(response) -> str: return "" - -# ========================================= -# Temperature Selector -# ========================================= def get_temperature(task: str) -> float: return { @@ -116,10 +98,6 @@ def get_temperature(task: str) -> float: "intent": INTENT_TEMPERATURE, }.get(task, CHAT_TEMPERATURE) - -# ========================================= -# Max Tokens Selector -# ========================================= def get_max_tokens(task: str) -> int: return { @@ -128,17 +106,9 @@ def get_max_tokens(task: str) -> int: "intent": INTENT_MAX_TOKENS, }.get(task, CHAT_MAX_TOKENS) - -# ========================================= -# Prompt Safety -# ========================================= def safe_prompt(prompt: str, max_chars: int = 12000) -> str: return prompt[-max_chars:] - -# ========================================= -# Response Quality Check (🔥 NEW) -# ========================================= def is_bad_response(text: str) -> bool: if not text: @@ -146,11 +116,11 @@ def is_bad_response(text: str) -> bool: text = text.strip() - # ❌ only reject VERY bad outputs + if len(text) < 3: return True - # ❌ reject only pure assistant chatter + bad_phrases = [ "as an ai", "i can help you", @@ -164,10 +134,6 @@ def is_bad_response(text: str) -> bool: return False - -# ========================================= -# Main Generate Function -# ========================================= def generate_text( prompt: str, task: str = "chat", @@ -202,10 +168,10 @@ def generate_text( text = extract_text(response) - # 🔥 skip garbage responses + if is_bad_response(text): logger.warning("[LLM] Weak response, using anyway") - return text # 🔥 DO NOT RETRY + return text return text @@ -216,7 +182,7 @@ def generate_text( if provider_error: raise provider_error - # exponential backoff + time.sleep(RETRY_DELAY_SECONDS * (attempt + 1)) logger.info(f"[LLM] switching model...") @@ -225,10 +191,6 @@ def generate_text( return "" - -# ========================================= -# Generate List (🔥 FIXED) -# ========================================= def generate_list(prompt: str, task="chat") -> List[str]: text = generate_text(prompt, task=task) diff --git a/src/recommendation_engine/llm_router.py b/src/recommendation_engine/llm_router.py index c06dc99935e838f5ce0f53bf46f2c550cc7d8eb4..87239dc525e7da28b0443f7568eb966ec0326f6c 100644 --- a/src/recommendation_engine/llm_router.py +++ b/src/recommendation_engine/llm_router.py @@ -1,7 +1,6 @@ import re from src.recommendation_engine.context_builder import extract_domain - def word_to_number(text: str): mapping = { "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, @@ -20,14 +19,13 @@ def word_to_number(text: str): return None - def analyze_user_input(user_input: str, state: dict) -> dict: text = user_input.lower() - # ============================= - # INTENT DETECTION (RULE BASED) - # ============================= + + + if any(w in text for w in ["idea", "project", "suggest", "recommend"]): intent = "idea" @@ -40,19 +38,19 @@ def analyze_user_input(user_input: str, state: dict) -> dict: else: intent = "chat" - # ============================= - # DOMAIN DETECTION - # ============================= + + + domain = extract_domain(text) - # ============================= - # NUMBER DETECTION (🔥 NEW) - # ============================= + + + number = word_to_number(text) - # ============================= - # PROJECT TITLE DETECTION - # ============================= + + + project_title = None if intent == "chat" and len(text.split()) > 4: @@ -63,4 +61,4 @@ def analyze_user_input(user_input: str, state: dict) -> dict: "domain": domain, "project_title": project_title, "number": number - } \ No newline at end of file + } diff --git a/src/recommendation_engine/local_llm.py b/src/recommendation_engine/local_llm.py index 4210e476ea7d957ef821acbe58abe86b51077057..88bdae42348df7ef0417bd97f979e176c2d9cd7f 100644 --- a/src/recommendation_engine/local_llm.py +++ b/src/recommendation_engine/local_llm.py @@ -1,17 +1,15 @@ - import requests import time OLLAMA_URL = "http://localhost:11434/api/generate" MODEL_NAME = "mistral" - def generate_local(prompt: str, retries: int = 2) -> str: """ Stable + fast local LLM call (Ollama) """ - # 🔥 cut prompt (مهم جدًا للسرعة) + prompt = prompt[:1200] payload = { @@ -20,7 +18,7 @@ def generate_local(prompt: str, retries: int = 2) -> str: "stream": False, "options": { "temperature": 0.7, - "num_predict": 150, # يقلل الوقت + "num_predict": 150, "top_p": 0.9 } } @@ -37,13 +35,13 @@ def generate_local(prompt: str, retries: int = 2) -> str: duration = round(time.time() - start_time, 2) - # ❌ bad status + if response.status_code != 200: print(f"❌ Local status {response.status_code}") time.sleep(1) continue - # ❌ invalid json + try: data = response.json() except Exception: @@ -53,7 +51,7 @@ def generate_local(prompt: str, retries: int = 2) -> str: result = data.get("response", "").strip() - # ❌ empty response + if not result or len(result) < 10: print(f"⚠️ Empty/short response ({duration}s)") time.sleep(1) @@ -70,7 +68,6 @@ def generate_local(prompt: str, retries: int = 2) -> str: print("❌ Local error:", e) time.sleep(1) - # ❌ failed completely + print("🚨 Local LLM failed → fallback") return "" - diff --git a/src/recommendation_engine/memory_store.py b/src/recommendation_engine/memory_store.py index 4175de35598edc42f1357e67c7fc76b08f86c1ea..24e983f6f39e0841655a5ee969d4ea913991e989 100644 --- a/src/recommendation_engine/memory_store.py +++ b/src/recommendation_engine/memory_store.py @@ -1,19 +1,13 @@ -# src/recommendation_engine/memory_store.py - import time import uuid from copy import deepcopy MEMORY = {} -TTL = 86400 # 24 hours +TTL = 86400 MAX_HISTORY = 20 MAX_CHATS = 10 - -# ===================================================== -# Default State -# ===================================================== def default_state(): return { "project_title": "", @@ -31,10 +25,6 @@ def default_state(): "category": "" } - -# ===================================================== -# Create New Chat -# ===================================================== def create_chat(user_id: str): user = MEMORY.setdefault(user_id, { @@ -62,10 +52,6 @@ def create_chat(user_id: str): return chat_id - -# ===================================================== -# Get Active Chat -# ===================================================== def get_user_memory(user_id: str): user = MEMORY.get(user_id) @@ -74,7 +60,7 @@ def get_user_memory(user_id: str): create_chat(user_id) user = MEMORY[user_id] - # TTL reset + if time.time() - user["timestamp"] > TTL: MEMORY[user_id] = {} create_chat(user_id) @@ -87,10 +73,6 @@ def get_user_memory(user_id: str): return user["chats"][chat_id] - -# ===================================================== -# Merge State (FIXED 🔥) -# ===================================================== def merge_state(old: dict, new: dict): merged = deepcopy(old) @@ -100,12 +82,12 @@ def merge_state(old: dict, new: dict): if value is None: continue - # 🔥 overwrite critical fields + if key in ["project_title", "description"]: merged[key] = value continue - # 🔥 lists → replace if empty OR extend uniquely + if isinstance(value, list): if not value: @@ -123,7 +105,7 @@ def merge_state(old: dict, new: dict): seen.add(norm) combined.append(item) - # limit growth + merged[key] = combined[:20] else: @@ -131,10 +113,6 @@ def merge_state(old: dict, new: dict): return merged - -# ===================================================== -# Save Chat -# ===================================================== def save_user_memory(user_id: str, data: dict): user = MEMORY.get(user_id) @@ -157,10 +135,6 @@ def save_user_memory(user_id: str, data: dict): user["timestamp"] = time.time() - -# ===================================================== -# Get All Chats -# ===================================================== def get_all_chats(user_id: str): user = MEMORY.get(user_id, {}) @@ -175,10 +149,6 @@ def get_all_chats(user_id: str): for chat_id, chat in chats.items() ] - -# ===================================================== -# Switch Chat -# ===================================================== def switch_chat(user_id: str, chat_id: str): user = MEMORY.get(user_id) @@ -189,11 +159,7 @@ def switch_chat(user_id: str, chat_id: str): if chat_id in user.get("chats", {}): user["active_chat_id"] = chat_id - -# ===================================================== -# Clear User -# ===================================================== def clear_user_memory(user_id: str): if user_id in MEMORY: - del MEMORY[user_id] \ No newline at end of file + del MEMORY[user_id] diff --git a/src/recommendation_engine/novelty_checker.py b/src/recommendation_engine/novelty_checker.py index bf3aed07ee7e41d2284a0bf793c02b7fd0adff92..dfe185a416595dd06fb0ce8185bf88a6d39c63b9 100644 --- a/src/recommendation_engine/novelty_checker.py +++ b/src/recommendation_engine/novelty_checker.py @@ -1,5 +1,3 @@ -# src/recommendation_engine/novelty_checker.py - import logging import re @@ -18,10 +16,6 @@ from src.recommendation_engine.config import ( logger = logging.getLogger(__name__) - -# ===================================================== -# Generic weak patterns -# ===================================================== GENERIC_PATTERNS = [ "dashboard", "platform", @@ -34,10 +28,6 @@ GENERIC_PATTERNS = [ "management system" ] - -# ===================================================== -# Text normalization -# ===================================================== def normalize(text: str) -> str: text = str(text).lower().strip() @@ -48,10 +38,6 @@ def normalize(text: str) -> str: return text - -# ===================================================== -# Generic detector -# ===================================================== def is_generic(text: str) -> bool: low = normalize(text) @@ -63,10 +49,6 @@ def is_generic(text: str) -> bool: return False - -# ===================================================== -# Token overlap similarity -# ===================================================== def token_overlap_score(a: str, b: str) -> float: a_tokens = set(normalize(a).split()) @@ -82,10 +64,6 @@ def token_overlap_score(a: str, b: str) -> float: return overlap / union - -# ===================================================== -# Feature Novelty -# ===================================================== def is_feature_novel( feature: str, existing_features: List[str] @@ -96,18 +74,18 @@ def is_feature_novel( if not feature: return False - # ========================================= - # Reject generic weak features - # ========================================= + + + if is_generic(feature): logger.debug(f"[GENERIC FEATURE] {feature}") return False - # ========================================= - # Exact duplicate - # ========================================= + + + existing_norm = [ normalize(f) for f in existing_features @@ -119,27 +97,27 @@ def is_feature_novel( return False - # ========================================= - # Semantic comparison - # ========================================= + + + for old in existing_norm: if not old: continue - # embedding similarity + semantic_score = compare_two_ideas( feature, old ) - # token overlap + overlap_score = token_overlap_score( feature, old ) - # combined reasoning + final_score = max( semantic_score, overlap_score @@ -151,7 +129,7 @@ def is_feature_novel( f"overlap={overlap_score:.2f})" ) - # stricter threshold + if final_score >= ( FEATURE_DUPLICATE_THRESHOLD + 0.08 ): @@ -165,10 +143,6 @@ def is_feature_novel( return True - -# ===================================================== -# Duplicate Feature Filter -# ===================================================== def filter_duplicate_features( generated_features: List[str], existing_features: List[str] @@ -202,10 +176,6 @@ def filter_duplicate_features( return final - -# ===================================================== -# Cached DB originality check -# ===================================================== @lru_cache(maxsize=256) def _cached_db_check(idea: str) -> float: @@ -234,20 +204,16 @@ def _cached_db_check(idea: str) -> float: if scores: - # strongest similarity + return max(scores) except Exception as e: logger.warning(f"[DB ERROR] {e}") - # safe fallback + return 0.0 - -# ===================================================== -# Idea Novelty -# ===================================================== def is_idea_novel(idea_title: str) -> bool: idea_title = normalize(idea_title) @@ -271,10 +237,6 @@ def is_idea_novel(idea_title: str) -> bool: return score < IDEA_DUPLICATE_THRESHOLD - -# ===================================================== -# Optional novelty score -# ===================================================== def score_feature_novelty( feature: str, existing_features: List[str] @@ -311,4 +273,4 @@ def score_feature_novelty( return round( 1.0 - max(scores), 4 - ) \ No newline at end of file + ) diff --git a/src/recommendation_engine/prompt_builder.py b/src/recommendation_engine/prompt_builder.py index f160b384f2630a2afcad6a402a7c02d5c00f408b..f6f97e8cef79e26363c27609a00331fb2fa67fcd 100644 --- a/src/recommendation_engine/prompt_builder.py +++ b/src/recommendation_engine/prompt_builder.py @@ -1,11 +1,5 @@ -# src/recommendation_engine/prompt_builder.py - from typing import Dict, Any, List - -# ===================================================== -# Helpers -# ===================================================== def list_to_text(items: List[str], max_items: int = 10) -> str: if not items: @@ -29,10 +23,6 @@ def list_to_text(items: List[str], max_items: int = 10) -> str: return ", ".join(cleaned) - -# ===================================================== -# Feature Prompt -# ===================================================== def build_feature_prompt( context: Dict[str, Any], count: int = 10, @@ -108,10 +98,6 @@ BAD FEATURE EXAMPLES: OUTPUT: """.strip() - -# ===================================================== -# Idea Prompt -# ===================================================== def build_idea_prompt( context: Dict[str, Any], count: int = 10, @@ -177,10 +163,6 @@ BAD IDEA EXAMPLES: OUTPUT: """.strip() - -# ===================================================== -# Description Prompt -# ===================================================== def build_description_prompt(context: Dict[str, Any]) -> str: return f""" @@ -214,10 +196,6 @@ OUTPUT: Professional structured paragraph """.strip() - -# ===================================================== -# Chat Prompt -# ===================================================== def build_chat_prompt(context: Dict[str, Any]) -> str: return f""" @@ -250,10 +228,6 @@ RULES: - Keep all answers related to the current project """.strip() - -# ===================================================== -# Full Project Prompt -# ===================================================== def build_full_project_prompt(context): title = context.get("project_title", "") @@ -349,4 +323,4 @@ FUTURE_WORK: METHODOLOGY: (step-by-step implementation process) -""".strip() \ No newline at end of file +""".strip() diff --git a/src/recommendation_engine/response_formatter.py b/src/recommendation_engine/response_formatter.py index 2a19919901d75db14a839b8774a311df346c4dc1..ee73a2ae9dbaec8b828c5dc072a88540cb600c03 100644 --- a/src/recommendation_engine/response_formatter.py +++ b/src/recommendation_engine/response_formatter.py @@ -1,11 +1,5 @@ -# src/recommendation_engine/response_formatter.py - from typing import Dict, Any, List - -# ===================================================== -# Helpers -# ===================================================== def unique_items(items: List[str]) -> List[str]: seen = set() @@ -25,7 +19,6 @@ def unique_items(items: List[str]) -> List[str]: return final - def format_list( items: List[str], prefix: str = "- " @@ -41,7 +34,6 @@ def format_list( for item in items ) - def clean_text(text: str) -> str: if not text: @@ -49,7 +41,7 @@ def clean_text(text: str) -> str: text = str(text).strip() - # remove duplicated assistant prefixes + bad_prefixes = [ "assistant:", "🤖 assistant:", @@ -66,19 +58,14 @@ def clean_text(text: str) -> str: return text - def divider() -> str: return "━━━━━━━━━━━━━━━━━━━━━━" - -# ===================================================== -# Dynamic Next Step -# ===================================================== def next_step(state: Dict[str, Any]) -> str: - # ========================================= - # No project yet - # ========================================= + + + if not state.get("project_title"): return ( @@ -86,9 +73,9 @@ def next_step(state: Dict[str, Any]) -> str: "or specify a domain." ) - # ========================================= - # No features yet - # ========================================= + + + if not state.get("features"): return ( @@ -97,19 +84,15 @@ def next_step(state: Dict[str, Any]) -> str: "2️⃣ Generate another idea\n" ) - # ========================================= - # No description yet - # ========================================= - # ========================================= - # Full project exists - # ========================================= + + + + + + - -# ===================================================== -# Full Project Overview -# ===================================================== def format_full_project(state: Dict[str, Any]) -> str: title = state.get("project_title", "Not defined") @@ -156,10 +139,6 @@ def format_full_project(state: Dict[str, Any]) -> str: {context_strength} """.strip() - -# ===================================================== -# Main Formatter -# ===================================================== def format_response( intent: str, raw_response: str, @@ -180,9 +159,9 @@ def format_response( state.get("description", "") ) - # ================================================= - # IDEA RESPONSE - # ================================================= + + + if intent == "idea": if not ideas: @@ -197,9 +176,9 @@ def format_response( {next_step(state)} """.strip() - # ================================================= - # FEATURE RESPONSE - # ================================================= + + + if intent == "feature": if not features: @@ -215,9 +194,9 @@ def format_response( {next_step(state)} """.strip() - # ================================================= - # DESCRIPTION RESPONSE - # ================================================= + + + if intent == "description": return f""" @@ -231,9 +210,9 @@ def format_response( {next_step(state)} """.strip() - # ================================================= - # FULL PROJECT - # ================================================= + + + if intent == "full_project": return f""" @@ -243,9 +222,9 @@ def format_response( 👉 You can continue improving the project. """.strip() - # ================================================= - # CHAT RESPONSE - # ================================================= + + + cleaned = clean_text(raw_response) return f""" @@ -259,4 +238,4 @@ def format_response( {divider()} {next_step(state)} -""".strip() \ No newline at end of file +""".strip() diff --git a/src/recommendation_engine/semantic_intent_classifier.py b/src/recommendation_engine/semantic_intent_classifier.py index 338838824a1bf7e24c4ccfafb55b43beea7f882a..9104ca2b83c90694812ce344370a64871b249876 100644 --- a/src/recommendation_engine/semantic_intent_classifier.py +++ b/src/recommendation_engine/semantic_intent_classifier.py @@ -1,11 +1,5 @@ -# src/recommendation_engine/semantic_intent_classifier.py - from src.recommendation_engine.llm_client import generate_text - -# ===================================================== -# Allowed intents -# ===================================================== VALID_INTENTS = { "idea", "feature", @@ -13,10 +7,6 @@ VALID_INTENTS = { "chat" } - -# ===================================================== -# Semantic Intent Detection (🔥 upgraded) -# ===================================================== def detect_intent_semantic(user_input: str, state: dict) -> str: prompt = f""" @@ -56,17 +46,17 @@ Return ONLY ONE WORD from the list. result = generate_text(prompt, task="intent").lower().strip() - # ========================================= - # strict cleanup (🔥 important) - # ========================================= + + + result = result.split()[0].strip() if result in VALID_INTENTS: return result - # ========================================= - # fallback logic (🔥 critical safety) - # ========================================= + + + text = user_input.lower() if any(w in text for w in ["idea", "project", "suggest"]): @@ -78,4 +68,4 @@ Return ONLY ONE WORD from the list. if any(w in text for w in ["full", "all details", "complete"]): return "full_project" - return "chat" \ No newline at end of file + return "chat" diff --git a/src/recommendation_engine/state_manager.py b/src/recommendation_engine/state_manager.py index 9c0ca62ac7a4ffe21e6414b27effafcc055a88b8..42f01f8e09007317a1acfde1b789b52fb78f90c2 100644 --- a/src/recommendation_engine/state_manager.py +++ b/src/recommendation_engine/state_manager.py @@ -1,13 +1,8 @@ from typing import Dict, Any, List - -# ===================================================== -# HELPERS -# ===================================================== def normalize(text: str) -> str: return str(text).strip().lower() - def merge_lists(old: List[str], new: List[str]) -> List[str]: seen = set() @@ -29,10 +24,6 @@ def merge_lists(old: List[str], new: List[str]) -> List[str]: return result - -# ===================================================== -# SMART UPDATE STATE -# ===================================================== def update_state( state: Dict[str, Any], new_data: Dict[str, Any], @@ -47,9 +38,9 @@ def update_state( if value is None or value == "": continue - # ========================================= - # ALWAYS REPLACE - # ========================================= + + + if key in [ "ideas", "project_title", @@ -65,9 +56,9 @@ def update_state( state[key] = value continue - # ========================================= - # FEATURES - # ========================================= + + + if key == "features": if mode == "replace": @@ -80,9 +71,9 @@ def update_state( continue - # ========================================= - # LIST FIELDS - # ========================================= + + + if isinstance(value, list): if mode == "replace": @@ -98,9 +89,9 @@ def update_state( continue - # ========================================= - # STRING FIELDS - # ========================================= + + + if isinstance(value, str): clean_val = value.strip() @@ -119,17 +110,13 @@ def update_state( continue - # ========================================= - # OTHER TYPES - # ========================================= + + + state[key] = value return state - -# ===================================================== -# RESET HELPERS -# ===================================================== def reset_for_new_idea( state: Dict[str, Any] ) -> Dict[str, Any]: @@ -171,7 +158,6 @@ def reset_for_new_idea( "context_strength": None } - def reset_features_only( state: Dict[str, Any] ) -> Dict[str, Any]: @@ -180,4 +166,4 @@ def reset_features_only( state["description"] = "" - return state \ No newline at end of file + return state diff --git a/src/recommendation_engine/test.py b/src/recommendation_engine/test.py deleted file mode 100644 index 73f7df70d2a976ef4a1ad27bf800f23f1a89f65b..0000000000000000000000000000000000000000 --- a/src/recommendation_engine/test.py +++ /dev/null @@ -1,123 +0,0 @@ - -### idea generator test - -# from src.recommendation_engine.idea_generator import generate_ideas - -# result = generate_ideas( -# domain="healthcare", -# top_k=5 -# ) - -# print("\nFINAL IDEAS:") -# for i, idea in enumerate(result["final_ideas"], 1): -# print(f"{i}. {idea}") - -###featuregeneratoretest - -# from src.recommendation_engine.feature_generator import generate_features - -# # ========================================= -# # TEST 1: First generation -# # ========================================= -# print("\n========== FIRST RUN ==========\n") - -# first = generate_features( -# title="Smart Hospital System", -# description=""" -# AI-based hospital system with chatbot, -# patient management, QR booking, -# and smart recommendations. -# """, -# features=["chatbot", "search", "qr booking"], -# top_k=2 -# ) - -# for i, f in enumerate(first["recommended_features"], 1): -# print(f"{i}. {f}") - - -# # ========================================= -# # TEST 2: Second generation (NO duplicates) -# # ========================================= -# print("\n========== SECOND RUN (NO DUPLICATES) ==========\n") - -# second = generate_features( -# title="Smart Hospital System", -# description=""" -# AI-based hospital system with chatbot, -# patient management, QR booking, -# and smart recommendations. -# """, -# features=["chatbot", "search", "qr booking"], -# previous_generated_features=first["recommended_features"], -# top_k=2 -# ) - -# for i, f in enumerate(second["recommended_features"], 1): -# print(f"{i}. {f}") - - -# # ========================================= -# # TEST 3: Check overlap manually -# # ========================================= -# print("\n========== DUPLICATE CHECK ==========\n") - -# duplicates = set(first["recommended_features"]) & set(second["recommended_features"]) - -# if duplicates: -# print(" DUPLICATES FOUND:") -# for d in duplicates: -# print("-", d) -# else: -# print(" NO DUPLICATES — SYSTEM WORKING PERFECT") - - -## chatbot generator test - - - -# from src.recommendation_engine.chatbot_engine import chatbot - -# user_id = "user_1" - -# print("\n=== IDEA ===") -# print(chatbot(user_id, "I want healthcare project idea")) - -# print("\n=== FEATURES ===") -# print(chatbot(user_id, "add smart features")) - -# print("\n=== DESCRIPTION ===") -# print(chatbot(user_id, "describe it")) - -# print("\n=== MEMORY TEST ===") -# print(chatbot(user_id, "improve it more")) - - - - -# run_test.py - -from src.recommendation_engine.chatbot_engine import chatbot - -USER_ID = "test_user" - - - -def interactive(): - print("\n===== INTERACTIVE MODE =====\n") - - while True: - user_input = input("YOU: ") - - if user_input.lower() in ["exit", "quit"]: - break - - response = chatbot(USER_ID, user_input) - - print("\nBOT:") - print(response) - print("\n----------------------\n") - - - -interactive() \ No newline at end of file diff --git a/src/recommendation_engine/validator.py b/src/recommendation_engine/validator.py index 31127dbe4de0ffa32f398017f9b368d6e7c65555..66d2609b47be8f108479ff28f4766cb60ba4b7ee 100644 --- a/src/recommendation_engine/validator.py +++ b/src/recommendation_engine/validator.py @@ -1,12 +1,6 @@ -# src/recommendation_engine/validator.py - import re from typing import List - -# ===================================================== -# Generic / weak patterns -# ===================================================== GENERIC_PATTERNS = [ "dashboard", "login", @@ -27,7 +21,6 @@ GENERIC_PATTERNS = [ "user management" ] - BAD_STARTS = [ "here are", "below are", @@ -37,7 +30,6 @@ BAD_STARTS = [ "features include" ] - LOW_VALUE_WORDS = [ "system", "platform", @@ -46,10 +38,6 @@ LOW_VALUE_WORDS = [ "solution" ] - -# ===================================================== -# CLEAN TEXT -# ===================================================== def clean_text(text: str) -> str: if not text: @@ -57,40 +45,36 @@ def clean_text(text: str) -> str: text = str(text).strip() - # remove numbering + text = re.sub(r"^\d+[\)\.\-\s]+", "", text) - # remove bullets + text = re.sub(r"^[\-\*\•\→\▪\s]+", "", text) - # remove markdown + text = text.replace("**", "") - # remove quotes + text = text.replace('"', "").replace("'", "") - # remove brackets + text = re.sub(r"\(.*?\)", "", text) - # remove long explanations after colon + if ":" in text and len(text.split()) > 6: text = text.split(":")[0] - # remove assistant prefixes + text = re.sub(r"^(assistant|bot)\s*[:\-]\s*", "", text, flags=re.I) - # remove trailing punctuation + text = re.sub(r"[.,\-:;]+$", "", text) - # normalize spaces + text = re.sub(r"\s+", " ", text).strip() return text - -# ===================================================== -# NORMALIZATION -# ===================================================== def normalize_key(text: str) -> str: text = text.lower() @@ -101,10 +85,6 @@ def normalize_key(text: str) -> str: return text - -# ===================================================== -# Generic detector -# ===================================================== def is_generic(text: str) -> bool: low = normalize_key(text) @@ -115,29 +95,25 @@ def is_generic(text: str) -> bool: return False - -# ===================================================== -# Weak phrase detector -# ===================================================== def is_low_quality(text: str) -> bool: low = normalize_key(text) words = low.split() - # too short + if len(words) < 3: return True - # too long + if len(words) > 12: return True - # starts badly + if any(low.startswith(x) for x in BAD_STARTS): return True - # mostly weak words + weak_count = sum( 1 for w in words if w in LOW_VALUE_WORDS @@ -148,29 +124,21 @@ def is_low_quality(text: str) -> bool: return False - -# ===================================================== -# VALIDATION RULES -# ===================================================== def is_valid_item(text: str) -> bool: if not text: return False - # reject generic garbage + if is_generic(text): return False - # reject low-quality phrases + if is_low_quality(text): return False return True - -# ===================================================== -# FILTER ITEMS -# ===================================================== def filter_items(items: List[str]) -> List[str]: final = [] @@ -189,16 +157,16 @@ def filter_items(items: List[str]) -> List[str]: key = normalize_key(text) - # exact duplicate + if key in seen: continue - # semantic-ish duplicate + duplicate = False for old in seen: - # overlap similarity + overlap = set(key.split()) & set(old.split()) if len(overlap) >= max(2, min(len(key.split()), len(old.split())) - 1): @@ -214,10 +182,6 @@ def filter_items(items: List[str]) -> List[str]: return final - -# ===================================================== -# SMART SPLITTER -# ===================================================== def smart_split(text: str) -> List[str]: if not text: @@ -234,7 +198,7 @@ def smart_split(text: str) -> List[str]: if not line: continue - # split inline numbering + parts = re.split(r"\d+[\.\)]\s*", line) for p in parts: @@ -244,7 +208,7 @@ def smart_split(text: str) -> List[str]: if not p: continue - # split bullets inline + subparts = re.split(r"\s*[-•▪]\s*", p) for sp in subparts: @@ -256,10 +220,6 @@ def smart_split(text: str) -> List[str]: return lines - -# ===================================================== -# MAIN VALIDATOR -# ===================================================== def validate_generated_list( text: str, top_k: int = 10 @@ -272,4 +232,4 @@ def validate_generated_list( cleaned = filter_items(raw_items) - return cleaned[:top_k] \ No newline at end of file + return cleaned[:top_k] diff --git a/src/similarity_model/__init__.py b/src/similarity_model/__init__.py index b4fe4a7a823741744cbd42cd1a913761b936d659..41ab9830d005f35d2cdc0df14d1d001f30375cc2 100644 --- a/src/similarity_model/__init__.py +++ b/src/similarity_model/__init__.py @@ -40,4 +40,4 @@ from .similarity_engine import ( from .sync_projects import ( sync_projects -) \ No newline at end of file +) diff --git a/src/similarity_model/__pycache__/__init__.cpython-313.pyc b/src/similarity_model/__pycache__/__init__.cpython-313.pyc index 21bc1a14f4ed98654b8d1dde395ce57a35208777..9a72544b4e8e81cdee74ffbc6b8df778dd35c6f5 100644 Binary files a/src/similarity_model/__pycache__/__init__.cpython-313.pyc and b/src/similarity_model/__pycache__/__init__.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/embedding_engine.cpython-313.pyc b/src/similarity_model/__pycache__/embedding_engine.cpython-313.pyc index ebaf8535ebbe76e653bde3af79984ba4f09f6e41..b2411441256a638bccfedef6fdf530673d40bf75 100644 Binary files a/src/similarity_model/__pycache__/embedding_engine.cpython-313.pyc and b/src/similarity_model/__pycache__/embedding_engine.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/feature_similarity.cpython-313.pyc b/src/similarity_model/__pycache__/feature_similarity.cpython-313.pyc index 5dba9604ce4f18084127c0e7cd63eadf9bdb169d..d0804f22ef60907417e0d64da82a61924b5e7993 100644 Binary files a/src/similarity_model/__pycache__/feature_similarity.cpython-313.pyc and b/src/similarity_model/__pycache__/feature_similarity.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/hybrid_ranker.cpython-313.pyc b/src/similarity_model/__pycache__/hybrid_ranker.cpython-313.pyc index b871dd0940d7d06da37f52c78f575c1541aa3d2d..a3fa338e1475425394de93c9b1c695b6aec0b0cf 100644 Binary files a/src/similarity_model/__pycache__/hybrid_ranker.cpython-313.pyc and b/src/similarity_model/__pycache__/hybrid_ranker.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/preprocessing.cpython-313.pyc b/src/similarity_model/__pycache__/preprocessing.cpython-313.pyc index cf4ee3065431b95e70bb73e7d0a3953e53609fea..9bd0813521101a484514de0c98198e2ab7804189 100644 Binary files a/src/similarity_model/__pycache__/preprocessing.cpython-313.pyc and b/src/similarity_model/__pycache__/preprocessing.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/semantic_search.cpython-313.pyc b/src/similarity_model/__pycache__/semantic_search.cpython-313.pyc index 9ca2a9df79d9c39de51a278403d50dcd7c874126..5f6f892fd2d4ada04bacecf5dfeb4364e13ba192 100644 Binary files a/src/similarity_model/__pycache__/semantic_search.cpython-313.pyc and b/src/similarity_model/__pycache__/semantic_search.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/similarity_engine.cpython-313.pyc b/src/similarity_model/__pycache__/similarity_engine.cpython-313.pyc index a424f777a0d5f84096eb9540ede12ed25d5844ba..8eb7f6276204466ff04e3e8260faaf4001fd3b5e 100644 Binary files a/src/similarity_model/__pycache__/similarity_engine.cpython-313.pyc and b/src/similarity_model/__pycache__/similarity_engine.cpython-313.pyc differ diff --git a/src/similarity_model/embedding_engine.py b/src/similarity_model/embedding_engine.py index 59a423e90a04a7de6624590b9ada7cec92c72c9a..0f61253901d6750d02ead3a73e3c09c44cb1fd1d 100644 --- a/src/similarity_model/embedding_engine.py +++ b/src/similarity_model/embedding_engine.py @@ -16,7 +16,6 @@ TEXT_COL = "clean_text" TITLE_COL = "project_title" TECH_COL = "technologies" -# Resolve paths relative to the project root _PROJECT_ROOT = Path(__file__).resolve().parents[2] MODEL_DIR = _PROJECT_ROOT / "models" INDEX_PATH = MODEL_DIR / "faiss_index.bin" @@ -97,4 +96,4 @@ def train_embedding_engine(): engine.save_artifacts() logger.info("Embedding engine completed successfully.") - return engine \ No newline at end of file + return engine diff --git a/src/similarity_model/evaluation.py b/src/similarity_model/evaluation.py index 254eb566234d17ffb355f69090b1a47e2f56526d..8eff6656383e40bef3bceda2c57063f528cc3a73 100644 --- a/src/similarity_model/evaluation.py +++ b/src/similarity_model/evaluation.py @@ -1,17 +1,9 @@ -# src/evaluation.py -# FINAL PROFESSIONAL EVALUATION VERSION -# Self Test + Real Query Test + Overall Judgment - import logging import pandas as pd from src.similarity_model import find_similar_projects from src.similarity_model import load_metadata - -# ===================================================== -# Logging -# ===================================================== logging.basicConfig( level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s" @@ -21,9 +13,6 @@ logger = logging.getLogger(__name__) TOP_K = 5 SELF_TEST_SAMPLES = 20 -# ===================================================== -# SELF RETRIEVAL TEST -# ===================================================== def run_self_test(): df = load_metadata() @@ -62,10 +51,6 @@ def run_self_test(): return score - -# ===================================================== -# REAL QUERY TEST -# ===================================================== def run_real_queries(): queries = [ @@ -165,10 +150,6 @@ def run_real_queries(): return avg - -# ===================================================== -# FINAL JUDGMENT -# ===================================================== def final_status( self_score, query_score @@ -197,18 +178,3 @@ def final_status( print("Overall Score:", round(final_score, 4)) print("==============================\n") - - -# ===================================================== -# RUN -# ===================================================== -if __name__ == "__main__": - - self_score = run_self_test() - - query_score = run_real_queries() - - final_status( - self_score, - query_score - ) \ No newline at end of file diff --git a/src/similarity_model/feature_similarity.py b/src/similarity_model/feature_similarity.py index 7d0ef0ffbacb18698aeb10f44ee8271937b9c24e..f22580471f8305fe8cdc3bf214bbcced96a7542f 100644 --- a/src/similarity_model/feature_similarity.py +++ b/src/similarity_model/feature_similarity.py @@ -125,7 +125,7 @@ def compute_feature_similarity( sim_matrix = cosine_similarity(emb_a, emb_b) - # Hungarian match + row_idx, col_idx = linear_sum_assignment(-sim_matrix) matches = [] diff --git a/src/similarity_model/hybrid_ranker.py b/src/similarity_model/hybrid_ranker.py index 8794dd79ad387111eb652a1dfe4c37fc1faa3d98..8a49094b8fc26003f9e998a5b372abae513a42f6 100644 --- a/src/similarity_model/hybrid_ranker.py +++ b/src/similarity_model/hybrid_ranker.py @@ -1,5 +1,3 @@ -# src/hybrid_ranker.py - import logging from typing import List, Dict, Any @@ -11,24 +9,15 @@ from src.similarity_model import ( safe_feature_list ) -# ===================================================== -# Logging -# ===================================================== logging.basicConfig( level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s" ) logger = logging.getLogger(__name__) -# ===================================================== -# Columns -# ===================================================== TITLE_COL = "project_title" FEATURE_COL = "features" -# ===================================================== -# Weights / Thresholds -# ===================================================== DEFAULT_SEMANTIC_WEIGHT = 0.65 DEFAULT_FEATURE_WEIGHT = 0.35 @@ -38,16 +27,12 @@ LOW_FEATURE_WEIGHT = 0.20 BONUS_WEIGHT = 0.05 MIN_HYBRID_SCORE = 0.05 -# ===================================================== -# Helpers -# ===================================================== def clamp(value: float) -> float: """ Keep score inside [0,1] """ return max(0.0, min(1.0, float(value))) - def get_dynamic_weights( feature_count: int, coverage: float @@ -57,21 +42,17 @@ def get_dynamic_weights( Returns (semantic_w, feature_w, coverage_w) — always sum to 1.0 """ - # Rich features → trust feature matching more + if feature_count >= 5 and coverage >= 0.60: return 0.40, 0.45, 0.15 - # Sparse features → trust semantic more + if feature_count <= 2: return 0.70, 0.20, 0.10 - # Default balance + return 0.55, 0.35, 0.10 - -# ===================================================== -# Score Engines -# ===================================================== def compute_hybrid_score( semantic_score, feature_score, @@ -86,15 +67,15 @@ def compute_hybrid_score( shared_count = feature_count - unique_query_count - # ========================================== - # Near-duplicate fast path - # ========================================== + + + if shared_count >= 6 and unique_query_count <= 1: return 0.95 - # ========================================== - # Strong feature overlap fast path - # ========================================== + + + if coverage >= 0.90 and feature_score >= 0.65: return round( clamp( @@ -105,15 +86,15 @@ def compute_hybrid_score( 4 ) - # ========================================== - # No feature overlap → rely on semantics only - # ========================================== + + + if feature_score == 0 or coverage == 0: return round(clamp(0.60 * semantic_score), 4) - # ========================================== - # Normal scoring with dynamic weights - # ========================================== + + + semantic_w, feature_w, coverage_w = get_dynamic_weights( feature_count, coverage ) @@ -124,17 +105,16 @@ def compute_hybrid_score( coverage_w * coverage ) - # ========================================== - # Penalty for weak feature overlap - # (prevents generic keyword matches from scoring too high) - # ========================================== + + + + if feature_score < 0.40: penalty = 0.50 + (feature_score / 0.40) * 0.50 score *= penalty return round(clamp(score), 4) - def compute_originality( hybrid_score: float, unique_query_features: int, @@ -147,21 +127,20 @@ def compute_originality( hybrid_score = clamp(hybrid_score) if hybrid_score >= 0.39: - # > 0.39 is a strong semantic match -> Originality 0-25% + originality = 25.0 - ((hybrid_score - 0.39) / 0.11) * 25.0 elif hybrid_score >= 0.35: - # 0.35 to 0.39 is a moderate match -> Originality 25-75% + originality = 75.0 - ((hybrid_score - 0.35) / 0.04) * 50.0 elif hybrid_score >= 0.25: - # 0.25 to 0.35 is a weak match -> Originality 75-95% + originality = 95.0 - ((hybrid_score - 0.25) / 0.10) * 20.0 else: - # < 0.25 is effectively unrelated -> Originality 95-100% + originality = 100.0 - (hybrid_score / 0.25) * 5.0 return round(max(0.0, min(100.0, originality)), 2) - def compute_confidence( semantic_score: float, feature_score: float, @@ -179,7 +158,6 @@ def compute_confidence( return round(clamp(confidence), 4) - def risk_label(score: float) -> str: """ Duplicate risk label. @@ -199,10 +177,6 @@ def risk_label(score: float) -> str: return "Very Low" - -# ===================================================== -# Core Comparison -# ===================================================== def compare_single_candidate( query_row: Dict[str, Any], candidate_row: Dict[str, Any], @@ -301,10 +275,6 @@ def compare_single_candidate( candidate_features } - -# ===================================================== -# Rank Candidates -# ===================================================== def rank_candidates( query_row, candidate_rows, @@ -354,52 +324,3 @@ def rank_candidates( ).reset_index(drop=True) return results - - -# ===================================================== -# Example Run -# ===================================================== -if __name__ == "__main__": - - query_project = { - "project_title": "Clinic AI System", - "features": [ - "appointment booking", - "ai chatbot", - "patient records", - "doctor dashboard" - ] - } - - candidate_projects = [ - { - "project_title": - "Hospital Reservation Platform", - - "features": [ - "online booking", - "chatbot assistant", - "medical records", - "analytics dashboard" - ] - }, - { - "project_title": - "Smart Farming Monitor", - - "features": [ - "crop analysis", - "soil sensors" - ] - } - ] - - semantic_scores = [0.84, 0.33] - - ranked = rank_candidates( - query_row=query_project, - candidate_rows=candidate_projects, - semantic_scores=semantic_scores - ) - - print(ranked) \ No newline at end of file diff --git a/src/similarity_model/semantic_search.py b/src/similarity_model/semantic_search.py index f4009725ed4464661c6afb2c45ea85ae29f18a80..1f2f308b33aad96a9b7103473be20f4eeb1f7e23 100644 --- a/src/similarity_model/semantic_search.py +++ b/src/similarity_model/semantic_search.py @@ -107,8 +107,8 @@ def build_results( final_score = float(score) - # --------------------------------------------- - # --------------------------------------------- + + if query_words: title_words = tokenize(row[TITLE_COL]) diff --git a/src/similarity_model/similarity_engine.py b/src/similarity_model/similarity_engine.py index d48dcba57fb6fa5b7883ffe545012e6016431057..0e93b630dd3825d985e9dd8b84f1b1366e11919c 100644 --- a/src/similarity_model/similarity_engine.py +++ b/src/similarity_model/similarity_engine.py @@ -1,7 +1,3 @@ -# src/similarity_engine.py -# FINAL VERSION -# Full Similarity Engine with Auto Feature Extraction + Hybrid Ranking - import logging from typing import Dict, Any, List, Optional @@ -23,30 +19,20 @@ from src.similarity_model import ( risk_label ) -# ===================================================== -# Logging -# ===================================================== logging.basicConfig( level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s" ) logger = logging.getLogger(__name__) -# ===================================================== -# Config -# ===================================================== TITLE_COL = "project_title" FEATURE_COL = "features" DEFAULT_TOP_K = 5 DEFAULT_SEARCH_POOL = 50 -DEFAULT_MIN_SEMANTIC_SCORE = 0.30 +DEFAULT_MIN_SEMANTIC_SCORE = 0.10 MAX_QUERY_FEATURES = 12 - -# ===================================================== -# Query Builders -# ===================================================== def build_raw_text( title: str = "", abstract: str = "", @@ -57,7 +43,7 @@ def build_raw_text( """ parts = [ str(title).strip(), - str(title).strip(), # title weighted + str(title).strip(), str(abstract).strip(), str(description).strip() ] @@ -66,7 +52,6 @@ def build_raw_text( [p for p in parts if p] ).strip() - def merge_features( title: str = "", abstract: str = "", @@ -100,7 +85,7 @@ def merge_features( final = [] seen = set() - # manual first + for feat in manual_features + auto_features: feat = str(feat).strip().lower() @@ -111,7 +96,6 @@ def merge_features( return final[:MAX_QUERY_FEATURES] - def build_query_project( title: str, abstract: str, @@ -126,7 +110,6 @@ def build_query_project( FEATURE_COL: features } - def build_query_text( title: str, abstract: str, @@ -149,10 +132,6 @@ def build_query_text( return normalize_text(text) - -# ===================================================== -# Compare Candidate -# ===================================================== def compare_candidate( query_project: Dict[str, Any], candidate_row, @@ -197,7 +176,6 @@ def compare_candidate( total_query_features=query_feature_count ) - confidence_score = compute_confidence( semantic_score=semantic_score, feature_score=feature_score, @@ -245,10 +223,6 @@ def compare_candidate( candidate_features } - -# ===================================================== -# Main Engine -# ===================================================== def find_similar_projects( title: str = "", abstract: str = "", @@ -279,9 +253,9 @@ def find_similar_projects( feature_model = load_feature_model() df = load_metadata() - # ============================================= - # Query preprocessing - # ============================================= + + + logger.info( "Preparing query..." ) @@ -327,7 +301,6 @@ def find_similar_projects( auto_extract=auto_extract_features ) - query_project = build_query_project( title=title, abstract=abstract, @@ -342,9 +315,9 @@ def find_similar_projects( features=final_features ) - # ============================================= - # Stage 1 Semantic Retrieval - # ============================================= + + + logger.info( "Running semantic retrieval..." ) @@ -358,9 +331,9 @@ def find_similar_projects( if "message" in semantic_results.columns: return semantic_results - # ============================================= - # Stage 2 Hybrid Ranking - # ============================================= + + + logger.info( "Running hybrid ranking..." ) @@ -407,13 +380,13 @@ def find_similar_projects( ascending=False ).head(top_k).reset_index(drop=True) - # Force originality = 0 for any near-duplicate (same content, different title) + final_df.loc[ final_df["hybrid_score"] >= 0.90, "originality_score" ] = 0.0 - # add debug info + final_df["query_features_used"] = [ final_features ] * len(final_df) @@ -423,4 +396,3 @@ def find_similar_projects( ] * len(final_df) return final_df - diff --git a/src/similarity_model/sync_projects.py b/src/similarity_model/sync_projects.py index 62829c5cd5c6c09e1713247d10df11e45ce3a4d5..01528b8312bce2bf74f9e8f8613b2e132c866be0 100644 --- a/src/similarity_model/sync_projects.py +++ b/src/similarity_model/sync_projects.py @@ -7,22 +7,18 @@ from Data.database.sql_connector import engine from src.similarity_model.preprocessing import preprocess_dataset from src.similarity_model.embedding_engine import train_embedding_engine -# ===================================================== -# Logging -# ===================================================== logging.basicConfig( level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s" ) logger = logging.getLogger(__name__) - def sync_projects(): logger.info("Initializing project synchronization service...") - # Verify database connection first + try: - # Since engine is a LazyEngine, calling connect() tests connection + with engine.connect() as conn: logger.info("Database connection verified successfully.") except Exception as exc: @@ -33,7 +29,7 @@ def sync_projects(): ) sys.exit(1) - # 1. Fetch raw active projects + projects_query = """ SELECT * FROM Projects @@ -48,7 +44,7 @@ def sync_projects(): projects_df = pd.read_sql(projects_query, conn) logger.info(f"Loaded {len(projects_df)} active projects from database.") - # 2. Fetch existing preprocessed projects + logger.info("Fetching existing records from 'PreProcessed_Projects'...") with engine.connect() as conn: existing_df = pd.read_sql("SELECT id FROM PreProcessed_Projects", conn) @@ -58,7 +54,7 @@ def sync_projects(): changed = False - # 3. Remove projects no longer active or allowed + ids_to_remove = processed_ids - allowed_ids if ids_to_remove: logger.info(f"Found {len(ids_to_remove)} projects to remove (status changed or deleted).") @@ -71,17 +67,17 @@ def sync_projects(): logger.info(f"Successfully removed {len(ids_to_remove)} projects from 'PreProcessed_Projects'.") changed = True - # 4. Filter for new projects + new_projects = projects_df[~projects_df["Id"].isin(processed_ids)].copy() if len(new_projects) > 0: logger.info(f"Found {len(new_projects)} new projects to preprocess and insert.") - # Run preprocessing (cleaning, tokenization, feature extraction) + processed_df = preprocess_dataset(new_projects) if len(processed_df) > 0: - # Map columns to target schema + cols_to_keep = [ "id", "submittedat", @@ -99,7 +95,7 @@ def sync_projects(): "features" ] - # Ensure columns exist before filtering + for col in cols_to_keep: if col not in processed_df.columns: processed_df[col] = "" @@ -115,10 +111,10 @@ def sync_projects(): } ) - # Stringify feature lists for SQL insertion + processed_df["features"] = processed_df["features"].apply(json.dumps) - # Insert into database + logger.info("Uploading preprocessed records to database...") with engine.begin() as conn: processed_df.to_sql( @@ -134,14 +130,10 @@ def sync_projects(): else: logger.info("No new projects found.") - # 5. Rebuild local FAISS index & embeddings if anything changed + if changed: logger.info("Changes detected. Rebuilding local embeddings and FAISS index...") train_embedding_engine() logger.info("Local embeddings and index updated successfully.") else: logger.info("No database changes detected. Embeddings remain in sync.") - - -if __name__ == "__main__": - sync_projects()