diff --git a/.env b/.env new file mode 100644 index 0000000000000000000000000000000000000000..0cff7486498cec9857c67044cb3d0f31b7c02141 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +GEMINI_API_KEY=AIzaSyAkFsaN3BKoSQmRW4FzTahhZXbq-ldsDZ4 +GEMINI_MODEL_NAME=gemini-2.5-flash \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..3b664107303df336bab8010caad42ddaed24550e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "git.ignoreLimitWarning": true +} \ No newline at end of file diff --git a/Data/database/__pycache__/sql_connector.cpython-311.pyc b/Data/database/__pycache__/sql_connector.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b46d6a3772df76a9096a6c53d99d562797d4a0e Binary files /dev/null and b/Data/database/__pycache__/sql_connector.cpython-311.pyc differ diff --git a/Data/database/__pycache__/sql_connector.cpython-313.pyc b/Data/database/__pycache__/sql_connector.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a65d9fecae3e6c78008ced81b43d6a1fd82a882 Binary files /dev/null and b/Data/database/__pycache__/sql_connector.cpython-313.pyc differ diff --git a/Data/database/sql_connector.py b/Data/database/sql_connector.py new file mode 100644 index 0000000000000000000000000000000000000000..7c9fc5631b0fce62ed662503ba40f56e8a3d5506 --- /dev/null +++ b/Data/database/sql_connector.py @@ -0,0 +1,60 @@ +from sqlalchemy import create_engine +import pandas as pd +import urllib +import json + +SERVER = "innotrack-sql-server.database.windows.net" +DATABASE = "InnoTrackDB" +USERNAME = "innotrackadmin" +PASSWORD = "Innotrack@admin233" + +params = urllib.parse.quote_plus( + f"DRIVER={{ODBC Driver 18 for SQL Server}};" + f"SERVER={SERVER};" + f"DATABASE={DATABASE};" + f"UID={USERNAME};" + f"PWD={PASSWORD};" + "Encrypt=yes;" + "TrustServerCertificate=no;" + "Connection Timeout=30;" +) + +connection_string = ( + f"mssql+pyodbc:///?odbc_connect={params}" +) + +engine = create_engine(connection_string) + +try: + + with engine.connect() as conn: + print("SQL Connected Successfully") + +except Exception as e: + print("Connection Failed") + print(e) + + + + + +def load_preprocessed_projects(): + + query = """ + SELECT * + FROM PreProcessed_Projects + """ + + df = pd.read_sql( + query, + engine + ) + + if "features" in df.columns: + + df["features"] = ( + df["features"] + .apply(json.loads) + ) + + return df \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..52f4dc5e4bda41fe02ee9b9bf6fd45218ad56209 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-slim + +RUN useradd -m -u 1000 user + +USER user + +ENV HOME=/home/user \ + PATH=/home/user/.local/bin:$PATH \ + PYTHONUNBUFFERED=1 + +WORKDIR $HOME/app + +COPY --chown=user requirements.txt requirements.txt + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +COPY --chown=user . . + +CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "7860"] \ No newline at end of file diff --git a/Notebooks/TEST.ipynb b/Notebooks/TEST.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f2b31d1027f40bd4363422d461e90a96978e133b --- /dev/null +++ b/Notebooks/TEST.ipynb @@ -0,0 +1,5276 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "executionInfo": { + "elapsed": 49, + "status": "ok", + "timestamp": 1777121871809, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "C89EBjEsx9Gq", + "outputId": "e76f8c9c-cdcf-4fc7-94b5-4e8a30009518" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'e:\\\\gradution project'" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "import os\n", + "os.getcwd()\n", + "os.chdir(\"/gradution project\")\n", + "os.getcwd()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 11569, + "status": "ok", + "timestamp": 1777121884875, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "Eel6SUFWvNMW", + "outputId": "10ac3316-9b05-44f4-81b8-4ba3ff26cc99" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "All modules imported successfully\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from src.similarity_model import preprocess_dataset\n", + "from src.similarity_model import train_embedding_engine\n", + "from src.similarity_model import search_by_text\n", + "from src.similarity_model import find_similar_projects\n", + "\n", + "print(\"All modules imported successfully\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 712 + }, + "executionInfo": { + "elapsed": 93, + "status": "ok", + "timestamp": 1777121887495, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "QlD5ukNTDS6S", + "outputId": "43dab901-85ac-4386-d77a-2bb24af4b111" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rows: 301\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDSubmitted AtTitleStudent NamesSupervisorYearCategoryTechnologiesKeywordsAbstractDescriptionProblem StatementProposed SolutionObjectivesai summary
296OCR_36NaTFind Missing people using Face RecognitionNaNDr. Mai Kamal2023surveillancekotlin, python flaskartificial intelligence, machine learning, dee...Our project \"Find Missed People Using Face Rec...The problem of missing persons and children is...• The absence of a dedicated digital system fo...Our project \"Find Missed People Using Face Rec...• Develop a user-friendly application accessib...This graduation project, titled \"Find Missing ...
297OCR_37NaTRemember Me Early diagnosis of Alzheimer's dis...NaNDr. Mostafa Elgendy2023medicaldart, flutter, HTML, css, js, php, mvc approac...healthcare, medical diagnosis, patient care, m...According to the WHO, around 50 million people...In this project, we aimed to collect all the t...• The absence of a dedicated digital system fo...According to the WHO, around 50 million people...• Develop a user-friendly application accessib...This graduation project, titled \"Remember Me E...
298OCR_40NaTMobile Based System for Traffic Monitoring and...NaNDr. Sahr fawzy2012GeneralAndroidgeneral, software system, graduation project, ...The theme of the system/application is the usa...No one can ignore the fact that traffic proble...• The absence of a dedicated digital system fo...The theme of the system/application is the usa...• Develop a user-friendly application accessib...This graduation project, titled \"Mobile Based ...
299OCR_41NaTDetecting Diseases Using Chatbot and Booking C...NaNDr. Shadia Sarhan2024medicalphp, laravel, mysql, mvc, livewire package, ui...healthcare, medical diagnosis, patient care, m...A Project aim to create application that conta...Many pcople find difficult to search for docto...• The absence of a dedicated digital system fo...A Project aim to create application that conta...• Develop a user-friendly application accessib...This graduation project, titled \"Detecting Dis...
300OCR_46NaTElectronic SchoolNaNDr. Shaker Elsabagh2022Educationhtml, css, JavaScript, jquery, bootstrap, apie-learning, education platform, student manage...Human history is observing a very strange time...The world we live in has been through a lot of...• The absence of a dedicated digital system fo...Human history is observing a very strange time...• Achieve accurate and reliable detection/diag...This graduation project, titled \"Electronic Sc...
\n", + "
" + ], + "text/plain": [ + " ID Submitted At Title \\\n", + "296 OCR_36 NaT Find Missing people using Face Recognition \n", + "297 OCR_37 NaT Remember Me Early diagnosis of Alzheimer's dis... \n", + "298 OCR_40 NaT Mobile Based System for Traffic Monitoring and... \n", + "299 OCR_41 NaT Detecting Diseases Using Chatbot and Booking C... \n", + "300 OCR_46 NaT Electronic School \n", + "\n", + " Student Names Supervisor Year Category \\\n", + "296 NaN Dr. Mai Kamal 2023 surveillance \n", + "297 NaN Dr. Mostafa Elgendy 2023 medical \n", + "298 NaN Dr. Sahr fawzy 2012 General \n", + "299 NaN Dr. Shadia Sarhan 2024 medical \n", + "300 NaN Dr. Shaker Elsabagh 2022 Education \n", + "\n", + " Technologies \\\n", + "296 kotlin, python flask \n", + "297 dart, flutter, HTML, css, js, php, mvc approac... \n", + "298 Android \n", + "299 php, laravel, mysql, mvc, livewire package, ui... \n", + "300 html, css, JavaScript, jquery, bootstrap, api \n", + "\n", + " Keywords \\\n", + "296 artificial intelligence, machine learning, dee... \n", + "297 healthcare, medical diagnosis, patient care, m... \n", + "298 general, software system, graduation project, ... \n", + "299 healthcare, medical diagnosis, patient care, m... \n", + "300 e-learning, education platform, student manage... \n", + "\n", + " Abstract \\\n", + "296 Our project \"Find Missed People Using Face Rec... \n", + "297 According to the WHO, around 50 million people... \n", + "298 The theme of the system/application is the usa... \n", + "299 A Project aim to create application that conta... \n", + "300 Human history is observing a very strange time... \n", + "\n", + " Description \\\n", + "296 The problem of missing persons and children is... \n", + "297 In this project, we aimed to collect all the t... \n", + "298 No one can ignore the fact that traffic proble... \n", + "299 Many pcople find difficult to search for docto... \n", + "300 The world we live in has been through a lot of... \n", + "\n", + " Problem Statement \\\n", + "296 • The absence of a dedicated digital system fo... \n", + "297 • The absence of a dedicated digital system fo... \n", + "298 • The absence of a dedicated digital system fo... \n", + "299 • The absence of a dedicated digital system fo... \n", + "300 • The absence of a dedicated digital system fo... \n", + "\n", + " Proposed Solution \\\n", + "296 Our project \"Find Missed People Using Face Rec... \n", + "297 According to the WHO, around 50 million people... \n", + "298 The theme of the system/application is the usa... \n", + "299 A Project aim to create application that conta... \n", + "300 Human history is observing a very strange time... \n", + "\n", + " Objectives \\\n", + "296 • Develop a user-friendly application accessib... \n", + "297 • Develop a user-friendly application accessib... \n", + "298 • Develop a user-friendly application accessib... \n", + "299 • Develop a user-friendly application accessib... \n", + "300 • Achieve accurate and reliable detection/diag... \n", + "\n", + " ai summary \n", + "296 This graduation project, titled \"Find Missing ... \n", + "297 This graduation project, titled \"Remember Me E... \n", + "298 This graduation project, titled \"Mobile Based ... \n", + "299 This graduation project, titled \"Detecting Dis... \n", + "300 This graduation project, titled \"Electronic Sc... " + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "file_path = r\"E:\\gradution project\\Data\\raw\\Graduation Projects - Sheet1.xlsx\"\n", + "\n", + "df_raw = pd.read_excel(file_path)\n", + "\n", + "print(\"Rows:\", len(df_raw))\n", + "df_raw.tail()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 695 + }, + "executionInfo": { + "elapsed": 8196, + "status": "ok", + "timestamp": 1777121896550, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "CbhNr97MF3G_", + "outputId": "92994227-fc55-42ee-db94-53815d8e0b49" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-05-09 03:44:02,005 | INFO | Starting preprocessing...\n", + "2026-05-09 03:44:02,126 | INFO | Removed duplicates: 7\n", + "2026-05-09 03:44:02,137 | INFO | Extracting features...\n", + "2026-05-09 03:47:15,627 | INFO | Final rows: 294\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rows after cleaning: 294\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idsubmitted_attitlestudent_namessupervisoryearcategorytechnologieskeywordsabstractdescriptionproblem_statementproposed_solutionobjectivesai_summaryproject_titlefull_contentclean_textword_countfeatures
289OCR_36NaTFind Missing people using Face RecognitionNaNDr. Mai Kamal2023surveillancekotlin, python flaskartificial intelligence, machine learning, dee...Our project \"Find Missed People Using Face Rec...The problem of missing persons and children is...• The absence of a dedicated digital system fo...Our project \"Find Missed People Using Face Rec...• Develop a user-friendly application accessib...This graduation project, titled \"Find Missing .... . Our project \"Find Missed People Using Face.... . our project find missed people using face ...451[missing person face, facial recognition techn...
290OCR_37NaTRemember Me Early diagnosis of Alzheimer's dis...NaNDr. Mostafa Elgendy2023medicaldart, flutter, HTML, css, js, php, mvc approac...healthcare, medical diagnosis, patient care, m...According to the WHO, around 50 million people...In this project, we aimed to collect all the t...• The absence of a dedicated digital system fo...According to the WHO, around 50 million people...• Develop a user-friendly application accessib...This graduation project, titled \"Remember Me E.... . According to the WHO, around 50 million pe.... . according to the who around 50 million peo...478[alzheimer disease ad, delay alzheimer symptom...
291OCR_40NaTMobile Based System for Traffic Monitoring and...NaNDr. Sahr fawzy2012GeneralAndroidgeneral, software system, graduation project, ...The theme of the system/application is the usa...No one can ignore the fact that traffic proble...• The absence of a dedicated digital system fo...The theme of the system/application is the usa...• Develop a user-friendly application accessib...This graduation project, titled \"Mobile Based .... . The theme of the system/application is the.... . the theme of the system/application is the...369[location based alternatives, roads provides u...
292OCR_41NaTDetecting Diseases Using Chatbot and Booking C...NaNDr. Shadia Sarhan2024medicalphp, laravel, mysql, mvc, livewire package, ui...healthcare, medical diagnosis, patient care, m...A Project aim to create application that conta...Many pcople find difficult to search for docto...• The absence of a dedicated digital system fo...A Project aim to create application that conta...• Develop a user-friendly application accessib...This graduation project, titled \"Detecting Dis.... . A Project aim to create application that c.... . a project aim to create application that c...310[doctors users chatbot, disease symptom paragr...
293OCR_46NaTElectronic SchoolNaNDr. Shaker Elsabagh2022Educationhtml, css, JavaScript, jquery, bootstrap, apie-learning, education platform, student manage...Human history is observing a very strange time...The world we live in has been through a lot of...• The absence of a dedicated digital system fo...Human history is observing a very strange time...• Achieve accurate and reliable detection/diag...This graduation project, titled \"Electronic Sc.... . Human history is observing a very strange .... . human history is observing a very strange ...511[learning electronic school, online learning p...
\n", + "
" + ], + "text/plain": [ + " id submitted_at title \\\n", + "289 OCR_36 NaT Find Missing people using Face Recognition \n", + "290 OCR_37 NaT Remember Me Early diagnosis of Alzheimer's dis... \n", + "291 OCR_40 NaT Mobile Based System for Traffic Monitoring and... \n", + "292 OCR_41 NaT Detecting Diseases Using Chatbot and Booking C... \n", + "293 OCR_46 NaT Electronic School \n", + "\n", + " student_names supervisor year category \\\n", + "289 NaN Dr. Mai Kamal 2023 surveillance \n", + "290 NaN Dr. Mostafa Elgendy 2023 medical \n", + "291 NaN Dr. Sahr fawzy 2012 General \n", + "292 NaN Dr. Shadia Sarhan 2024 medical \n", + "293 NaN Dr. Shaker Elsabagh 2022 Education \n", + "\n", + " technologies \\\n", + "289 kotlin, python flask \n", + "290 dart, flutter, HTML, css, js, php, mvc approac... \n", + "291 Android \n", + "292 php, laravel, mysql, mvc, livewire package, ui... \n", + "293 html, css, JavaScript, jquery, bootstrap, api \n", + "\n", + " keywords \\\n", + "289 artificial intelligence, machine learning, dee... \n", + "290 healthcare, medical diagnosis, patient care, m... \n", + "291 general, software system, graduation project, ... \n", + "292 healthcare, medical diagnosis, patient care, m... \n", + "293 e-learning, education platform, student manage... \n", + "\n", + " abstract \\\n", + "289 Our project \"Find Missed People Using Face Rec... \n", + "290 According to the WHO, around 50 million people... \n", + "291 The theme of the system/application is the usa... \n", + "292 A Project aim to create application that conta... \n", + "293 Human history is observing a very strange time... \n", + "\n", + " description \\\n", + "289 The problem of missing persons and children is... \n", + "290 In this project, we aimed to collect all the t... \n", + "291 No one can ignore the fact that traffic proble... \n", + "292 Many pcople find difficult to search for docto... \n", + "293 The world we live in has been through a lot of... \n", + "\n", + " problem_statement \\\n", + "289 • The absence of a dedicated digital system fo... \n", + "290 • The absence of a dedicated digital system fo... \n", + "291 • The absence of a dedicated digital system fo... \n", + "292 • The absence of a dedicated digital system fo... \n", + "293 • The absence of a dedicated digital system fo... \n", + "\n", + " proposed_solution \\\n", + "289 Our project \"Find Missed People Using Face Rec... \n", + "290 According to the WHO, around 50 million people... \n", + "291 The theme of the system/application is the usa... \n", + "292 A Project aim to create application that conta... \n", + "293 Human history is observing a very strange time... \n", + "\n", + " objectives \\\n", + "289 • Develop a user-friendly application accessib... \n", + "290 • Develop a user-friendly application accessib... \n", + "291 • Develop a user-friendly application accessib... \n", + "292 • Develop a user-friendly application accessib... \n", + "293 • Achieve accurate and reliable detection/diag... \n", + "\n", + " ai_summary project_title \\\n", + "289 This graduation project, titled \"Find Missing ... \n", + "290 This graduation project, titled \"Remember Me E... \n", + "291 This graduation project, titled \"Mobile Based ... \n", + "292 This graduation project, titled \"Detecting Dis... \n", + "293 This graduation project, titled \"Electronic Sc... \n", + "\n", + " full_content \\\n", + "289 . . Our project \"Find Missed People Using Face... \n", + "290 . . According to the WHO, around 50 million pe... \n", + "291 . . The theme of the system/application is the... \n", + "292 . . A Project aim to create application that c... \n", + "293 . . Human history is observing a very strange ... \n", + "\n", + " clean_text word_count \\\n", + "289 . . our project find missed people using face ... 451 \n", + "290 . . according to the who around 50 million peo... 478 \n", + "291 . . the theme of the system/application is the... 369 \n", + "292 . . a project aim to create application that c... 310 \n", + "293 . . human history is observing a very strange ... 511 \n", + "\n", + " features \n", + "289 [missing person face, facial recognition techn... \n", + "290 [alzheimer disease ad, delay alzheimer symptom... \n", + "291 [location based alternatives, roads provides u... \n", + "292 [doctors users chatbot, disease symptom paragr... \n", + "293 [learning electronic school, online learning p... " + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clean_df = preprocess_dataset(df_raw)\n", + "\n", + "print(\"Rows after cleaning:\", len(clean_df))\n", + "clean_df.tail()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 9, + "status": "ok", + "timestamp": 1777121896561, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "Cddf8xCADlIr", + "outputId": "89f60abf-fcf6-4d05-e53e-5c23404229c9" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "['payment disputes service', 'secure intelligent service', 'technician recommendation module', 'maintain trust marketplace', 'escrow released technicians', 'customers qualified', 'manage account verification', 'service completion confirmed', 'workflow aims', 'feature rule based']\n" + ] + } + ], + "source": [ + "print(type(clean_df.loc[0, \"features\"]))\n", + "print(clean_df.loc[0, \"features\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 8, + "status": "ok", + "timestamp": 1777121896571, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "FVpnglAmGbyU", + "outputId": "347ad1f0-b83e-44f7-bdf0-7ac682fd3be6" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['id', 'submitted_at', 'title', 'student_names', 'supervisor', 'year', 'category', 'technologies', 'keywords', 'abstract', 'description', 'problem_statement', 'proposed_solution', 'objectives', 'ai_summary', 'project_title', 'full_content', 'clean_text', 'word_count', 'features']\n" + ] + } + ], + "source": [ + "print(clean_df.columns.tolist())" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\yossef\\AppData\\Local\\Temp\\ipykernel_14992\\1799785526.py:1: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.\n", + "See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.\n", + " object_cols = clean_df.select_dtypes(include=\"object\").columns\n" + ] + } + ], + "source": [ + "object_cols = clean_df.select_dtypes(include=\"object\").columns\n", + "\n", + "for col in object_cols:\n", + " clean_df[col] = clean_df[col].astype(str)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 28, + "status": "ok", + "timestamp": 1777121896600, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "viSKYOBUSy9-", + "outputId": "9cd3450e-2ad3-47c9-e64f-29317d682ab3" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved cleaned dataset\n" + ] + } + ], + "source": [ + "clean_df.to_parquet(\"Data/processed/projects_clean.parquet\", index=False)\n", + "clean_df.to_csv(\"Data/processed/projects_clean.csv\", index=False)\n", + "\n", + "print(\"Saved cleaned dataset\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 66, + "referenced_widgets": [ + "e88a3bd74a0d45a9a6f4c75fa912f14b", + "84524d1a1b544ba8b279bcf255414b46", + "13a35346277d46e4a0e8ce3c97af9d80", + "eea5a279cf4745ea95f436cd5fc5275c", + "26153e11473049bd94767d6e2ade78ce", + "1ceff15e67754111b9d28aa79545998e", + "b3ad8424cabb497fb65961b7a707c48d", + "3c076741fb62422a90f2572c455f3495", + "7979057fb043438cb98737e36403187b", + "cb460715a0974fceb2fdeb31e37b03dc", + "c4dffeedf7d349008a7228a24b325d7d" + ] + }, + "executionInfo": { + "elapsed": 1317, + "status": "ok", + "timestamp": 1777121906722, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "5i8JvSFnHdTI", + "outputId": "aca927cb-4c37-4ca3-d2c2-a164597cb5b2" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-05-09 03:47:15,851 | INFO | Loading processed dataset...\n", + "2026-05-09 03:47:15,897 | INFO | Loading embedding model: all-MiniLM-L6-v2\n", + "2026-05-09 03:47:15,898 | INFO | Load pretrained SentenceTransformer: all-MiniLM-L6-v2\n", + "2026-05-09 03:47:26,117 | INFO | Use pytorch device_name: cpu\n", + "2026-05-09 03:47:26,122 | INFO | Generating embeddings for 294 projects...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b7be93be1cc5411caaca8f372cb5ce2f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Batches: 0%| | 0/5 [00:00" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from importlib import reload\n", + "import src.similarity_model as fs\n", + "\n", + "reload(fs)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 29, + "status": "ok", + "timestamp": 1777121906780, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "N31P2msk757o", + "outputId": "eb03780d-e519-4ea2-820c-ab701a5cee4b" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from importlib import reload\n", + "import src.similarity_model as sim\n", + "\n", + "reload(sim)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 302 + }, + "executionInfo": { + "elapsed": 193, + "status": "ok", + "timestamp": 1777123794574, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "T77gmDkqHuTI", + "outputId": "515a8d56-2978-4efa-f4d4-b1d4c631adc2" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-05-09 03:47:40,400 | INFO | Loading models and artifacts...\n", + "2026-05-09 03:47:40,403 | INFO | Loading model: all-MiniLM-L6-v2\n", + "2026-05-09 03:47:40,404 | INFO | Load pretrained SentenceTransformer: all-MiniLM-L6-v2\n", + "'(ReadTimeoutError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)\"), '(Request ID: f493475b-2e06-4bce-b590-67ca67d747aa)')' thrown while requesting HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/README.md\n", + "2026-05-09 03:47:53,875 | WARNING | '(ReadTimeoutError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)\"), '(Request ID: f493475b-2e06-4bce-b590-67ca67d747aa)')' thrown while requesting HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/README.md\n", + "Retrying in 1s [Retry 1/5].\n", + "2026-05-09 03:47:53,875 | WARNING | Retrying in 1s [Retry 1/5].\n", + "2026-05-09 03:47:57,229 | INFO | Use pytorch device_name: cpu\n", + "2026-05-09 03:47:57,233 | INFO | Loading FAISS index...\n", + "2026-05-09 03:47:57,242 | INFO | Loading feature model: all-MiniLM-L6-v2\n", + "2026-05-09 03:47:57,243 | INFO | Load pretrained SentenceTransformer: all-MiniLM-L6-v2\n", + "2026-05-09 03:47:59,435 | INFO | Use pytorch device_name: cpu\n", + "2026-05-09 03:47:59,438 | INFO | Loading metadata...\n", + "2026-05-09 03:47:59,466 | INFO | Preparing query...\n", + "2026-05-09 03:47:59,529 | INFO | Running semantic retrieval...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "eb9136a7eafd48a5b62a4881e2984ee9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Batches: 0%| | 0/1 [00:00\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
project_idproject_titlesemantic_scorefeature_scorecoveragehybrid_scoreoriginality_scoreconfidence_scoreduplicate_riskshared_features_countmatched_featuresunique_query_featuresunique_candidate_featuresquery_features_usedquery_clean_text
02020.61510.56550.10.60270.49780.4922Medium1[{'feature_a': 'chatbot book recommendation', ...[library chatbot book, app library chatbot, li...[library systems egypt, collaborative reading ...[library chatbot book, app library chatbot, li...smart library app. smart library app. library ...
110.41370.60190.10.48460.59230.4262Low1[{'feature_a': 'library chatbot', 'feature_b':...[library chatbot book, app library chatbot, ch...[doctors users chatbot, help patients nearest,...[library chatbot book, app library chatbot, li...smart library app. smart library app. library ...
22250.41590.50880.10.45340.61730.3899Low1[{'feature_a': 'library chatbot', 'feature_b':...[library chatbot book, app library chatbot, ch...[websites features ai, tourists visiting egypt...[library chatbot book, app library chatbot, li...smart library app. smart library app. library ...
3290.38460.56480.10.45270.61780.3998Low1[{'feature_a': 'library chatbot', 'feature_b':...[library chatbot book, app library chatbot, ch...[reading managing computer, books intuitive in...[library chatbot book, app library chatbot, li...smart library app. smart library app. library ...
41830.33460.60190.10.43320.63340.3946Low1[{'feature_a': 'library chatbot', 'feature_b':...[library chatbot book, app library chatbot, ch...[islam mirath calculator, islamic inheritance ...[library chatbot book, app library chatbot, li...smart library app. smart library app. library ...
\n", + "" + ], + "text/plain": [ + " project_id project_title semantic_score feature_score coverage \\\n", + "0 202 0.6151 0.5655 0.1 \n", + "1 1 0.4137 0.6019 0.1 \n", + "2 225 0.4159 0.5088 0.1 \n", + "3 29 0.3846 0.5648 0.1 \n", + "4 183 0.3346 0.6019 0.1 \n", + "\n", + " hybrid_score originality_score confidence_score duplicate_risk \\\n", + "0 0.6027 0.4978 0.4922 Medium \n", + "1 0.4846 0.5923 0.4262 Low \n", + "2 0.4534 0.6173 0.3899 Low \n", + "3 0.4527 0.6178 0.3998 Low \n", + "4 0.4332 0.6334 0.3946 Low \n", + "\n", + " shared_features_count matched_features \\\n", + "0 1 [{'feature_a': 'chatbot book recommendation', ... \n", + "1 1 [{'feature_a': 'library chatbot', 'feature_b':... \n", + "2 1 [{'feature_a': 'library chatbot', 'feature_b':... \n", + "3 1 [{'feature_a': 'library chatbot', 'feature_b':... \n", + "4 1 [{'feature_a': 'library chatbot', 'feature_b':... \n", + "\n", + " unique_query_features \\\n", + "0 [library chatbot book, app library chatbot, li... \n", + "1 [library chatbot book, app library chatbot, ch... \n", + "2 [library chatbot book, app library chatbot, ch... \n", + "3 [library chatbot book, app library chatbot, ch... \n", + "4 [library chatbot book, app library chatbot, ch... \n", + "\n", + " unique_candidate_features \\\n", + "0 [library systems egypt, collaborative reading ... \n", + "1 [doctors users chatbot, help patients nearest,... \n", + "2 [websites features ai, tourists visiting egypt... \n", + "3 [reading managing computer, books intuitive in... \n", + "4 [islam mirath calculator, islamic inheritance ... \n", + "\n", + " query_features_used \\\n", + "0 [library chatbot book, app library chatbot, li... \n", + "1 [library chatbot book, app library chatbot, li... \n", + "2 [library chatbot book, app library chatbot, li... \n", + "3 [library chatbot book, app library chatbot, li... \n", + "4 [library chatbot book, app library chatbot, li... \n", + "\n", + " query_clean_text \n", + "0 smart library app. smart library app. library ... \n", + "1 smart library app. smart library app. library ... \n", + "2 smart library app. smart library app. library ... \n", + "3 smart library app. smart library app. library ... \n", + "4 smart library app. smart library app. library ... " + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "final_results = find_similar_projects(\n", + " title=\"Smart Library app\",\n", + " description=\"\"\"\n", + " Library chatbot book recommendation search engine \n", + " \"\"\",\n", + " top_k=5\n", + ")\n", + "final_results" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 27107, + "status": "ok", + "timestamp": 1777121936703, + "user": { + "displayName": "yossef assem", + "userId": "00346058032754430335" + }, + "user_tz": -180 + }, + "id": "suEasutzr33J", + "outputId": "dacf67d9-6725-4437-bb10-6dd8e67d03f3" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "==============================\n", + "SELF RETRIEVAL TEST\n", + "==============================\n", + "Projects Tested : 20\n", + "Top1 Accuracy : 100.00%\n", + "==============================\n", + "\n", + "==============================\n", + "REAL QUERY TEST\n", + "==============================\n", + "\n", + "Query: AI Clinic Management System\n", + "Top Match: \n", + "Score: 0.6201\n", + "Risk: Medium\n", + "\n", + "Query: Smart Library Assistant\n", + "Top Match: \n", + "Score: 0.6127\n", + "Risk: Medium\n", + "\n", + "Query: Attendance Face Recognition\n", + "Top Match: \n", + "Score: 0.6796\n", + "Risk: Medium\n", + "\n", + "Query: E-commerce Recommendation Platform\n", + "Top Match: \n", + "Score: 0.5985\n", + "Risk: Medium\n", + "\n", + "==============================\n", + "Average Query Score: 0.6277\n", + "==============================\n", + "\n", + "==============================\n", + "FINAL MODEL STATUS\n", + "==============================\n", + "VERY GOOD ✅\n", + "Overall Score: 0.8511\n", + "==============================\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-05-09 03:48:08,243 | INFO | Load pretrained SentenceTransformer: all-MiniLM-L6-v2\n", + "'(ReadTimeoutError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)\"), '(Request ID: 396bd896-b6ff-4645-a377-6950f068eab0)')' thrown while requesting HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/modules.json\n", + "2026-05-09 03:48:18,553 | WARNING | '(ReadTimeoutError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)\"), '(Request ID: 396bd896-b6ff-4645-a377-6950f068eab0)')' thrown while requesting HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/modules.json\n", + "Retrying in 1s [Retry 1/5].\n", + "2026-05-09 03:48:18,553 | WARNING | Retrying in 1s [Retry 1/5].\n", + "'(ReadTimeoutError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)\"), '(Request ID: 7211a374-1e2e-41f0-924c-d68ebbe389e6)')' thrown while requesting HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/modules.json\n", + "2026-05-09 03:48:29,876 | WARNING | '(ReadTimeoutError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)\"), '(Request ID: 7211a374-1e2e-41f0-924c-d68ebbe389e6)')' thrown while requesting HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/modules.json\n", + "Retrying in 2s [Retry 2/5].\n", + "2026-05-09 03:48:29,876 | WARNING | Retrying in 2s [Retry 2/5].\n", + "2026-05-09 03:48:42,058 | INFO | Use pytorch device_name: cpu\n", + "2026-05-09 03:48:42,064 | INFO | Loading faiss with AVX2 support.\n", + "2026-05-09 03:48:42,089 | INFO | Successfully loaded faiss with AVX2 support.\n", + "2026-05-09 03:48:42,105 | INFO | Loading metadata...\n", + "2026-05-09 03:48:42,167 | INFO | Loading models and artifacts...\n", + "2026-05-09 03:48:42,167 | INFO | Loading model: all-MiniLM-L6-v2\n", + "2026-05-09 03:48:42,167 | INFO | Load pretrained SentenceTransformer: all-MiniLM-L6-v2\n", + "2026-05-09 03:48:45,345 | INFO | Use pytorch device_name: cpu\n", + "2026-05-09 03:48:45,347 | INFO | Loading FAISS index...\n", + "2026-05-09 03:48:45,347 | INFO | Loading feature model: all-MiniLM-L6-v2\n", + "2026-05-09 03:48:45,348 | INFO | Load pretrained SentenceTransformer: all-MiniLM-L6-v2\n", + "2026-05-09 03:48:47,664 | INFO | Use pytorch device_name: cpu\n", + "2026-05-09 03:48:47,665 | INFO | Preparing query...\n", + "2026-05-09 03:48:51,925 | INFO | Running semantic retrieval...\n", + "\n", + "Batches: 0%| | 0/1 [00:00 bool: + + if not title: + return True + + title = title.strip() + + words = title.split() + + # too short + if len(words) < 4: + return True + + weak_words = { + "system", + "platform", + "app", + "website", + "application", + "project", + "ai", + "smart", + "tool" + } + + meaningful = [ + w.lower() + for w in words + if w.lower() not in weak_words + ] + + return len(words) < 3 + + +def is_generic_project_reference(text: str) -> bool: + + text = text.strip().lower() + + generic_titles = { + "my project", + "this project", + "the project", + "my system", + "this system", + "my app", + "my application", + "my idea", + "project", + "system", + "app", + "idea" + } + + return text in generic_titles + +def looks_like_real_project_title(title: str) -> bool: + + if not title: + return False + + title = title.strip() + + words = title.split() + + # minimum words + if len(words) < 2: + return False + + # too repetitive + unique_ratio = len(set(words)) / len(words) + + if unique_ratio < 0.5: + return False + + # gibberish detection + nonsense_patterns = [ + "asd", + "qwe", + "zxc", + "testtest", + "aaaa", + "xxxxx" + ] + + lowered = title.lower() + + + # ========================= + # QUESTION / NORMAL CHAT FILTER + # ========================= + question_starts = ( + "how ", + "what ", + "why ", + "when ", + "where ", + "can ", + "could ", + "should ", + "is ", + "are ", + "do ", + "does " + ) + + + + for p in nonsense_patterns: + if p in lowered: + return False + + # at least one meaningful keyword + keywords = { + + # AI / Systems + "management", + "analysis", + "detection", + "tracking", + "recognition", + "monitoring", + "security", + "attendance", + "automation", + "prediction", + "dashboard", + "diagnosis", + "learning", + "recommendation", + "classification", + "authentication", + "optimization", + + # domains + "healthcare", + "fintech", + "education", + "library", + "hospital", + "school", + "medical", + "industrial", + "agriculture", + "transport", + + # technologies + "ai", + "iot", + "blockchain", + "cloud", + "robotics", + "vision", + "embedded", + + # project words + "system", + "platform", + "application", + "app" +} + + if not any( + k in lowered + for k in keywords + ): + return False + + return True + + +FOLLOWUP_WORDS = [ + "another", + "more", + "again", + "other ideas", + "more ideas", + "more features", + "another features" +] + +# ===================================================== +# RESPONSE FINALIZER +# ===================================================== +def finalize_response( + user_input, + response, + history, + state, + user_id +): + + history.append({ + "role": "user", + "content": user_input + }) + + history.append({ + "role": "assistant", + "content": response + }) + + history = history[-20:] + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return response + + +def is_gibberish_text(text: str) -> bool: + + text = text.strip().lower() + + # allow menu choices + if text in {"1", "2", "3"}: + return False + + # very short non-numeric random text + if len(text) < 3: + + # allow short meaningful commands + allowed_short = { + "hi", + "hey", + "hello", + "ai", + "ml", + "ui", + "ux", + "vr", + "ar", + "iot" + } + + if text in allowed_short: + return False + + return True + + gibberish_patterns = [ + "asd", + "qwe", + "zxc", + "aaa", + "bbb", + "ccc", + "xxx", + "testtest" + ] + + for p in gibberish_patterns: + if p in text: + return True + + words = text.split() + + # repeated same words + if len(words) >= 3: + + unique_ratio = len(set(words)) / len(words) + + if unique_ratio < 0.5: + return True + + return False + +def is_project_related(text: str) -> bool: + + text = text.lower().strip() + + keywords = [ + + # project words + "project", + "system", + "platform", + "application", + "app", + "website", + "dashboard", + "management", + + # technical + "ai", + "ml", + "machine learning", + "deep learning", + "computer vision", + "blockchain", + "iot", + "web", + "mobile", + "cloud", + "security", + "database", + "api", + + # actions + "generate", + "feature", + "features", + "idea", + "ideas", + "improve", + "description", + "technologies", + "architecture", + + # domains + "healthcare", + "education", + "fintech", + "smart", + "attendance", + "monitoring", + "tracking", + "analysis", + "recognition" + ] + + return any( + keyword in text + for keyword in keywords + ) + +# ===================================================== +# MAIN CHATBOT +# ===================================================== +def chatbot(user_id: str, user_input: str): + + text = user_input.lower().strip() + + + # ========================================= + # EXPLICIT IDEA REQUESTS + # ========================================= + explicit_idea_requests = [ + "idea", + "new idea", + "another idea", + "project idea", + "give me idea", + "i want idea", + "suggest idea", + "generate idea", + "generate ideas", + "give me ideas", + "i need idea", + "ideas" + ] + + if any(x in text for x in explicit_idea_requests): + + state = get_user_memory(user_id).get("state") or default_state() + + state["waiting_for_domain"] = True + + save_user_memory(user_id, { + "history": get_user_memory(user_id).get("history", []), + "state": state + }) + + return ( + "🎯 Specify a domain (AI, healthcare, fintech...)" + ) + + # ================================================= + # ALLOW MENU NUMBERS + # ================================================= + if text in {"1", "2", "3"}: + pass + # ================================================= + # GIBBERISH DETECTION + # ================================================= + if is_gibberish_text(text): + + return ( + "⚠️ I could not understand your request.\n\n" + "Try something meaningful like:\n" + "- generate AI project ideas\n" + "- generate features for smart hospital system\n" + "- improve my project\n" + "- suggest technologies for fintech app" + ) + # ================================================= + # COMMANDS + # ================================================= + if is_command(user_input): + return handle_command(user_input) + + # ================================================= + # MEMORY + # ================================================= + memory = get_user_memory(user_id) + + history = memory.get("history", []) + + state = memory.get("state") or default_state() + + # ensure keys + state.setdefault("menu_mode", False) + state.setdefault("selected_option", None) + state.setdefault("waiting_for_domain", False) + state.setdefault("waiting_for_project_action", False) + state.setdefault("project_chat_mode", False) + state.setdefault("last_action", None) + state.setdefault("domain", None) + state.setdefault("weak_title_candidate", None) + state.setdefault("waiting_for_feature_title", False) + state.setdefault("waiting_for_full_project_domain", False) + state.setdefault("waiting_for_full_project_selection", False) + + # ================================================= + # GREETING + # ================================================= + if text in ["","hi", "hello", "hey"]: + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + "👋 Welcome!\n\n" + "I can help you with:\n" + "• Graduation project ideas\n" + "• Smart feature generation\n" + "• Project improvement\n" + "• Technology suggestions\n" + "• Project descriptions\n\n" + "Try saying things like:\n" + "- give me AI project ideas\n" + "- generate features for smart hospital system\n" + "- improve my graduation project\n" + "- suggest technologies for fintech app" + ) + # ================================================= + # FULL PROJECT STARTER + # ================================================= + if any(x in text for x in [ + + "generate full project", + "final project", + "complete project", + "full graduation project", + "all fields", + "fields" + + ]): + + # ============================================ + # IF PROJECT TITLE ALREADY EXISTS + # ============================================ + if state.get("project_title"): + + # Generate features automatically if missing + if not state.get("features"): + + feature_result = generate_features( + + title=state.get("project_title"), + + description=state.get("description", ""), + + features=[], + + previous_generated_features=[], + + top_k=8 + ) + + state["features"] = feature_result.get( + "recommended_features", + [] + ) + + # Generate full project + result = generate_full_project( + + title=state.get("project_title"), + + features=state.get("features", []), + + description=state.get("description", ""), + + abstract=state.get("abstract", "") + ) + + state = update_state( + state, + result, + mode="merge" + ) + + response = f""" +📦 Full Project Generated + +📌 Title: +{state.get("project_title")} + +📄 Abstract: +{state.get("abstract")} + +📄 Description: +{state.get("description")} + +⚙️ Features: +{chr(10).join("- " + x for x in state.get("features", []))} + +🛠 Technologies: +{chr(10).join("- " + x for x in state.get("technologies", []))} + +🎯 Objectives: +{chr(10).join("- " + x for x in state.get("objectives", []))} + +⚡ Methodology: +{state.get("methodology")} + +🚀 Future Work: +{chr(10).join("- " + x for x in state.get("future_work", []))} + +📂 Category: +{state.get("category")} + +🏷 Keywords: +{", ".join(state.get("keywords", []))} + +❗ Problem Statement: +{state.get("problem_statement")} + +💡 Proposed Solution: +{state.get("proposed_solution")} + +🤖 AI Summary: +{state.get("ai_summary")} + """ + + return finalize_response( + user_input, + response, + history, + state, + user_id + ) + + # ============================================ + # NO PROJECT TITLE → ASK FOR DOMAIN + # ============================================ + state["waiting_for_full_project_domain"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + "🎯 What domain do you want?\n\n" + "Examples:\n" + "- AI\n" + "- healthcare\n" + "- education\n" + "- fintech" + ) + # ================================================= + # MENU HANDLER + # ================================================= + if state.get("menu_mode"): + + if text in ["1", "2", "3"]: + + state["menu_mode"] = False + state["selected_option"] = text + + if text == "1": + + state["waiting_for_domain"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return "🎯 Enter a domain (AI, healthcare, fintech...)" + + elif text == "2": + + if not state.get("project_title"): + + state["waiting_for_feature_title"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + "⚠️ Please provide the project title first.\n\n" + "Example:\n" + "- AI-powered smart library recommendation system\n\n" + "👉 You can also ask me to generate an idea first." + ) + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return chatbot(user_id, "generate features") + + elif text == "3": + + if not state.get("project_title"): + + return ( + "⚠️ Please generate or provide " + "a project idea first." + ) + + state["project_chat_mode"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + f"💬 Project discussion mode enabled for:\n" + f"{state.get('project_title')}" + ) + + state["menu_mode"] = False + + # ================================================= + # PROJECT ACTION HANDLER + # ================================================= + if state.get("waiting_for_project_action"): + + feature_words = [ + "1", + "feature", + "features", + "generate features", + "add features", + "create features" + ] + + chat_words = [ + "2", + "chat", + "talk", + "discussion", + "discuss", + "improve", + "improve idea", + "talk about project" + ] + + full_project_words = [ + "3", + "full project", + "generate full project", + "complete project", + "final project" +] + + # ========================================= + # FEATURE GENERATION + # ========================================= + if any(w in text for w in feature_words): + + state["waiting_for_project_action"] = False + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return chatbot(user_id, "generate features") + + # ========================================= + # PROJECT DISCUSSION + # ========================================= + elif any(w in text for w in chat_words): + + state["waiting_for_project_action"] = False + + state["project_chat_mode"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + f"💬 Chat mode enabled for:\n" + f"{state.get('project_title')}\n\n" + "You can now discuss or improve the project." + ) + + # ========================================= + # FEATURE GENERATION + # ========================================= + if any(w in text for w in feature_words): + + state["waiting_for_project_action"] = False + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return chatbot(user_id, "generate features") + + + # ========================================= + # PROJECT DISCUSSION + # ========================================= + elif any(w in text for w in chat_words): + + state["waiting_for_project_action"] = False + + state["project_chat_mode"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + f"💬 Chat mode enabled for:\n" + f"{state.get('project_title')}\n\n" + "You can now discuss or improve the project." + ) + + + # ========================================= + # FULL PROJECT GENERATION + # ========================================= + elif any(w in text for w in full_project_words): + + state["waiting_for_project_action"] = False + + # Generate features first if missing + if not state.get("features"): + + feature_result = generate_features( + + title=state.get("project_title"), + + description=state.get("description", ""), + + features=[], + + previous_generated_features=[], + + top_k=8 + ) + + state["features"] = feature_result.get( + "recommended_features", + [] + ) + + # Generate full project + result = generate_full_project( + + title=state.get("project_title"), + + features=state.get("features", []), + + description=state.get("description", ""), + + abstract=state.get("abstract", "") + ) + + state = update_state( + state, + result, + mode="merge" + ) + + response = f""" +📦 Full Project Generated + +📌 Title: +{state.get("project_title")} + +📄 Abstract: +{state.get("abstract")} + +📄 Description: +{state.get("description")} + +⚙️ Features: +{chr(10).join("- " + x for x in state.get("features", []))} + +🛠 Technologies: +{chr(10).join("- " + x for x in state.get("technologies", []))} + +🎯 Objectives: +{chr(10).join("- " + x for x in state.get("objectives", []))} + +⚡ Methodology: +{state.get("methodology")} + +🚀 Future Work: +{chr(10).join("- " + x for x in state.get("future_work", []))} + +📂 Category: +{state.get("category")} + +🏷 Keywords: +{", ".join(state.get("keywords", []))} + +❗ Problem Statement: +{state.get("problem_statement")} + +💡 Proposed Solution: +{state.get("proposed_solution")} + +🤖 AI Summary: +{state.get("ai_summary")} +""" + + return finalize_response( + user_input, + response, + history, + state, + user_id + ) + + # ================================================= + # WEAK TITLE IMPROVEMENT HANDLER + # ================================================= + if state.get("weak_title_candidate"): + + improve_words = [ + "1", + "make it descriptive", + "improve title", + "rewrite title", + "make better" + ] + + if any(w in text for w in improve_words): + + weak_title = state.get("weak_title_candidate") + + prompt = f""" + You are a senior software architect. + + Convert this weak graduation project title + into a professional and descriptive title. + + Weak title: + {weak_title} + + Rules: + - Keep original meaning + - Make it specific + - Make it professional + - Keep it concise + - Return ONLY the improved title + """ + + improved_title = generate_text( + prompt, + task="chat" + ).strip() + + state["weak_title_candidate"] = None + state["project_title"] = improved_title + state["waiting_for_project_action"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + f"📌 Improved Project Title:\n" + f"{improved_title}\n\n" + "Choose what you want next:\n" + "1️⃣ Generate features\n" + "2️⃣ Talk with chatbot about the idea\n\n" + "👉 You can also say:\n" + "- generate features\n" + "- discuss project" + ) + + + # ================================================= + # WAITING FOR FEATURE TITLE + # ================================================= + if state.get("waiting_for_feature_title"): + + state["waiting_for_feature_title"] = False + + possible_title = user_input.strip() + + # weak title + if is_weak_project_title(possible_title): + + state["weak_title_candidate"] = possible_title + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + "⚠️ Your project title is too short or unclear.\n\n" + "Examples:\n" + "- AI-powered smart library recommendation system\n" + "- Smart hospital emergency response platform\n\n" + "Choose:\n" + "1️⃣ Make the title more descriptive\n" + "2️⃣ Enter another title" + ) + + # valid title + state["project_title"] = possible_title + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return chatbot(user_id, "generate features") + + # ================================================= + # DIRECT FEATURE TITLE EXTRACTION + # ================================================= + feature_patterns = [ + # standard + "generate features for", + "generate feature for", + "features for", + "feature for", + "create features for", + "suggest features for", + + # casual + "give me features for", + "show features for", + "i need features for", + "can you generate features for", + "can you suggest features for", + + # project related + "add features to", + "improve features for", + "feature ideas for", + "smart features for", + + # short forms + "features:", + "project:", + "idea:", + + # typo tolerance + "gen features for", + "generate faetures for", + "fetures for", + + # direct requests + "make features for", + "build features for", + "what features for", + "best features for", + + # ai chatbot style + "i have project", + "my project is", + "project title is", + ] + + analysis = None + + for pattern in feature_patterns: + + if text.startswith(pattern + " "): + + extracted_title = user_input.split( + pattern, 1 + )[1].strip() + + if ( + extracted_title + and not is_generic_project_reference(extracted_title) + ): + + analysis = { + "intent": "feature", + "project_title": extracted_title + } + + break + + # ================================================= + # SIMPLE PROJECT TITLE DETECTION (BEFORE LLM) + # ================================================= + if analysis is None: + + blocked_starts = ( + "how ", + "what ", + "why ", + "when ", + "where ", + "can ", + "could ", + "should ", + "is ", + "are ", + "do ", + "does ", + "help ", + ) + action_words = { + "generate", + "create", + "suggest", + "give", + "show", + "make", + "build", + "improve", + "discuss", + "help", + "recommend", + "need" +} + + command_phrases = { + + # idea commands + "new idea", + "another idea", + "generate idea", + "generate ideas", + "project ideas", + "more ideas", + "idea", + + # feature commands + "generate features", + "generate feature", + "more features", + "features", + + # full project + "generate full project", + "full project", + "complete project", + + # generic commands + "another", + "again", + "more", + "chat", + "help", + "menu", + "discuss", + "discuss project" +} + + clean_text = text.strip() + + # detect short titles directly + words = clean_text.lower().split() + + if ( + 2 <= len(words) <= 8 + and not clean_text.endswith("?") + and not clean_text.startswith(blocked_starts) + and clean_text.lower() not in command_phrases + and not any( + w in action_words + for w in words + ) + ): + + analysis = { + "intent": "project_title", + "project_title": user_input.strip() + } + + else: + analysis = analyze_user_input(user_input, state) + + # ================================================= + # IDEA SELECTION DETECTION + # ================================================= + existing_ideas = state.get("ideas", []) + + normalized_input = text.strip().lower() + + for idea in existing_ideas: + + idea_norm = idea.strip().lower() + + input_words = set(normalized_input.split()) + + # ignore extremely short inputs + if len(input_words) < 2: + continue + + idea_words = set(idea_norm.split()) + + overlap = input_words & idea_words + + overlap_ratio = ( + len(overlap) / max(1, len(input_words)) + ) + + if ( + normalized_input == idea_norm + or normalized_input in idea_norm + or overlap_ratio >= 0.6 + ): + + state["project_title"] = idea + + state["waiting_for_project_action"] = True + + state["project_chat_mode"] = False + + state["waiting_for_full_project_selection"] = False + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return finalize_response( + + user_input, + + f"""📌 Project Selected: + +{idea} + +Choose what you want: + +1️⃣ Generate Features +2️⃣ Discuss Project +3️⃣ Generate Full Project + +👉 You can also type: +- features +- discuss +- full project +""", + + history, + state, + user_id + ) + + + # ===================================== + # NORMAL IDEA SELECTION + # ===================================== + state["project_title"] = idea + state["waiting_for_project_action"] = True + state["project_chat_mode"] = False + + save_user_memory(user_id, { + "history": history, + "state": state + }) + return ( + f"📌 Project selected:\n" + f"{idea}\n\n" + "Choose what you want next:\n" + "1️⃣ Generate features\n" + "2️⃣ Talk with chatbot about the idea\n\n" + "👉 You can also say:\n" + "- generate features\n" + "- discuss project" + ) + + # ================================================= + # FOLLOW-UP CONTINUATION + # ================================================= + if any(w in text for w in FOLLOWUP_WORDS): + + # continue ideas + if state.get("last_action") == "idea": + + analysis["intent"] = "idea" + analysis["domain"] = state.get("domain") + + # continue features + elif state.get("last_action") == "feature": + + analysis["intent"] = "feature" + + # ================================================= + # DOMAIN FOLLOW-UP + # ================================================= + if state.get("waiting_for_domain"): + + detected = extract_domain(user_input) + + if detected: + + state["waiting_for_domain"] = False + + analysis["domain"] = detected + analysis["intent"] = "idea" + + else: + + return ( + "⚠️ Please enter a valid domain " + "(AI, healthcare, fintech...)" + ) + + intent = analysis.get("intent", "chat") + + domain = analysis.get("domain") + + user_project = analysis.get("project_title") + + # ================================================= + # FULL PROJECT DOMAIN HANDLER + # ================================================= + if state.get("waiting_for_full_project_domain"): + + domain = extract_domain(user_input) + + if not domain: + + return ( + "⚠️ Please enter valid domain.\n\n" + "Example:\n" + "- AI\n" + "- healthcare\n" + "- education" + ) + + state["waiting_for_full_project_domain"] = False + + result = generate_ideas( + domain=domain, + top_k=5 + ) + + ideas = result.get("final_ideas", []) + + state["ideas"] = ideas + + state["waiting_for_full_project_selection"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + "💡 Choose one project title:\n\n" + + "\n".join( + f"- {idea}" + for idea in ideas + ) + ) + # ================================================= + # FALLBACK DOMAIN + # ================================================= + if not domain: + domain = extract_domain(text) + + # ================================================= + # DOMAIN-ONLY INPUT DETECTION + # ================================================= + full_project_commands = [ + "fill all", + "fill all columns", + "fill all fields", + "full project", + "final project", + "complete project", + "generate full project", + "generate final project", + "generate all details", + "project specification", + "complete details" + ] + + if ( + domain + and intent == "chat" + and len(text.split()) <= 3 + and not any(cmd in text for cmd in full_project_commands) + ): + intent = "idea" + # ================================================= + # FALLBACK PROJECT TITLE DETECTION + # ================================================= + if ( + not user_project + and intent == "chat" + and 3 <= len(text.split()) <= 12 + ): + idea_request_patterns = [ + + "new idea", + "generate idea", + "generate ideas", + "project ideas", + "idea for", + "ideas for", + "i need idea", + "i need ideas", + "need idea", + "need ideas", + "another idea", + "more ideas", + +] + # reject obvious commands/questions + blocked_starts = ( + "how ", + "what ", + "why ", + "when ", + "where ", + "can ", + "could ", + "should ", + "is ", + "are ", + "do ", + "does ", + "help ", + ) + + if text.endswith("?"): + pass + + elif text.startswith(blocked_starts): + pass + + else: + + command_phrases = { + "generate features", + "generate feature", + "generate ideas", + "generate idea", + "more", + "another", + "again", + "discuss project", + "chat", + "help", + "menu" + } + + if ( + text not in command_phrases + and not any( + p in text + for p in idea_request_patterns + ) + ): + + # treat short non-question text as project title + user_project = user_input.strip() + # ================================================= + # USER PROVIDED PROJECT + # ================================================= + if user_project: + + # ============================================= + # WEAK TITLE VALIDATION + # ============================================= + if is_weak_project_title(user_project): + + state["weak_title_candidate"] = user_project + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + "⚠️ Your project title is too short or unclear.\n\n" + "Examples:\n" + "- AI-powered smart library recommendation system\n" + "- Smart hospital emergency response platform\n" + "- Blockchain-secured academic certificate verification system\n\n" + "Choose:\n" + "1️⃣ Make the title more descriptive\n" + "2️⃣ Enter another title\n\n" + "👉 You can also say:\n" + "- make it descriptive" + ) + + state = default_state() + + state["project_title"] = user_project + state["waiting_for_project_action"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return ( + f"📌 Project detected:\n" + f"{user_project}\n\n" + "Choose what you want next:\n" + "1️⃣ Generate features\n" + "2️⃣ Talk with chatbot about the idea\n" + "3️⃣ Generate full project\n\n" + "👉 You can also say:\n" + "- generate features\n" + "- discuss project\n" + "- generate full project" +) + # ================================================= + # FEATURE REQUEST WITHOUT TITLE + # ================================================= + feature_request_phrases = [ + "generate features", + "generate feature", + "i need features", + "feature ideas", + "add features", + "create features", + "suggest features", + "improve my project", + "improve project", + "improve my idea", + "enhance my project", + "discuss my project", + "talk about my project", + "help my project", + "improve" + ] + + if ( + any(p in text for p in feature_request_phrases) + and not state.get("project_title") + and not analysis.get("project_title") + ): + + return ( + "Please give me the project title first.\n\n" + "Example:\n" + "- AI-powered smart library recommendation system\n" + "- Smart hospital emergency response platform" + ) + # ================================================= + # IDEA GENERATION + # ================================================= + if intent == "idea": + + if not domain: + + state["waiting_for_domain"] = True + + save_user_memory(user_id, { + "history": history, + "state": state + }) + + return "🎯 Specify a domain (AI, healthcare, fintech...)" + + top_k = analysis.get("number") or extract_number( + user_input, + 5 + ) + + result = generate_ideas( + domain=domain, + top_k=top_k, + previous_generated_ideas=state.get("ideas", []) + ) + + ideas = result.get("final_ideas", []) + + state["ideas"] = ideas + state["domain"] = domain + state["last_action"] = "idea" + + response = format_response("idea", "", state) + + return finalize_response( + user_input, + response, + history, + state, + user_id + ) + + # ================================================= + # FEATURE GENERATION + # ================================================= + elif intent == "feature": + + # ========================================= + # DIRECT PROJECT TITLE FROM USER INPUT + # ========================================= + if analysis.get("project_title"): + + state["project_title"] = ( + analysis["project_title"] + ) + + if not state.get("project_title"): + + return ( + "Please give me the project title first." + ) + + top_k = analysis.get("number") or extract_number( + user_input, + 5 + ) + + result = generate_features( + title=state.get("project_title"), + description=state.get("description", ""), + features=[], + previous_generated_features=[], + top_k=top_k + ) + + features = result.get( + "recommended_features", + [] + ) + + # IMPORTANT: + # Replace old features completely + state["features"] = features + + state["last_action"] = "feature" + + response = format_response("feature", "", state) + return finalize_response( + user_input, + response, + history, + state, + user_id + ) + + + + # ================================================= + # FULL PROJECT GENERATION + # ================================================= + elif any(x in text for x in [ + + "fill all", + "fill all columns", + "fill all fields", + "full project", + "final project", + "complete project", + "generate full project", + "generate final project", + "generate all details", + "project specification", + "complete details" + +]): + + if not state.get("project_title"): + + return ( + "Please generate project idea first." + ) + + if not state.get("features"): + + return ( + "Please generate features first." + ) + + result = generate_full_project( + + title=state.get("project_title"), + + features=state.get("features", []), + + description=state.get("description", ""), + + abstract=state.get("abstract", "") + ) + + state = update_state( + state, + result, + mode="merge" + ) + + response = f""" +📦 Full Project Generated + +📌 Title: +{state.get("project_title")} + +📄 Abstract: +{state.get("abstract")} + +📄 Description: +{state.get("description")} + +🛠 Technologies: +{chr(10).join("- " + x for x in state.get("technologies", []))} + +🎯 Objectives: +{chr(10).join("- " + x for x in state.get("objectives", []))} + +⚡ Methodology: +{state.get("methodology")} + +🚀 Future Work: +{chr(10).join("- " + x for x in state.get("future_work", []))} + +📂 Category: +{state.get("category")} + +🏷 Keywords: +{", ".join(state.get("keywords", []))} + +❗ Problem Statement: +{state.get("problem_statement")} + +💡 Proposed Solution: +{state.get("proposed_solution")} + +🤖 AI Summary: +{state.get("ai_summary")} +""" + + return finalize_response( + user_input, + response, + history, + state, + user_id + ) + + # ================================================= + # POSSIBLE NEW PROJECT DETECTION + # ================================================= + elif ( + state.get("project_title") + and len(text.split()) >= 3 + and not any(w in text for w in FOLLOWUP_WORDS) + ): + + current_title = ( + state.get("project_title", "") + .lower() + .split() + ) + + new_input_words = text.split() + + overlap = set(current_title) & set(new_input_words) + + overlap_ratio = ( + len(overlap) + / max(1, len(new_input_words)) + ) + + # low overlap may indicate new project/topic + if ( + overlap_ratio < 0.2 + and analysis.get("intent") == "chat" + ): + + return ( + "⚠️ This seems unrelated to the current project.\n\n" + f"Current project:\n" + f"{state.get('project_title')}\n\n" + "Do you want to:\n" + "1️⃣ Start a new project\n" + "2️⃣ Continue discussing the current project" + ) + + system_prompt = build_chat_prompt(state) + + formatted_history = "\n".join([ + f"{m['role']}: {m['content']}" + for m in history[-6:] + ]) + + full_prompt = f""" +{system_prompt} + +Conversation: +{formatted_history} + +User: +{user_input} +""" + + raw = generate_text( + full_prompt, + task="chat" + ) + + response = format_response( + "chat", + raw, + state + ) + return finalize_response( + user_input, + response, + history, + state, + user_id + ) + # ================================================= + # NON PROJECT FILTER + # ================================================= + if not is_project_related(text): + + return ( + "⚠️ This chatbot is specialized for graduation projects only.\n\n" + "Try things like:\n" + "- generate AI project ideas\n" + "- generate features for smart hospital system\n" + "- suggest technologies for fintech app\n" + "- improve my graduation project" + ) + # ================================================= + # GENERAL CHAT + # ================================================= + else: + + system_prompt = build_chat_prompt(state) + + formatted_history = "\n".join([ + f"{m['role']}: {m['content']}" + for m in history[-6:] + ]) + + full_prompt = f""" +{system_prompt} + +Conversation: +{formatted_history} + +User: +{user_input} +""" + + raw = generate_text( + full_prompt, + task="chat" + ) + + response = format_response( + "chat", + raw, + state + ) + return finalize_response( + user_input, + response, + history, + state, + user_id + ) + + \ No newline at end of file diff --git a/src/recommendation_engine/command_handler.py b/src/recommendation_engine/command_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..95f428a7bbdc64877cb168921fb0574942068e67 --- /dev/null +++ b/src/recommendation_engine/command_handler.py @@ -0,0 +1,86 @@ +# src/recommendation_engine/command_handler.py + +import re + + +# ===================================================== +# Command Detection (Improved) +# ===================================================== +def is_command(text: str) -> bool: + """ + Detect if input is a system/terminal command + """ + + if not text: + return False + + text = text.strip().lower() + + # ========================================= + # Exact commands + # ========================================= + if text in {"exit", "quit", "clear"}: + return True + + # ========================================= + # Command-like patterns (regex) + # ========================================= + command_patterns = [ + r"^python\s+\S+\.py", # python file.py + r"^pip\s+install", # pip install + r"^npm\s+install", # npm install + r"^node\s+\S+", # node file.js + r"^cd\s+.+", # cd folder + r"^ls\b", # ls + r"^dir\b", # dir + r"^git\s+.+", # git command + r"^sudo\s+.+", # sudo command + r".+\.py\s*$", # ends with file.py (ONLY if standalone) + ] + + for pattern in command_patterns: + if re.match(pattern, text): + return True + + # ========================================= + # Inline execution attempts + # ========================================= + if "run" in text and re.search(r"\.py\b", text): + return True + + return False + + +# ===================================================== +# Command Handling +# ===================================================== +def handle_command(text: str) -> str: + """ + Return safe response for command-like inputs + """ + + text = text.lower().strip() + + # exit commands + if text in {"exit", "quit"}: + return "👋 Session ended. Start a new chat anytime." + + # python execution + if re.search(r"python\s+\S+\.py", text): + return ( + "⚠️ This looks like a code execution command.\n" + "I only help with graduation project ideas and development." + ) + + # package installation + if "pip install" in text or "npm install" in text: + return ( + "⚠️ Installation commands are outside my scope.\n" + "I can help you design your graduation project instead." + ) + + # generic fallback + return ( + "⚠️ This looks like a system command.\n" + "Please ask about graduation projects (ideas, features, or system design)." + ) \ No newline at end of file diff --git a/src/recommendation_engine/config.py b/src/recommendation_engine/config.py new file mode 100644 index 0000000000000000000000000000000000000000..f3095e7c7b290109f74d4be913f86d83c9fd68ee --- /dev/null +++ b/src/recommendation_engine/config.py @@ -0,0 +1,132 @@ +# src/recommendation_engine/config.py + +import os +from pathlib import Path +from dotenv import load_dotenv + +# ===================================================== +# Load Environment Variables +# ===================================================== +BASE_DIR = Path(__file__).resolve().parent.parent.parent +ENV_PATH = BASE_DIR / ".env" + +if ENV_PATH.exists(): + load_dotenv(ENV_PATH) + +# ===================================================== +# App Mode +# ===================================================== +ENV = os.getenv("ENV", "development") +DEBUG_MODE = os.getenv("DEBUG_MODE", "true").lower() == "true" + +# ===================================================== +# Gemini API Config +# ===================================================== +GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "") + +MODEL_CANDIDATES = [ + os.getenv("PRIMARY_MODEL", "gemini-3.1-flash-lite-preview"), + os.getenv("FAST_MODEL", "gemini-2.5-flash-lite"), + os.getenv("BALANCED_MODEL", "gemini-2.5-flash"), + os.getenv("QUALITY_MODEL", "gemini-2.5-pro"), +] + +# remove duplicates safely +_seen = set() +MODEL_CANDIDATES = [ + m for m in MODEL_CANDIDATES + if m and not (m in _seen or _seen.add(m)) +] + +# ===================================================== +# Generation Settings +# ===================================================== +IDEA_TEMPERATURE = float(os.getenv("IDEA_TEMPERATURE", 0.9)) +FEATURE_TEMPERATURE = float(os.getenv("FEATURE_TEMPERATURE", 0.6)) +CHAT_TEMPERATURE = float(os.getenv("CHAT_TEMPERATURE", 0.7)) +INTENT_TEMPERATURE = float(os.getenv("INTENT_TEMPERATURE", 0.0)) + +# Max tokens +IDEA_MAX_TOKENS = int(os.getenv("IDEA_MAX_TOKENS", 800)) +FEATURE_MAX_TOKENS = int(os.getenv("FEATURE_MAX_TOKENS", 600)) +CHAT_MAX_TOKENS = int(os.getenv("CHAT_MAX_TOKENS", 700)) +INTENT_MAX_TOKENS = int(os.getenv("INTENT_MAX_TOKENS", 20)) + +TOP_P = float(os.getenv("TOP_P", 0.95)) +TOP_K = int(os.getenv("TOP_K", 40)) + +# ===================================================== +# Retry / Timeout +# ===================================================== +REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", 30)) +MAX_RETRIES = int(os.getenv("MAX_RETRIES", 3)) +RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", 2)) + +# ===================================================== +# Recommendation Defaults +# ===================================================== +DEFAULT_FEATURE_COUNT = int(os.getenv("DEFAULT_FEATURE_COUNT", 5)) +DEFAULT_IDEA_COUNT = int(os.getenv("DEFAULT_IDEA_COUNT", 5)) +GENERATION_BATCH_SIZE = int(os.getenv("GENERATION_BATCH_SIZE", 10)) + +# ===================================================== +# Duplicate Detection +# ===================================================== +IDEA_DUPLICATE_THRESHOLD = float( + os.getenv("IDEA_DUPLICATE_THRESHOLD", 0.82) +) + +FEATURE_DUPLICATE_THRESHOLD = float( + os.getenv("FEATURE_DUPLICATE_THRESHOLD", 0.75) +) + +# ===================================================== +# Idea Generator Control +# ===================================================== +MAX_IDEA_RETRIES = int(os.getenv("MAX_IDEA_RETRIES", 3)) +MAX_IDEA_LIMIT = int(os.getenv("MAX_IDEA_LIMIT", 20)) + +# ===================================================== +# Similarity Engine +# ===================================================== +SIMILARITY_TOP_K = int(os.getenv("SIMILARITY_TOP_K", 5)) + +# ===================================================== +# State / Memory Control +# ===================================================== +MAX_HISTORY = int(os.getenv("MAX_HISTORY", 20)) +MAX_FEATURES = int(os.getenv("MAX_FEATURES", 20)) +MAX_IDEAS = int(os.getenv("MAX_IDEAS", 10)) + +# ===================================================== +# Semantic Routing +# ===================================================== +ENABLE_SEMANTIC_INTENT = os.getenv( + "ENABLE_SEMANTIC_INTENT", "true" +).lower() == "true" + +# ===================================================== +# Logging +# ===================================================== +ENABLE_LOGGING = os.getenv("ENABLE_LOGGING", "true").lower() == "true" +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") + +# ===================================================== +# Safety Check (🔥 IMPORTANT) +# ===================================================== +if not GEMINI_API_KEY: + raise ValueError( + "❌ GEMINI_API_KEY is missing. Please set it in .env file." + ) + +# ===================================================== +# Debug Print +# ===================================================== +if DEBUG_MODE and ENV == "development": + print("\n CONFIG LOADED:") + print(f"ENV: {ENV}") + print(f"DEBUG_MODE: {DEBUG_MODE}") + print(f"MODELS: {MODEL_CANDIDATES}") + print(f"MAX_RETRIES: {MAX_RETRIES}") + print(f"IDEA_TEMP: {IDEA_TEMPERATURE}") + print("=================================\n") \ No newline at end of file diff --git a/src/recommendation_engine/context_builder.py b/src/recommendation_engine/context_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..9bc9dd1d8336b49f1f671bc26000f4a73430cfcd --- /dev/null +++ b/src/recommendation_engine/context_builder.py @@ -0,0 +1,556 @@ +# src/recommendation_engine/context_builder.py + +import logging +import re + +from collections import Counter +from typing import Dict, Any, List +from functools import lru_cache +from difflib import get_close_matches + +import pandas as pd + +from src.similarity_model import ( + find_similar_projects, + extract_features +) + +from src.recommendation_engine.config import ( + SIMILARITY_TOP_K, + MAX_FEATURES +) + +logger = logging.getLogger(__name__) + + +# ===================================================== +# DOMAIN DEFINITIONS +# ===================================================== +DOMAIN_KEYWORDS = { + "artificial intelligence": [ + "ai", + "artificial intelligence", + "machine learning", + "ml", + "deep learning", + "neural network", + "nlp", + "computer vision" + ], + + "healthcare": [ + "hospital", + "health", + "medical", + "healthcare", + "clinic", + "patient" + ], + + "fintech": [ + "fintech", + "finance", + "bank", + "payment", + "crypto", + "blockchain" + ], + + "education": [ + "education", + "school", + "learning", + "edtech", + "student", + "university" + ], + + "ecommerce": [ + "ecommerce", + "shopping", + "retail", + "store", + "marketplace" + ], + + "agriculture": [ + "agriculture", + "farming", + "crop", + "livestock", + "smart farming" + ], + + "security": [ + "security", + "cyber", + "cybersecurity", + "threat", + "attack", + "malware" + ], + + "general": [ + "general", + "random", + "anything", + "any", + "whatever", + "surprise me", + "mixed", + "all", + "open", + "everything" + ] +} + + +# ===================================================== +# Helpers +# ===================================================== +def normalize(text: str) -> str: + + text = str(text).lower().strip() + + text = re.sub(r"[^a-z0-9\s]", " ", text) + + text = re.sub(r"\s+", " ", text).strip() + + return text + + +def clean_list( + items: List[str], + limit: int = 20 +) -> List[str]: + + final = [] + + seen = set() + + for item in items: + + val = normalize(item) + + if not val: + continue + + if val not in seen: + + seen.add(val) + + final.append(val) + + return final[:limit] + + +# ===================================================== +# Multi-domain detection +# ===================================================== +def detect_domains(text: str) -> List[str]: + + text = normalize(text) + + detected = [] + + words_in_text = set(text.split()) + + for domain, words in DOMAIN_KEYWORDS.items(): + + for w in words: + + # phrase + if " " in w: + + if w in text: + + detected.append(domain) + break + + # word + else: + + if w in words_in_text: + + detected.append(domain) + break + + return clean_list(detected, limit=3) + + +# ===================================================== +# Main domain extractor +# ===================================================== +def extract_domain(text: str) -> str: + + if not text: + return "" + + text = normalize(text) + + # hard normalization + if text in ["ai", "ml"]: + return "artificial intelligence" + + # direct domain match + if text in DOMAIN_KEYWORDS: + return text + + # multi-domain detection + domains = detect_domains(text) + + if domains: + + # prioritize non-general + for d in domains: + + if d != "general": + return d + + return domains[0] + + # fuzzy typo matching + all_words = [] + + word_map = {} + + for domain, words in DOMAIN_KEYWORDS.items(): + + for w in words: + + all_words.append(w) + + word_map[w] = domain + + match = get_close_matches( + text, + all_words, + n=1, + cutoff=0.75 + ) + + if match: + return word_map[match[0]] + + # partial match + for domain, words in DOMAIN_KEYWORDS.items(): + + for w in words: + + if text in w or w.startswith(text): + + return domain + + return "" + + +# ===================================================== +# Similarity cache +# ===================================================== +@lru_cache(maxsize=100) +def cached_similarity( + title: str, + description: str +): + + return find_similar_projects( + title=title, + description=description, + top_k=SIMILARITY_TOP_K + ) + + +# ===================================================== +# Common feature extraction +# ===================================================== +def extract_common_features( + results: pd.DataFrame +) -> List[str]: + + counter = Counter() + + if not isinstance(results, pd.DataFrame): + return [] + + for _, row in results.iterrows(): + + matches = row.get( + "matched_features", + [] + ) + + for item in matches: + + if isinstance(item, dict): + + feat = item.get( + "feature_b", + "" + ) + + feat = normalize(feat) + + if feat: + + counter[feat] += 1 + + return [ + feat + for feat, _ + in counter.most_common(12) + ] + + +# ===================================================== +# Similar titles +# ===================================================== +def extract_titles( + results: pd.DataFrame +) -> List[str]: + + if not isinstance(results, pd.DataFrame): + return [] + + titles = [ + str(row.get("project_title", "")).strip() + for _, row in results.iterrows() + if row.get("project_title") + ] + + return clean_list(titles, limit=10) + + +# ===================================================== +# Architecture hints +# ===================================================== +def build_architecture_hints( + domains: List[str] +) -> List[str]: + + hints = [] + + if "artificial intelligence" in domains: + hints.extend([ + "AI inference pipeline", + "Model prediction workflow", + "Data preprocessing module" + ]) + + if "healthcare" in domains: + hints.extend([ + "Emergency handling workflow", + "Patient monitoring logic", + "Medical alert system" + ]) + + if "security" in domains: + hints.extend([ + "Threat detection pipeline", + "Behavior anomaly analysis", + "Risk monitoring engine" + ]) + + if "education" in domains: + hints.extend([ + "Adaptive learning workflow", + "Student performance analytics", + "Recommendation engine" + ]) + + return clean_list(hints, limit=10) + + +# ===================================================== +# Project Context +# ===================================================== +def build_project_context( + title: str, + description: str, + abstract: str = "", + features: List[str] = None +) -> Dict[str, Any]: + + features = features or [] + + logger.info("Building project context") + + full_text = ( + f"{title}. " + f"{abstract}. " + f"{description}" + ) + + # ========================================= + # Domains + # ========================================= + domains = detect_domains(full_text) + + main_domain = ( + domains[0] + if domains + else "general" + ) + + # ========================================= + # Auto features + # ========================================= + auto_features = extract_features( + full_text + ) + + user_features = clean_list( + features + auto_features, + MAX_FEATURES + ) + + # ========================================= + # Similarity search + # ========================================= + try: + + results = cached_similarity( + title, + description + ) + + except Exception as e: + + logger.warning( + f"Similarity failed: {e}" + ) + + results = None + + # ========================================= + # Empty fallback + # ========================================= + if ( + not isinstance(results, pd.DataFrame) + or len(results) == 0 + or "message" in results.columns + ): + + return { + "project_title": title, + "domain": main_domain, + "domains": domains, + "features": user_features, + "similar_titles": [], + "common_features": [], + "unique_features": user_features, + "architecture_hints": build_architecture_hints(domains), + "originality_score": 1.0, + "context_strength": 0.0 + } + + # ========================================= + # Rich context + # ========================================= + similar_titles = extract_titles(results) + + common_features = extract_common_features( + results + ) + + unique_features = [ + f + for f in user_features + if f not in common_features + ] + + originality = float( + results.get( + "originality_score", + pd.Series([1]) + ).mean() + ) + + hybrid_scores = results.get( + "hybrid_score", + pd.Series([0]) + ) + + context_strength = float( + hybrid_scores.mean() + ) + + return { + "project_title": title, + "domain": main_domain, + "domains": domains, + "features": user_features, + "similar_titles": similar_titles, + "common_features": common_features, + "unique_features": unique_features, + "architecture_hints": build_architecture_hints(domains), + "originality_score": round(originality, 4), + "context_strength": round(context_strength, 4) + } + + +# ===================================================== +# Domain Context +# ===================================================== +def build_domain_context( + domain: str +) -> Dict[str, Any]: + + extracted = extract_domain(domain) + + if extracted: + domain_clean = extracted + else: + logger.warning( + f"[DOMAIN WARNING] Unknown domain: {domain}" + ) + + domain_clean = normalize(domain) + + logger.info( + f"Building domain context: {domain_clean}" + ) + + try: + + results = cached_similarity( + domain_clean, + domain_clean + ) + + except Exception as e: + + logger.warning( + f"Domain similarity failed: {e}" + ) + + results = None + + if ( + not isinstance(results, pd.DataFrame) + or len(results) == 0 + or "message" in results.columns + ): + + return { + "domain": domain_clean, + "existing_titles": [], + "common_features": [], + "architecture_hints": build_architecture_hints([domain_clean]), + "context_strength": 0.0 + } + + hybrid_scores = results.get( + "hybrid_score", + pd.Series([0]) + ) + + return { + "domain": domain_clean, + "existing_titles": extract_titles(results), + "common_features": extract_common_features(results), + "architecture_hints": build_architecture_hints([domain_clean]), + "context_strength": round( + float(hybrid_scores.mean()), + 4 + ) + } \ No newline at end of file diff --git a/src/recommendation_engine/feature_generator.py b/src/recommendation_engine/feature_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..1177e5a2431bb9ff2bb7786fbbf528e150328e85 --- /dev/null +++ b/src/recommendation_engine/feature_generator.py @@ -0,0 +1,326 @@ +# src/recommendation_engine/feature_generator.py + +import logging +from typing import List, Dict, Any, Set + +from src.recommendation_engine.context_builder import build_project_context +from src.recommendation_engine.prompt_builder import build_feature_prompt +from src.recommendation_engine.llm_client import generate_text +from src.recommendation_engine.validator import validate_generated_list +from src.recommendation_engine.novelty_checker import is_feature_novel + +from src.similarity_model import compare_two_ideas + +from src.recommendation_engine.config import ( + DEFAULT_FEATURE_COUNT, + GENERATION_BATCH_SIZE +) + +logger = logging.getLogger(__name__) + +MAX_RETRIES = 5 + +# 🔥 stronger duplicate control +SIMILARITY_THRESHOLD_LOCAL = 0.82 + + +# ===================================================== +# Helpers +# ===================================================== +def normalize(text: str) -> str: + return " ".join(str(text).strip().lower().split()) + + +# ===================================================== +# Generic feature detector +# ===================================================== +GENERIC_PATTERNS = [ + "dashboard", + "login", + "signup", + "authentication", + "analytics module", + "ai module", + "admin panel", + "settings page", + "reports system", + "user management" +] + + +def is_generic_feature(text: str) -> bool: + + low = normalize(text) + + if len(low.split()) < 2: + return True + + for bad in GENERIC_PATTERNS: + if bad in low: + return True + + return False + + +# ===================================================== +# Feature cleaner +# ===================================================== +def clean_features(features: List[str]) -> List[str]: + + final = [] + + for f in features: + + clean = str(f).strip() + + if not clean: + continue + + words = clean.split() + + # 🔥 ideal feature size + if len(words) < 3 or len(words) > 10: + continue + + if is_generic_feature(clean): + continue + + final.append(clean) + + return final + + +# ===================================================== +# Semantic duplicate checker +# ===================================================== +def is_duplicate_local(feature: str, existing: List[str]) -> bool: + + for old in existing: + + score = compare_two_ideas(feature, old) + + if score >= SIMILARITY_THRESHOLD_LOCAL: + logger.info(f"[LOCAL DUPLICATE] {feature} ~ {old} ({score:.2f})") + return True + + return False + + +# ===================================================== +# Better contextual fallback +# ===================================================== +def fallback_features(title: str) -> List[str]: + + title = (title or "").lower() + + # ========================================= + # Healthcare + # ========================================= + if any(k in title for k in ["health", "hospital", "medical", "clinic"]): + + return [ + "Real-time patient monitoring", + "Emergency alert notification system", + "AI-assisted diagnosis support", + "Medical data visualization dashboard", + "Predictive patient risk analysis" + ] + + # ========================================= + # Education + # ========================================= + if any(k in title for k in ["education", "learning", "student", "school"]): + + return [ + "Adaptive learning recommendation engine", + "Student performance prediction system", + "Automated assignment evaluation", + "Gamified engagement tracking", + "Personalized study path generation" + ] + + # ========================================= + # Security + # ========================================= + if any(k in title for k in ["security", "cyber", "threat"]): + + return [ + "Real-time threat detection engine", + "Behavior anomaly monitoring", + "Automated attack alert system", + "Security event visualization", + "Risk prediction analytics" + ] + + # ========================================= + # Default + # ========================================= + return [ + "Real-time intelligent monitoring", + "Predictive analytics engine", + "Smart recommendation system", + "Automated decision support", + "Dynamic performance optimization" + ] + + +# ===================================================== +# Main Generator +# ===================================================== +def generate_features( + title: str, + description: str, + abstract: str = "", + features: List[str] = None, + previous_generated_features: List[str] = None, + top_k: int = DEFAULT_FEATURE_COUNT +) -> Dict[str, Any]: + + features = features or [] + previous_generated_features = previous_generated_features or [] + + top_k = max(1, min(top_k, 20)) + + logger.info(f"Starting feature generation | title={title}") + + # ========================================= + # Build Context + # ========================================= + context = build_project_context( + title=title, + description=description, + abstract=abstract, + features=features + ) + + final_features: List[str] = [] + final_norm_set: Set[str] = set() + + existing_features = context.get("features", []) + + existing_norm = set( + normalize(f) + for f in existing_features + ) + + previous_norm = set( + normalize(f) + for f in previous_generated_features + ) + + attempts = 0 + + # ========================================= + # Generation Loop + # ========================================= + while len(final_features) < top_k and attempts < MAX_RETRIES: + + attempts += 1 + + logger.info(f"Generation attempt #{attempts}") + + generation_count = max( + top_k * 4, + GENERATION_BATCH_SIZE + ) + + # ===================================== + # Build Prompt + # ===================================== + prompt = build_feature_prompt( + context=context, + count=generation_count, + previous_features=previous_generated_features + ) + + # ===================================== + # LLM Generation + # ===================================== + raw_text = generate_text( + prompt, + task="feature" + ) + + if not raw_text: + logger.warning("Empty feature response") + continue + + # ===================================== + # Validation + # ===================================== + generated = validate_generated_list( + text=raw_text, + top_k=generation_count + ) + + generated = clean_features(generated) + + logger.info(f"Generated {len(generated)} candidate features") + + # ===================================== + # Filtering + # ===================================== + for feat in generated: + + norm = normalize(feat) + + if not norm: + continue + + # exact duplicate + if ( + norm in final_norm_set + or norm in existing_norm + or norm in previous_norm + ): + continue + + # semantic novelty + if not is_feature_novel(feat, existing_features): + continue + + # local semantic duplicate + if is_duplicate_local(feat, final_features): + continue + + final_features.append(feat) + final_norm_set.add(norm) + + logger.info(f"[NEW FEATURE] {feat}") + + if len(final_features) >= top_k: + break + + # ========================================= + # Fallback + # ========================================= + if len(final_features) < top_k: + + logger.warning("Using fallback features") + + fallback = fallback_features(title) + + for feat in fallback: + + norm = normalize(feat) + + if ( + norm not in final_norm_set + and norm not in existing_norm + ): + + final_features.append(feat) + final_norm_set.add(norm) + + if len(final_features) >= top_k: + break + + logger.info(f"Final generated features: {final_features}") + + return { + "project_title": context.get("project_title", title), + "current_features": existing_features, + "recommended_features": final_features, + "originality_score": context.get("originality_score", 1.0), + "similar_projects": context.get("similar_titles", []) + } \ No newline at end of file diff --git a/src/recommendation_engine/full_project_generator.py b/src/recommendation_engine/full_project_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c2911c9ee707d98cfbf9663a2a53dc06370ee243 --- /dev/null +++ b/src/recommendation_engine/full_project_generator.py @@ -0,0 +1,227 @@ +from src.recommendation_engine.llm_client import generate_text +from src.recommendation_engine.prompt_builder import ( + build_full_project_prompt +) + +import re + + +# ===================================================== +# SECTION EXTRACTOR +# ===================================================== +def extract_section(text, section_name): + + text = text.strip() + + marker = section_name + ":" + + if marker not in text: + return "" + + start = text.find(marker) + len(marker) + + sections = [ + "CATEGORY:", + "ABSTRACT:", + "DESCRIPTION:", + "TECHNOLOGIES:", + "KEYWORDS:", + "PROBLEM_STATEMENT:", + "PROPOSED_SOLUTION:", + "OBJECTIVES:", + "AI_SUMMARY:", + "FUTURE_WORK:", + "METHODOLOGY:" + ] + + end = len(text) + + for s in sections: + + if s == marker: + continue + + pos = text.find(s, start) + + if pos != -1 and pos < end: + end = pos + + return text[start:end].strip() + + +# ===================================================== +# BULLET PARSER +# ===================================================== +def parse_bullets(text): + + lines = text.splitlines() + + final = [] + + for line in lines: + + line = line.strip() + + if not line: + continue + + line = re.sub( + r"^[-•*0-9.\)\s]+", + "", + line + ) + + if line: + final.append(line) + + return final + + +# ===================================================== +# FULL PROJECT GENERATOR +# ===================================================== +def generate_full_project( + title, + features, + description="", + abstract="" +): + + context = { + "project_title": title, + "features": features, + "description": description, + "abstract": abstract + } + + prompt = build_full_project_prompt(context) + + raw = generate_text( + prompt, + task="chat" + ) + + result = { + + # ============================================= + # BASIC + # ============================================= + "project_title": title, + + "category": + extract_section(raw, "CATEGORY"), + + "abstract": + extract_section(raw, "ABSTRACT"), + + "description": + extract_section(raw, "DESCRIPTION"), + + # ============================================= + # LISTS + # ============================================= + "technologies": + parse_bullets( + extract_section(raw, "TECHNOLOGIES") + ), + + "keywords": + parse_bullets( + extract_section(raw, "KEYWORDS") + ), + + "objectives": + parse_bullets( + extract_section(raw, "OBJECTIVES") + ), + + "future_work": + parse_bullets( + extract_section(raw, "FUTURE_WORK") + ), + + # ============================================= + # TEXT FIELDS + # ============================================= + "problem_statement": + extract_section( + raw, + "PROBLEM_STATEMENT" + ), + + "proposed_solution": + extract_section( + raw, + "PROPOSED_SOLUTION" + ), + + "methodology": + extract_section( + raw, + "METHODOLOGY" + ), + + "ai_summary": + extract_section( + raw, + "AI_SUMMARY" + ) + } + + # ================================================= + # FALLBACKS + # ================================================= + + if not result.get("category"): + result["category"] = "General AI System" + + if not result.get("keywords"): + result["keywords"] = [ + "Artificial Intelligence", + "Automation", + "Smart System" + ] + + if not result.get("problem_statement"): + result["problem_statement"] = ( + "Current traditional systems suffer from " + "limited automation, inefficiency, and " + "lack of intelligent decision-making." + ) + + if not result.get("proposed_solution"): + result["proposed_solution"] = ( + "The proposed system uses AI-driven " + "automation and intelligent analytics " + "to improve operational efficiency." + ) + + if not result.get("objectives"): + result["objectives"] = [ + "Improve automation efficiency", + "Enhance system accuracy", + "Reduce operational costs" + ] + + if not result.get("methodology"): + result["methodology"] = ( + "The system will be developed using " + "data collection, preprocessing, " + "AI model training, testing, and deployment." + ) + + if not result.get("future_work"): + result["future_work"] = [ + "Cloud integration", + "Mobile application support", + "Advanced AI optimization" + ] + + if not result.get("ai_summary"): + result["ai_summary"] = ( + f"{title} is an intelligent AI-powered " + f"graduation project designed to provide " + f"automation, monitoring, and predictive analysis." + ) + + return result \ No newline at end of file diff --git a/src/recommendation_engine/idea_generator.py b/src/recommendation_engine/idea_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..b97cf64830c0b187f86a72e27d328af7e3c99f75 --- /dev/null +++ b/src/recommendation_engine/idea_generator.py @@ -0,0 +1,243 @@ +# src/recommendation_engine/idea_generator.py + +import logging +from typing import Dict, Any, List, Set + +from src.recommendation_engine.context_builder import build_domain_context +from src.recommendation_engine.prompt_builder import build_idea_prompt +from src.recommendation_engine.llm_client import generate_text +from src.recommendation_engine.validator import validate_generated_list + +from src.similarity_model import compare_two_ideas, load_metadata +from src.recommendation_engine.config import DEFAULT_IDEA_COUNT + +logger = logging.getLogger(__name__) + +MAX_RETRIES = 5 +SIMILARITY_THRESHOLD_LOCAL = 0.60 + + +# ========================================= +# 🔥 LOAD DATASET TITLES (ONCE) +# ========================================= +DATA_TITLES = set() + +def normalize_idea(text: str) -> str: + return " ".join(text.lower().strip().split()) + + +def init_dataset_titles(): + global DATA_TITLES + + df = load_metadata() + + if df is not None and "project_title" in df.columns: + DATA_TITLES = set( + normalize_idea(t) + for t in df["project_title"].dropna() + ) + +init_dataset_titles() + + +# ========================================= +# HELPERS +# ========================================= +def clean_ideas(ideas: List[str]) -> List[str]: + return [i.strip() for i in ideas if 2 <= len(i.split()) <= 8] + + +def is_duplicate_local(idea: str, existing: Set[str]) -> bool: + for old in existing: + if compare_two_ideas(idea, old) >= SIMILARITY_THRESHOLD_LOCAL: + return True + return False + + +# ========================================= +# FALLBACK +# ========================================= +def fallback_by_domain(domain: str) -> List[str]: + + domain = (domain or "general").lower() + + fallback_map = { + "education": [ + "AI adaptive learning system", + "Student performance prediction platform", + "Gamified learning mobile application", + "Automated grading system", + "Virtual classroom engagement analyzer" + ], + "healthcare": [ + "AI disease prediction system", + "Smart patient monitoring system", + "Medical diagnosis assistant", + "IoT health tracking device", + "Hospital resource optimization system" + ], + "fintech": [ + "Fraud detection AI system", + "Smart expense tracking app", + "Blockchain payment system", + "Credit risk prediction model", + "AI investment advisor" + ] + } + + return fallback_map.get(domain, [ + "AI recommendation system", + "Smart automation platform", + "Data analytics dashboard", + "Intelligent decision support system", + "Predictive analytics engine" + ]) + + +# ========================================= +# MAIN GENERATOR +# ========================================= +def generate_ideas( + domain: str, + top_k: int = DEFAULT_IDEA_COUNT, + previous_generated_ideas: List[str] = None +) -> Dict[str, Any]: + + if previous_generated_ideas is None: + previous_generated_ideas = [] + + top_k = max(1, min(top_k, 20)) + domain = domain or "general" + + logger.info(f"Starting idea generation | domain={domain} | top_k={top_k}") + + context = build_domain_context(domain) + + final_ideas: List[str] = [] + final_set: Set[str] = set() + previous_norm = set(normalize_idea(i) for i in previous_generated_ideas) + + all_generated: List[str] = [] + attempts = 0 + + # ========================================= + # GENERATION LOOP + # ========================================= + while len(final_ideas) < top_k and attempts < MAX_RETRIES: + + attempts += 1 + logger.info(f"Attempt #{attempts}") + + generation_count = max(top_k * 3, 12) + + prompt = build_idea_prompt( + context=context, + count=generation_count, + previous_ideas=previous_generated_ideas + ) + + raw_text = generate_text(prompt, task="idea") + + if not raw_text: + logger.warning("Empty LLM response") + continue + + generated = validate_generated_list( + text=raw_text, + top_k=generation_count + ) + + generated = clean_ideas(generated) + + logger.info(f"Generated {len(generated)} ideas") + all_generated.extend(generated) + + # ========================================= + # FILTERING + # ========================================= + for idea in generated: + + normalized = normalize_idea(idea) + + if not normalized: + continue + + # ❌ dataset duplicate + # 🔥 dataset exact match + if normalized in DATA_TITLES: + logger.info(f"[SKIP DATASET EXACT] {idea}") + continue + + # 🔥 dataset semantic match + skip_dataset_similar = False + for data_title in list(DATA_TITLES)[:500]: # limit for speed + if compare_two_ideas(idea, data_title) >= 0.85: + logger.info(f"[SKIP DATASET SIMILAR] {idea}") + skip_dataset_similar = True + break + + if skip_dataset_similar: + continue + + # ❌ duplicate inside current batch + if normalized in final_set: + continue + + # ❌ duplicate from previous session + if normalized in previous_norm: + logger.info(f"[SKIP PREVIOUS] {idea}") + continue + + + # 🔥 semantic check with previous ideas + skip_similar_prev = False + for old in previous_generated_ideas: + if compare_two_ideas(idea, old) >= 0.85: + logger.info(f"[SKIP SIMILAR PREVIOUS] {idea}") + skip_similar_prev = True + break + + if skip_similar_prev: + continue + + # ❌ semantic duplicate + if is_duplicate_local(idea, final_set): + logger.info(f"[SKIP SIMILAR] {idea}") + continue + + # ✅ accept idea + logger.info(f"[NEW IDEA] {idea}") + + final_ideas.append(idea) + final_set.add(normalized) + + if len(final_ideas) >= top_k: + break + + # ========================================= + # FALLBACK + # ========================================= + if len(final_ideas) < top_k: + + logger.warning("Using fallback ideas") + + fallback = fallback_by_domain(domain) + + for f in fallback: + + normalized = normalize_idea(f) + + if normalized not in final_set: + final_ideas.append(f) + final_set.add(normalized) + + if len(final_ideas) >= top_k: + break + + logger.info(f"Final ideas: {final_ideas}") + + return { + "domain": domain, + "generated_ideas": all_generated, + "final_ideas": final_ideas + } \ No newline at end of file diff --git a/src/recommendation_engine/intent_classifier.py b/src/recommendation_engine/intent_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..37bc23a034e42d5fa147b4543663561b115f9269 --- /dev/null +++ b/src/recommendation_engine/intent_classifier.py @@ -0,0 +1,137 @@ +# src/recommendation_engine/intent_classifier.py + +import logging +from typing import Dict + +from src.recommendation_engine.llm_client import generate_text + +logger = logging.getLogger(__name__) + + +# ========================================= +# VALID INTENTS +# ========================================= +VALID_INTENTS = { + "idea", + "feature", + "full_project", + "chat" +} + + +# ========================================= +# SAFE NORMALIZER +# ========================================= +def normalize_intent(text: str) -> str: + + if not text: + return "chat" + + text = text.lower().strip().split()[0] + + if text in VALID_INTENTS: + return text + + return "chat" + + +# ========================================= +# LLM INTENT CLASSIFIER +# ========================================= +def classify_with_llm(user_input: str, state: Dict) -> str: + + prompt = f""" +You are an intent classifier for a graduation project assistant. + +Return ONLY ONE word from: +idea +feature +description +full_project +chat + +Context: +Project Title: {state.get("project_title") or "None"} +Has Features: {"yes" if state.get("features") else "no"} +Has Description: {"yes" if state.get("description") else "no"} + +User: +"{user_input}" + +Rules: +- Asking for ideas → idea +- Asking for another idea → idea +- Giving a project idea → feature +- Asking for features → feature +- Asking for description → description +- Asking for full project → full_project +- Otherwise → chat +""" + + try: + result = generate_text(prompt, task="intent") + return normalize_intent(result) + + except Exception as e: + logger.warning(f"[INTENT ERROR] {e}") + return "chat" + + +# ========================================= +# HYBRID DETECTION (FINAL 🔥) +# ========================================= +def detect_intent(text: str, state: dict = None) -> str: + + if state is None: + state = {} + + text_clean = text.lower().strip() + + has_project = bool(state.get("project_title")) + has_features = bool(state.get("features")) + + # ========================================= + # HARD RULES (NO FLOW FORCING ❌) + # ========================================= + + # explicit idea requests + if any(x in text_clean for x in [ + "idea", "project idea", "new idea", "another idea", "suggest" + ]): + return "idea" + + # explicit feature requests + if "feature" in text_clean: + return "feature" + + + # full project + if any(x in text_clean for x in [ + "full project", "complete", "all details" + ]): + return "full_project" + + # ========================================= + # USER PROVIDED IDEA (🔥 important) + # ========================================= + # If user writes sentence like a project idea + if not has_project and len(text_clean.split()) >= 3: + return "feature" + + # ========================================= + # LLM fallback + # ========================================= + intent = classify_with_llm(text, state) + + logger.info(f"[INTENT] {intent}") + + # ========================================= + # SAFETY FIXES + # ========================================= + if intent == "feature" and not has_project: + return "feature" # user likely gave idea + + if intent == "description" and not has_project: + return "idea" + + return intent \ No newline at end of file diff --git a/src/recommendation_engine/llm_client.py b/src/recommendation_engine/llm_client.py new file mode 100644 index 0000000000000000000000000000000000000000..e667899fd39022239a29c4768e0e9897df9b08ad --- /dev/null +++ b/src/recommendation_engine/llm_client.py @@ -0,0 +1,234 @@ +# src/recommendation_engine/llm_client.py + +import time +import logging +from typing import List + +from google import genai + +from src.recommendation_engine.config import ( + GEMINI_API_KEY, + MODEL_CANDIDATES, + IDEA_TEMPERATURE, + FEATURE_TEMPERATURE, + CHAT_TEMPERATURE, + INTENT_TEMPERATURE, + IDEA_MAX_TOKENS, + FEATURE_MAX_TOKENS, + CHAT_MAX_TOKENS, + INTENT_MAX_TOKENS, + TOP_P, + TOP_K, + MAX_RETRIES, + RETRY_DELAY_SECONDS, + ENABLE_LOGGING +) + +from src.recommendation_engine.validator import validate_generated_list + +# ========================================= +# Logging +# ========================================= +logger = logging.getLogger(__name__) + +if ENABLE_LOGGING: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" + ) + + +class LLMProviderError(Exception): + def __init__(self, message: str, status_code: int = 503): + super().__init__(message) + self.message = message + self.status_code = status_code + + +def classify_provider_error(error: Exception): + text = str(error).lower() + + if ( + "reported as leaked" in text + or "permission_denied" in text + or "api key" in text + or "403" in text + ): + return LLMProviderError( + "Gemini API key was rejected. Create a new key, update .env, and restart the server.", + status_code=503 + ) + + if ( + "resource_exhausted" in text + or "quota" in text + or "rate limit" in text + or "429" in text + ): + return LLMProviderError( + "Gemini quota or rate limit is exhausted. Try again later or use another API key/project.", + status_code=429 + ) + + return None + +# ========================================= +# Init Gemini +# ========================================= +client = genai.Client(api_key=GEMINI_API_KEY) + + +# ========================================= +# Extract Text (Robust) +# ========================================= +def extract_text(response) -> str: + + if not response: + return "" + + # direct + text = getattr(response, "text", None) + if text: + return text.strip() + + # fallback + try: + candidates = getattr(response, "candidates", []) + if candidates: + parts = candidates[0].content.parts + return " ".join( + p.text for p in parts if hasattr(p, "text") + ).strip() + except Exception: + pass + + return "" + + +# ========================================= +# Temperature Selector +# ========================================= +def get_temperature(task: str) -> float: + + return { + "idea": IDEA_TEMPERATURE, + "feature": FEATURE_TEMPERATURE, + "intent": INTENT_TEMPERATURE, + }.get(task, CHAT_TEMPERATURE) + + +# ========================================= +# Max Tokens Selector +# ========================================= +def get_max_tokens(task: str) -> int: + + return { + "idea": IDEA_MAX_TOKENS, + "feature": FEATURE_MAX_TOKENS, + "intent": INTENT_MAX_TOKENS, + }.get(task, CHAT_MAX_TOKENS) + + +# ========================================= +# Prompt Safety +# ========================================= +def safe_prompt(prompt: str, max_chars: int = 12000) -> str: + return prompt[-max_chars:] + + +# ========================================= +# Response Quality Check (🔥 NEW) +# ========================================= +def is_bad_response(text: str) -> bool: + + if not text: + return True + + text = text.strip() + + # ❌ only reject VERY bad outputs + if len(text) < 3: + return True + + # ❌ reject only pure assistant chatter + bad_phrases = [ + "as an ai", + "i can help you", + "let me know" + ] + + lower = text.lower() + + if all(p in lower for p in bad_phrases): + return True + + return False + + +# ========================================= +# Main Generate Function +# ========================================= +def generate_text( + prompt: str, + task: str = "chat" +) -> str: + + prompt = safe_prompt(prompt) + + temperature = get_temperature(task) + max_tokens = get_max_tokens(task) + + for model_name in MODEL_CANDIDATES: + + for attempt in range(MAX_RETRIES): + + try: + logger.info( + f"[LLM] model={model_name} | task={task} | attempt={attempt+1}" + ) + + response = client.models.generate_content( + model=model_name, + contents=prompt, + config={ + "temperature": temperature, + "top_p": TOP_P, + "top_k": TOP_K, + "max_output_tokens": max_tokens + } + ) + + text = extract_text(response) + + # 🔥 skip garbage responses + if is_bad_response(text): + logger.warning("[LLM] Weak response, using anyway") + return text # 🔥 DO NOT RETRY + + return text + + except Exception as e: + logger.warning(f"[LLM ERROR] {e}") + + provider_error = classify_provider_error(e) + if provider_error: + raise provider_error + + # exponential backoff + time.sleep(RETRY_DELAY_SECONDS * (attempt + 1)) + + logger.info(f"[LLM] switching model...") + + logger.error("All LLM models failed") + + return "" + + +# ========================================= +# Generate List (🔥 FIXED) +# ========================================= +def generate_list(prompt: str, task="chat") -> List[str]: + + text = generate_text(prompt, task=task) + + return validate_generated_list(text) diff --git a/src/recommendation_engine/llm_router.py b/src/recommendation_engine/llm_router.py new file mode 100644 index 0000000000000000000000000000000000000000..c06dc99935e838f5ce0f53bf46f2c550cc7d8eb4 --- /dev/null +++ b/src/recommendation_engine/llm_router.py @@ -0,0 +1,66 @@ +import re +from src.recommendation_engine.context_builder import extract_domain + + +def word_to_number(text: str): + mapping = { + "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, + "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10 + } + + text = text.lower() + + for word, num in mapping.items(): + if word in text: + return num + + nums = re.findall(r'\d+', text) + if nums: + return int(nums[0]) + + return None + + +def analyze_user_input(user_input: str, state: dict) -> dict: + + text = user_input.lower() + + # ============================= + # INTENT DETECTION (RULE BASED) + # ============================= + if any(w in text for w in ["idea", "project", "suggest", "recommend"]): + intent = "idea" + + elif "feature" in text: + intent = "feature" + + elif "describe" in text or "description" in text: + intent = "description" + + else: + intent = "chat" + + # ============================= + # DOMAIN DETECTION + # ============================= + domain = extract_domain(text) + + # ============================= + # NUMBER DETECTION (🔥 NEW) + # ============================= + number = word_to_number(text) + + # ============================= + # PROJECT TITLE DETECTION + # ============================= + project_title = None + + if intent == "chat" and len(text.split()) > 4: + project_title = user_input + + return { + "intent": intent, + "domain": domain, + "project_title": project_title, + "number": number + } \ No newline at end of file diff --git a/src/recommendation_engine/local_llm.py b/src/recommendation_engine/local_llm.py new file mode 100644 index 0000000000000000000000000000000000000000..4210e476ea7d957ef821acbe58abe86b51077057 --- /dev/null +++ b/src/recommendation_engine/local_llm.py @@ -0,0 +1,76 @@ + +import requests +import time + +OLLAMA_URL = "http://localhost:11434/api/generate" +MODEL_NAME = "mistral" + + +def generate_local(prompt: str, retries: int = 2) -> str: + """ + Stable + fast local LLM call (Ollama) + """ + + # 🔥 cut prompt (مهم جدًا للسرعة) + prompt = prompt[:1200] + + payload = { + "model": MODEL_NAME, + "prompt": prompt, + "stream": False, + "options": { + "temperature": 0.7, + "num_predict": 150, # يقلل الوقت + "top_p": 0.9 + } + } + + for attempt in range(retries): + try: + start_time = time.time() + + response = requests.post( + OLLAMA_URL, + json=payload, + timeout=60 + ) + + duration = round(time.time() - start_time, 2) + + # ❌ bad status + if response.status_code != 200: + print(f"❌ Local status {response.status_code}") + time.sleep(1) + continue + + # ❌ invalid json + try: + data = response.json() + except Exception: + print("❌ Invalid JSON from local") + time.sleep(1) + continue + + result = data.get("response", "").strip() + + # ❌ empty response + if not result or len(result) < 10: + print(f"⚠️ Empty/short response ({duration}s)") + time.sleep(1) + continue + + print(f"✅ Local OK ({duration}s)") + return result + + except requests.exceptions.Timeout: + print(f"⏱️ Timeout ({attempt+1}/{retries})") + time.sleep(1) + + except Exception as e: + print("❌ Local error:", e) + time.sleep(1) + + # ❌ failed completely + print("🚨 Local LLM failed → fallback") + return "" + diff --git a/src/recommendation_engine/memory_store.py b/src/recommendation_engine/memory_store.py new file mode 100644 index 0000000000000000000000000000000000000000..4175de35598edc42f1357e67c7fc76b08f86c1ea --- /dev/null +++ b/src/recommendation_engine/memory_store.py @@ -0,0 +1,199 @@ +# src/recommendation_engine/memory_store.py + +import time +import uuid +from copy import deepcopy + +MEMORY = {} + +TTL = 86400 # 24 hours +MAX_HISTORY = 20 +MAX_CHATS = 10 + + +# ===================================================== +# Default State +# ===================================================== +def default_state(): + return { + "project_title": "", + "ideas": [], + "features": [], + "description": "", + "abstract": "", + "technologies": [], + "originality_score": None, + "context_strength": None, + "problem_statement": "", + "proposed_solution": "", + "keywords": [], + "ai_summary": "", + "category": "" + } + + +# ===================================================== +# Create New Chat +# ===================================================== +def create_chat(user_id: str): + + user = MEMORY.setdefault(user_id, { + "chats": {}, + "active_chat_id": None, + "timestamp": time.time() + }) + + if len(user["chats"]) >= MAX_CHATS: + oldest = sorted( + user["chats"].items(), + key=lambda x: x[1]["last_updated"] + )[0][0] + del user["chats"][oldest] + + chat_id = str(uuid.uuid4()) + + user["chats"][chat_id] = { + "history": [], + "state": default_state(), + "last_updated": time.time() + } + + user["active_chat_id"] = chat_id + + return chat_id + + +# ===================================================== +# Get Active Chat +# ===================================================== +def get_user_memory(user_id: str): + + user = MEMORY.get(user_id) + + if not user: + create_chat(user_id) + user = MEMORY[user_id] + + # TTL reset + if time.time() - user["timestamp"] > TTL: + MEMORY[user_id] = {} + create_chat(user_id) + user = MEMORY[user_id] + + chat_id = user.get("active_chat_id") + + if not chat_id or chat_id not in user["chats"]: + chat_id = create_chat(user_id) + + return user["chats"][chat_id] + + +# ===================================================== +# Merge State (FIXED 🔥) +# ===================================================== +def merge_state(old: dict, new: dict): + + merged = deepcopy(old) + + for key, value in new.items(): + + if value is None: + continue + + # 🔥 overwrite critical fields + if key in ["project_title", "description"]: + merged[key] = value + continue + + # 🔥 lists → replace if empty OR extend uniquely + if isinstance(value, list): + + if not value: + merged[key] = [] + continue + + existing = merged.get(key, []) + + combined = [] + seen = set() + + for item in existing + value: + norm = str(item).lower().strip() + if norm and norm not in seen: + seen.add(norm) + combined.append(item) + + # limit growth + merged[key] = combined[:20] + + else: + merged[key] = value + + return merged + + +# ===================================================== +# Save Chat +# ===================================================== +def save_user_memory(user_id: str, data: dict): + + user = MEMORY.get(user_id) + + if not user: + create_chat(user_id) + user = MEMORY[user_id] + + chat_id = user["active_chat_id"] + chat = user["chats"][chat_id] + + history = data.get("history", []) + new_state = data.get("state", {}) + + history = history[-MAX_HISTORY:] + + chat["state"] = merge_state(chat["state"], new_state) + chat["history"] = history + chat["last_updated"] = time.time() + + user["timestamp"] = time.time() + + +# ===================================================== +# Get All Chats +# ===================================================== +def get_all_chats(user_id: str): + + user = MEMORY.get(user_id, {}) + chats = user.get("chats", {}) + + return [ + { + "chat_id": chat_id, + "title": chat["state"].get("project_title") or "New Chat", + "last_updated": chat["last_updated"] + } + for chat_id, chat in chats.items() + ] + + +# ===================================================== +# Switch Chat +# ===================================================== +def switch_chat(user_id: str, chat_id: str): + + user = MEMORY.get(user_id) + + if not user: + return + + if chat_id in user.get("chats", {}): + user["active_chat_id"] = chat_id + + +# ===================================================== +# Clear User +# ===================================================== +def clear_user_memory(user_id: str): + + if user_id in MEMORY: + del MEMORY[user_id] \ No newline at end of file diff --git a/src/recommendation_engine/novelty_checker.py b/src/recommendation_engine/novelty_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..bf3aed07ee7e41d2284a0bf793c02b7fd0adff92 --- /dev/null +++ b/src/recommendation_engine/novelty_checker.py @@ -0,0 +1,314 @@ +# src/recommendation_engine/novelty_checker.py + +import logging +import re + +from typing import List +from functools import lru_cache + +from src.similarity_model import ( + compare_two_ideas, + find_similar_projects +) + +from src.recommendation_engine.config import ( + IDEA_DUPLICATE_THRESHOLD, + FEATURE_DUPLICATE_THRESHOLD +) + +logger = logging.getLogger(__name__) + + +# ===================================================== +# Generic weak patterns +# ===================================================== +GENERIC_PATTERNS = [ + "dashboard", + "platform", + "system", + "application", + "website", + "ai module", + "analytics module", + "smart system", + "management system" +] + + +# ===================================================== +# Text normalization +# ===================================================== +def normalize(text: str) -> str: + + text = str(text).lower().strip() + + text = re.sub(r"[^a-z0-9\s]", " ", text) + + text = re.sub(r"\s+", " ", text).strip() + + return text + + +# ===================================================== +# Generic detector +# ===================================================== +def is_generic(text: str) -> bool: + + low = normalize(text) + + for pattern in GENERIC_PATTERNS: + + if pattern in low: + return True + + return False + + +# ===================================================== +# Token overlap similarity +# ===================================================== +def token_overlap_score(a: str, b: str) -> float: + + a_tokens = set(normalize(a).split()) + + b_tokens = set(normalize(b).split()) + + if not a_tokens or not b_tokens: + return 0.0 + + overlap = len(a_tokens & b_tokens) + + union = len(a_tokens | b_tokens) + + return overlap / union + + +# ===================================================== +# Feature Novelty +# ===================================================== +def is_feature_novel( + feature: str, + existing_features: List[str] +) -> bool: + + feature = normalize(feature) + + if not feature: + return False + + # ========================================= + # Reject generic weak features + # ========================================= + if is_generic(feature): + + logger.debug(f"[GENERIC FEATURE] {feature}") + + return False + + # ========================================= + # Exact duplicate + # ========================================= + existing_norm = [ + normalize(f) + for f in existing_features + ] + + if feature in existing_norm: + + logger.debug(f"[EXACT DUP] {feature}") + + return False + + # ========================================= + # Semantic comparison + # ========================================= + for old in existing_norm: + + if not old: + continue + + # embedding similarity + semantic_score = compare_two_ideas( + feature, + old + ) + + # token overlap + overlap_score = token_overlap_score( + feature, + old + ) + + # combined reasoning + final_score = max( + semantic_score, + overlap_score + ) + + logger.debug( + f"[COMPARE] {feature} ~ {old} " + f"(semantic={semantic_score:.2f}, " + f"overlap={overlap_score:.2f})" + ) + + # stricter threshold + if final_score >= ( + FEATURE_DUPLICATE_THRESHOLD + 0.08 + ): + + logger.debug( + f"[FEATURE DUPLICATE] " + f"{feature} ~ {old}" + ) + + return False + + return True + + +# ===================================================== +# Duplicate Feature Filter +# ===================================================== +def filter_duplicate_features( + generated_features: List[str], + existing_features: List[str] +) -> List[str]: + + final = [] + + seen = set() + + for feat in generated_features: + + clean = str(feat).strip() + + norm = normalize(clean) + + if not clean: + continue + + if norm in seen: + continue + + if not is_feature_novel( + norm, + existing_features + final + ): + continue + + seen.add(norm) + + final.append(clean) + + return final + + +# ===================================================== +# Cached DB originality check +# ===================================================== +@lru_cache(maxsize=256) +def _cached_db_check(idea: str) -> float: + + try: + + results = find_similar_projects( + title=idea, + description=idea, + top_k=3 + ) + + if ( + hasattr(results, "iloc") + and len(results) > 0 + ): + + scores = [] + + for _, row in results.iterrows(): + + score = float( + row.get("hybrid_score", 0) + ) + + scores.append(score) + + if scores: + + # strongest similarity + return max(scores) + + except Exception as e: + + logger.warning(f"[DB ERROR] {e}") + + # safe fallback + return 0.0 + + +# ===================================================== +# Idea Novelty +# ===================================================== +def is_idea_novel(idea_title: str) -> bool: + + idea_title = normalize(idea_title) + + if not idea_title: + return False + + if is_generic(idea_title): + + logger.info( + f"[GENERIC IDEA REJECTED] " + f"{idea_title}" + ) + + return False + + score = _cached_db_check(idea_title) + + logger.info(f"[DB CHECK] {idea_title}") + logger.info(f"[SIMILARITY SCORE] {score:.4f}") + + return score < IDEA_DUPLICATE_THRESHOLD + + +# ===================================================== +# Optional novelty score +# ===================================================== +def score_feature_novelty( + feature: str, + existing_features: List[str] +) -> float: + + feature = normalize(feature) + + if not existing_features: + return 1.0 + + scores = [] + + for old in existing_features: + + old_norm = normalize(old) + + semantic = compare_two_ideas( + feature, + old_norm + ) + + overlap = token_overlap_score( + feature, + old_norm + ) + + scores.append( + max(semantic, overlap) + ) + + if not scores: + return 1.0 + + return round( + 1.0 - max(scores), + 4 + ) \ No newline at end of file diff --git a/src/recommendation_engine/prompt_builder.py b/src/recommendation_engine/prompt_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..f160b384f2630a2afcad6a402a7c02d5c00f408b --- /dev/null +++ b/src/recommendation_engine/prompt_builder.py @@ -0,0 +1,352 @@ +# src/recommendation_engine/prompt_builder.py + +from typing import Dict, Any, List + + +# ===================================================== +# Helpers +# ===================================================== +def list_to_text(items: List[str], max_items: int = 10) -> str: + + if not items: + return "None" + + seen = set() + cleaned = [] + + for item in items[:max_items]: + + val = str(item).strip() + + if not val: + continue + + key = val.lower() + + if key not in seen: + seen.add(key) + cleaned.append(val) + + return ", ".join(cleaned) + + +# ===================================================== +# Feature Prompt +# ===================================================== +def build_feature_prompt( + context: Dict[str, Any], + count: int = 10, + previous_features: List[str] = None +) -> str: + + previous_features = previous_features or [] + + return f""" +You are a senior software architect and AI systems designer. + +TASK: +Generate {count} intelligent and realistic system features +for the following graduation project. + +PROJECT TITLE: +{context.get("project_title")} + +CURRENT FEATURES: +{list_to_text(context.get("features", []), 15)} + +COMMON OVERUSED FEATURES TO AVOID: +{list_to_text(context.get("common_features", []), 15)} + +PREVIOUSLY GENERATED FEATURES: +{list_to_text(previous_features, 20)} + +PROJECT CONTEXT: +- Originality Score: {context.get("originality_score", 1.0)} +- Context Strength: {context.get("context_strength", 0.0)} + +IMPORTANT REQUIREMENTS: +- Features must belong to ONE coherent system +- Features must solve REAL problems +- Prefer intelligent automation and AI logic +- Focus on implementable engineering features +- Avoid vague or generic functionality +- Avoid repeating concepts with different wording +- Avoid simple CRUD/dashboard/login features +- Generate diverse feature types + +FEATURE TYPES TO MIX: +- Core system features +- AI/smart features +- Automation features +- Analytics/monitoring features +- Reliability/safety features +- User experience improvements + +STRICT RULES: +- One feature per line +- No numbering +- No explanations +- No repeated concepts +- Each feature should be concise +- Each feature should sound like a real product capability +- Prefer 3–10 words per feature + +GOOD FEATURE EXAMPLES: +- Real-time gesture recognition +- Predictive patient risk analysis +- AI-assisted diagnosis support +- Emergency response prioritization +- Adaptive learning recommendation engine + +BAD FEATURE EXAMPLES: +- Smart dashboard +- AI module +- Login system +- Reports page +- User management + +OUTPUT: +""".strip() + + +# ===================================================== +# Idea Prompt +# ===================================================== +def build_idea_prompt( + context: Dict[str, Any], + count: int = 10, + previous_ideas: List[str] = None +) -> str: + + previous_ideas = previous_ideas or [] + + domain = context.get("domain", "general") + + return f""" +You are a senior AI innovation consultant. + +TASK: +Generate {count} unique and high-quality graduation project ideas. + +DOMAIN: +{domain} + +PREVIOUS IDEAS: +{list_to_text(previous_ideas, 20)} + +IMPORTANT REQUIREMENTS: +- Ideas must solve REAL problems +- Ideas must be practical and implementable +- Prefer AI, automation, or intelligent systems +- Avoid generic software projects +- Avoid repeated themes or workflows +- Each idea must represent a DIFFERENT concept +- Encourage domain diversity and creativity +- Prefer modern technologies and impactful use-cases + +STRICT RULES: +- No repeated concepts +- No semantic duplicates +- No slight rewording +- Avoid overused ideas like: + prediction systems, + recommendation systems, + management systems, + analytics dashboards + +FORMAT RULES: +- One idea per line +- No numbering +- No explanations +- Keep each idea concise +- Prefer 4–12 words + +GOOD IDEA EXAMPLES: +- Smart traffic congestion prediction using drones +- AI-powered emergency sign language translator +- Blockchain-secured medical image sharing +- Computer vision livestock health monitoring +- Adaptive learning assistant for dyslexic students + +BAD IDEA EXAMPLES: +- AI management system +- Smart dashboard platform +- Recommendation application +- Analytics website + +OUTPUT: +""".strip() + + +# ===================================================== +# Description Prompt +# ===================================================== +def build_description_prompt(context: Dict[str, Any]) -> str: + + return f""" +You are a senior technical writer and software architect. + +TASK: +Write a professional graduation project description. + +PROJECT TITLE: +{context.get("project_title")} + +FEATURES: +{list_to_text(context.get("features", []), 20)} + +REQUIREMENTS: +- Explain the real-world problem +- Explain the proposed solution +- Connect all features logically +- Explain system intelligence and AI usage +- Keep the description realistic and implementable +- Write clearly and professionally +- Avoid unnecessary marketing language + +STRUCTURE: +1. Problem +2. Solution +3. System capabilities +4. Expected impact + +OUTPUT: +Professional structured paragraph +""".strip() + + +# ===================================================== +# Chat Prompt +# ===================================================== +def build_chat_prompt(context: Dict[str, Any]) -> str: + + return f""" +You are a senior software architect and graduation project consultant. + +PROJECT TITLE: +{context.get("project_title", "None")} + +PROJECT DESCRIPTION: +{context.get("description", "None")} + +PROJECT FEATURES: +{list_to_text(context.get("features", []), 20)} + +TECHNOLOGIES: +{list_to_text(context.get("technologies", []), 20)} + +YOUR ROLE: +- Help improve the graduation project +- Suggest technical improvements +- Answer project-related questions +- Keep responses professional and practical +- Focus on software engineering and AI systems +- Avoid unrelated discussions + +RULES: +- Be concise +- Be realistic +- Be technically accurate +- Keep all answers related to the current project +""".strip() + + +# ===================================================== +# Full Project Prompt +# ===================================================== +def build_full_project_prompt(context): + + title = context.get("project_title", "") + + features = context.get("features", []) + + description = context.get("description", "") + + abstract = context.get("abstract", "") + + features_text = "\n".join( + f"- {f}" + for f in features + ) + + return f""" +You are a senior software architect and academic researcher. + +Generate a COMPLETE graduation project specification. + +==================================================== + +PROJECT TITLE: +{title} + +CURRENT ABSTRACT: +{abstract} + +CURRENT DESCRIPTION: +{description} + +FEATURES: +{features_text} + +==================================================== + +STRICT RULES: + +1. Return ALL sections +2. NEVER skip any section +3. NEVER leave any field empty +4. If information is missing, intelligently generate it +5. Use professional academic style +6. Technologies and lists MUST use bullet points +7. Use EXACT section names +8. No explanations outside sections +9. Every section must contain meaningful content +10. OBJECTIVES must contain at least 5 objectives +11. FUTURE_WORK must contain at least 4 items +12. KEYWORDS must contain at least 5 keywords +13. TECHNOLOGIES must contain at least 5 technologies +14. METHODOLOGY must be detailed and multi-step +15. AI_SUMMARY must never be empty + +==================================================== + +CATEGORY: +(short category) + +ABSTRACT: +(full academic abstract) + +DESCRIPTION: +(full detailed description) + +TECHNOLOGIES: +- item +- item +- item + +KEYWORDS: +- keyword +- keyword +- keyword + +PROBLEM_STATEMENT: +(real-world problem explanation) + +PROPOSED_SOLUTION: +(system solution explanation) + +OBJECTIVES: +- objective +- objective +- objective + +AI_SUMMARY: +(short AI-generated summary) + +FUTURE_WORK: +- future enhancement +- future enhancement + +METHODOLOGY: +(step-by-step implementation process) +""".strip() \ No newline at end of file diff --git a/src/recommendation_engine/response_formatter.py b/src/recommendation_engine/response_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..2a19919901d75db14a839b8774a311df346c4dc1 --- /dev/null +++ b/src/recommendation_engine/response_formatter.py @@ -0,0 +1,262 @@ +# src/recommendation_engine/response_formatter.py + +from typing import Dict, Any, List + + +# ===================================================== +# Helpers +# ===================================================== +def unique_items(items: List[str]) -> List[str]: + + seen = set() + + final = [] + + for item in items: + + key = str(item).strip().lower() + + if not key: + continue + + if key not in seen: + seen.add(key) + final.append(str(item).strip()) + + return final + + +def format_list( + items: List[str], + prefix: str = "- " +) -> str: + + items = unique_items(items) + + if not items: + return "None" + + return "\n".join( + f"{prefix}{item}" + for item in items + ) + + +def clean_text(text: str) -> str: + + if not text: + return "" + + text = str(text).strip() + + # remove duplicated assistant prefixes + bad_prefixes = [ + "assistant:", + "🤖 assistant:", + "bot:" + ] + + lower = text.lower() + + for prefix in bad_prefixes: + + if lower.startswith(prefix): + + text = text[len(prefix):].strip() + + return text + + +def divider() -> str: + return "━━━━━━━━━━━━━━━━━━━━━━" + + +# ===================================================== +# Dynamic Next Step +# ===================================================== +def next_step(state: Dict[str, Any]) -> str: + + # ========================================= + # No project yet + # ========================================= + if not state.get("project_title"): + + return ( + "👉 Ask for project ideas " + "or specify a domain." + ) + + # ========================================= + # No features yet + # ========================================= + if not state.get("features"): + + return ( + "👉 Next options:\n" + "1️⃣ Generate features\n" + "2️⃣ Generate another idea\n" + ) + + # ========================================= + # No description yet + # ========================================= + + # ========================================= + # Full project exists + # ========================================= + + + +# ===================================================== +# Full Project Overview +# ===================================================== +def format_full_project(state: Dict[str, Any]) -> str: + + title = state.get("project_title", "Not defined") + + features = unique_items( + state.get("features", []) + ) + + description = clean_text( + state.get("description", "Not generated") + ) + + technologies = state.get("technologies", []) + + originality = state.get( + "originality_score", + "Unknown" + ) + + context_strength = state.get( + "context_strength", + "Unknown" + ) + + return f""" +📦 Full Project Overview + +📌 Project Title: +{title} + +⚙️ Smart Features: +{format_list(features)} + +📄 Description: +{description} + +🧠 Suggested Technologies: +{", ".join(technologies) if technologies else "Not defined"} + +⭐ Originality Score: +{originality} + +📊 Similarity Context: +{context_strength} +""".strip() + + +# ===================================================== +# Main Formatter +# ===================================================== +def format_response( + intent: str, + raw_response: str, + state: Dict[str, Any] +) -> str: + + title = state.get("project_title", "") + + ideas = unique_items( + state.get("ideas", []) + ) + + features = unique_items( + state.get("features", []) + ) + + description = clean_text( + state.get("description", "") + ) + + # ================================================= + # IDEA RESPONSE + # ================================================= + if intent == "idea": + + if not ideas: + return "⚠️ No project ideas generated." + + return f""" +💡 Suggested Project Ideas + +{format_list(ideas)} + +{divider()} +{next_step(state)} +""".strip() + + # ================================================= + # FEATURE RESPONSE + # ================================================= + if intent == "feature": + + if not features: + return "⚠️ No features generated." + + return f""" +⚙️ Smart Features for: +{title or "Selected Project"} + +{format_list(features)} + +{divider()} +{next_step(state)} +""".strip() + + # ================================================= + # DESCRIPTION RESPONSE + # ================================================= + if intent == "description": + + return f""" +📄 Project Description + +📌 {title or "Your Project"} + +{description} + +{divider()} +{next_step(state)} +""".strip() + + # ================================================= + # FULL PROJECT + # ================================================= + if intent == "full_project": + + return f""" +{format_full_project(state)} + +{divider()} +👉 You can continue improving the project. +""".strip() + + # ================================================= + # CHAT RESPONSE + # ================================================= + cleaned = clean_text(raw_response) + + return f""" +💬 Project Assistant + +{cleaned} + +{divider()} +📌 Current Project: +{title or "Not defined"} + +{divider()} +{next_step(state)} +""".strip() \ No newline at end of file diff --git a/src/recommendation_engine/semantic_intent_classifier.py b/src/recommendation_engine/semantic_intent_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..338838824a1bf7e24c4ccfafb55b43beea7f882a --- /dev/null +++ b/src/recommendation_engine/semantic_intent_classifier.py @@ -0,0 +1,81 @@ +# src/recommendation_engine/semantic_intent_classifier.py + +from src.recommendation_engine.llm_client import generate_text + + +# ===================================================== +# Allowed intents +# ===================================================== +VALID_INTENTS = { + "idea", + "feature", + "full_project", + "chat" +} + + +# ===================================================== +# Semantic Intent Detection (🔥 upgraded) +# ===================================================== +def detect_intent_semantic(user_input: str, state: dict) -> str: + + prompt = f""" +You are an intent classifier for a graduation project assistant. + +Classify the user intent into ONE of these: +idea +feature +description +full_project +chat + +================ CONTEXT ================= +Current Project Title: +{state.get("project_title") or "None"} + +Has Features: +{"yes" if state.get("features") else "no"} + +Has Description: +{"yes" if state.get("description") else "no"} + +================ USER ================= +"{user_input}" + +================ RULES ================= +- Asking for project ideas → idea +- Asking for another/new idea → idea +- Providing a project idea → feature +- Asking for features → featuregenerate features +- Asking for full project / full details → full_project +- If unclear → chat + +IMPORTANT: +Return ONLY ONE WORD from the list. +""" + + result = generate_text(prompt, task="intent").lower().strip() + + # ========================================= + # strict cleanup (🔥 important) + # ========================================= + result = result.split()[0].strip() + + if result in VALID_INTENTS: + return result + + # ========================================= + # fallback logic (🔥 critical safety) + # ========================================= + text = user_input.lower() + + if any(w in text for w in ["idea", "project", "suggest"]): + return "idea" + + if any(w in text for w in ["feature"]): + return "feature" + + if any(w in text for w in ["full", "all details", "complete"]): + return "full_project" + + return "chat" \ No newline at end of file diff --git a/src/recommendation_engine/state_manager.py b/src/recommendation_engine/state_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..9c0ca62ac7a4ffe21e6414b27effafcc055a88b8 --- /dev/null +++ b/src/recommendation_engine/state_manager.py @@ -0,0 +1,183 @@ +from typing import Dict, Any, List + + +# ===================================================== +# HELPERS +# ===================================================== +def normalize(text: str) -> str: + return str(text).strip().lower() + + +def merge_lists(old: List[str], new: List[str]) -> List[str]: + + seen = set() + + result = [] + + for item in (old or []) + (new or []): + + if not item: + continue + + key = normalize(item) + + if key not in seen: + + seen.add(key) + + result.append(str(item).strip()) + + return result + + +# ===================================================== +# SMART UPDATE STATE +# ===================================================== +def update_state( + state: Dict[str, Any], + new_data: Dict[str, Any], + mode: str = "merge" +) -> Dict[str, Any]: + + for key, value in new_data.items(): + + if key in ["user_input", "intent"]: + continue + + if value is None or value == "": + continue + + # ========================================= + # ALWAYS REPLACE + # ========================================= + if key in [ + "ideas", + "project_title", + "description", + "abstract", + "category", + "problem_statement", + "proposed_solution", + "methodology", + "ai_summary" + ]: + + state[key] = value + continue + + # ========================================= + # FEATURES + # ========================================= + if key == "features": + + if mode == "replace": + state[key] = value + else: + state[key] = merge_lists( + state.get(key, []), + value + ) + + continue + + # ========================================= + # LIST FIELDS + # ========================================= + if isinstance(value, list): + + if mode == "replace": + + state[key] = value + + else: + + state[key] = merge_lists( + state.get(key, []), + value + ) + + continue + + # ========================================= + # STRING FIELDS + # ========================================= + if isinstance(value, str): + + clean_val = value.strip() + + old_val = str( + state.get(key, "") + ).strip() + + if ( + mode == "replace" + or clean_val not in old_val + ): + state[key] = ( + f"{old_val}\n{clean_val}" + ).strip() + + continue + + # ========================================= + # OTHER TYPES + # ========================================= + state[key] = value + + return state + + +# ===================================================== +# RESET HELPERS +# ===================================================== +def reset_for_new_idea( + state: Dict[str, Any] +) -> Dict[str, Any]: + + return { + + "domain": state.get("domain"), + + "ideas": [], + + "project_title": "", + + "features": [], + + "description": "", + + "abstract": "", + + "technologies": [], + + "keywords": [], + + "category": "", + + "problem_statement": "", + + "proposed_solution": "", + + "objectives": [], + + "future_work": [], + + "methodology": "", + + "ai_summary": "", + + "originality_score": None, + + "context_strength": None + } + + +def reset_features_only( + state: Dict[str, Any] +) -> Dict[str, Any]: + + state["features"] = [] + + state["description"] = "" + + return state \ No newline at end of file diff --git a/src/recommendation_engine/test.py b/src/recommendation_engine/test.py new file mode 100644 index 0000000000000000000000000000000000000000..73f7df70d2a976ef4a1ad27bf800f23f1a89f65b --- /dev/null +++ b/src/recommendation_engine/test.py @@ -0,0 +1,123 @@ + +### idea generator test + +# from src.recommendation_engine.idea_generator import generate_ideas + +# result = generate_ideas( +# domain="healthcare", +# top_k=5 +# ) + +# print("\nFINAL IDEAS:") +# for i, idea in enumerate(result["final_ideas"], 1): +# print(f"{i}. {idea}") + +###featuregeneratoretest + +# from src.recommendation_engine.feature_generator import generate_features + +# # ========================================= +# # TEST 1: First generation +# # ========================================= +# print("\n========== FIRST RUN ==========\n") + +# first = generate_features( +# title="Smart Hospital System", +# description=""" +# AI-based hospital system with chatbot, +# patient management, QR booking, +# and smart recommendations. +# """, +# features=["chatbot", "search", "qr booking"], +# top_k=2 +# ) + +# for i, f in enumerate(first["recommended_features"], 1): +# print(f"{i}. {f}") + + +# # ========================================= +# # TEST 2: Second generation (NO duplicates) +# # ========================================= +# print("\n========== SECOND RUN (NO DUPLICATES) ==========\n") + +# second = generate_features( +# title="Smart Hospital System", +# description=""" +# AI-based hospital system with chatbot, +# patient management, QR booking, +# and smart recommendations. +# """, +# features=["chatbot", "search", "qr booking"], +# previous_generated_features=first["recommended_features"], +# top_k=2 +# ) + +# for i, f in enumerate(second["recommended_features"], 1): +# print(f"{i}. {f}") + + +# # ========================================= +# # TEST 3: Check overlap manually +# # ========================================= +# print("\n========== DUPLICATE CHECK ==========\n") + +# duplicates = set(first["recommended_features"]) & set(second["recommended_features"]) + +# if duplicates: +# print(" DUPLICATES FOUND:") +# for d in duplicates: +# print("-", d) +# else: +# print(" NO DUPLICATES — SYSTEM WORKING PERFECT") + + +## chatbot generator test + + + +# from src.recommendation_engine.chatbot_engine import chatbot + +# user_id = "user_1" + +# print("\n=== IDEA ===") +# print(chatbot(user_id, "I want healthcare project idea")) + +# print("\n=== FEATURES ===") +# print(chatbot(user_id, "add smart features")) + +# print("\n=== DESCRIPTION ===") +# print(chatbot(user_id, "describe it")) + +# print("\n=== MEMORY TEST ===") +# print(chatbot(user_id, "improve it more")) + + + + +# run_test.py + +from src.recommendation_engine.chatbot_engine import chatbot + +USER_ID = "test_user" + + + +def interactive(): + print("\n===== INTERACTIVE MODE =====\n") + + while True: + user_input = input("YOU: ") + + if user_input.lower() in ["exit", "quit"]: + break + + response = chatbot(USER_ID, user_input) + + print("\nBOT:") + print(response) + print("\n----------------------\n") + + + +interactive() \ No newline at end of file diff --git a/src/recommendation_engine/validator.py b/src/recommendation_engine/validator.py new file mode 100644 index 0000000000000000000000000000000000000000..31127dbe4de0ffa32f398017f9b368d6e7c65555 --- /dev/null +++ b/src/recommendation_engine/validator.py @@ -0,0 +1,275 @@ +# src/recommendation_engine/validator.py + +import re +from typing import List + + +# ===================================================== +# Generic / weak patterns +# ===================================================== +GENERIC_PATTERNS = [ + "dashboard", + "login", + "signup", + "authentication", + "admin panel", + "analytics system", + "analytics platform", + "management system", + "tracking system", + "monitoring system", + "ai module", + "smart system", + "web platform", + "mobile app", + "website", + "reports page", + "user management" +] + + +BAD_STARTS = [ + "here are", + "below are", + "these are", + "the following", + "project ideas", + "features include" +] + + +LOW_VALUE_WORDS = [ + "system", + "platform", + "application", + "website", + "solution" +] + + +# ===================================================== +# CLEAN TEXT +# ===================================================== +def clean_text(text: str) -> str: + + if not text: + return "" + + text = str(text).strip() + + # remove numbering + text = re.sub(r"^\d+[\)\.\-\s]+", "", text) + + # remove bullets + text = re.sub(r"^[\-\*\•\→\▪\s]+", "", text) + + # remove markdown + text = text.replace("**", "") + + # remove quotes + text = text.replace('"', "").replace("'", "") + + # remove brackets + text = re.sub(r"\(.*?\)", "", text) + + # remove long explanations after colon + if ":" in text and len(text.split()) > 6: + text = text.split(":")[0] + + # remove assistant prefixes + text = re.sub(r"^(assistant|bot)\s*[:\-]\s*", "", text, flags=re.I) + + # remove trailing punctuation + text = re.sub(r"[.,\-:;]+$", "", text) + + # normalize spaces + text = re.sub(r"\s+", " ", text).strip() + + return text + + +# ===================================================== +# NORMALIZATION +# ===================================================== +def normalize_key(text: str) -> str: + + text = text.lower() + + text = re.sub(r"[^a-z0-9\s]", "", text) + + text = re.sub(r"\s+", " ", text).strip() + + return text + + +# ===================================================== +# Generic detector +# ===================================================== +def is_generic(text: str) -> bool: + + low = normalize_key(text) + + for pattern in GENERIC_PATTERNS: + if pattern in low: + return True + + return False + + +# ===================================================== +# Weak phrase detector +# ===================================================== +def is_low_quality(text: str) -> bool: + + low = normalize_key(text) + + words = low.split() + + # too short + if len(words) < 3: + return True + + # too long + if len(words) > 12: + return True + + # starts badly + if any(low.startswith(x) for x in BAD_STARTS): + return True + + # mostly weak words + weak_count = sum( + 1 for w in words + if w in LOW_VALUE_WORDS + ) + + if weak_count >= len(words) / 2: + return True + + return False + + +# ===================================================== +# VALIDATION RULES +# ===================================================== +def is_valid_item(text: str) -> bool: + + if not text: + return False + + # reject generic garbage + if is_generic(text): + return False + + # reject low-quality phrases + if is_low_quality(text): + return False + + return True + + +# ===================================================== +# FILTER ITEMS +# ===================================================== +def filter_items(items: List[str]) -> List[str]: + + final = [] + + seen = set() + + for item in items: + + text = clean_text(item) + + if not text: + continue + + if not is_valid_item(text): + continue + + key = normalize_key(text) + + # exact duplicate + if key in seen: + continue + + # semantic-ish duplicate + duplicate = False + + for old in seen: + + # overlap similarity + overlap = set(key.split()) & set(old.split()) + + if len(overlap) >= max(2, min(len(key.split()), len(old.split())) - 1): + duplicate = True + break + + if duplicate: + continue + + seen.add(key) + + final.append(text) + + return final + + +# ===================================================== +# SMART SPLITTER +# ===================================================== +def smart_split(text: str) -> List[str]: + + if not text: + return [] + + text = text.replace("\r", "\n") + + lines = [] + + for line in text.split("\n"): + + line = line.strip() + + if not line: + continue + + # split inline numbering + parts = re.split(r"\d+[\.\)]\s*", line) + + for p in parts: + + p = p.strip() + + if not p: + continue + + # split bullets inline + subparts = re.split(r"\s*[-•▪]\s*", p) + + for sp in subparts: + + sp = sp.strip() + + if sp: + lines.append(sp) + + return lines + + +# ===================================================== +# MAIN VALIDATOR +# ===================================================== +def validate_generated_list( + text: str, + top_k: int = 10 +) -> List[str]: + + if not text: + return [] + + raw_items = smart_split(text) + + cleaned = filter_items(raw_items) + + return cleaned[:top_k] \ No newline at end of file diff --git a/src/similarity_model/__init__.py b/src/similarity_model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5d351f21bf18fdbafade19b7c895c04168f337ff --- /dev/null +++ b/src/similarity_model/__init__.py @@ -0,0 +1,39 @@ +from .preprocessing import ( + normalize_text, + extract_features, + preprocess_dataset +) + +from .semantic_search import ( + load_model, + load_faiss_index, + load_metadata, + search_by_text, + search_by_project_id, + compare_two_ideas +) + +from .feature_similarity import ( + load_feature_model, + safe_feature_list, + compute_feature_similarity, + compare_projects, + compare_project_against_many +) + +from .hybrid_ranker import ( + compute_hybrid_score, + compute_originality, + compute_confidence, + risk_label, + rank_candidates +) + +from .embedding_engine import ( + train_embedding_engine, + ProjectEmbedder +) + +from .similarity_engine import ( + find_similar_projects +) \ No newline at end of file diff --git a/src/similarity_model/__pycache__/__init__.cpython-311.pyc b/src/similarity_model/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aaa93bb2202999695b9bf696ba310f2393162308 Binary files /dev/null and b/src/similarity_model/__pycache__/__init__.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/__init__.cpython-313.pyc b/src/similarity_model/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de67cb8f1ea60c39df879b905d56ae155e14ced1 Binary files /dev/null and b/src/similarity_model/__pycache__/__init__.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/embedding_engine.cpython-311.pyc b/src/similarity_model/__pycache__/embedding_engine.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bdc2a3aab4f7b22438cdcb60e0c43871786ce82a Binary files /dev/null and b/src/similarity_model/__pycache__/embedding_engine.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/embedding_engine.cpython-313.pyc b/src/similarity_model/__pycache__/embedding_engine.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77227e321764561366e6f09c86d875837beb6beb Binary files /dev/null and b/src/similarity_model/__pycache__/embedding_engine.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/evaluation.cpython-311.pyc b/src/similarity_model/__pycache__/evaluation.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f5478935165faee601cc998c88f9b3b2d085220 Binary files /dev/null and b/src/similarity_model/__pycache__/evaluation.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/feature_similarity.cpython-311.pyc b/src/similarity_model/__pycache__/feature_similarity.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce636714bb56468df9e545d3b5e3587f7a08eaa6 Binary files /dev/null and b/src/similarity_model/__pycache__/feature_similarity.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/feature_similarity.cpython-313.pyc b/src/similarity_model/__pycache__/feature_similarity.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..893716c8dced3155e7352bba43aa3f8adaba95f0 Binary files /dev/null and b/src/similarity_model/__pycache__/feature_similarity.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/hybrid_ranker.cpython-311.pyc b/src/similarity_model/__pycache__/hybrid_ranker.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35dc7a31586ba49da04de34faa3360a7ed1f46f3 Binary files /dev/null and b/src/similarity_model/__pycache__/hybrid_ranker.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/hybrid_ranker.cpython-313.pyc b/src/similarity_model/__pycache__/hybrid_ranker.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a24561e9d05adaf4a4e10d1b691f78d31b6b99ed Binary files /dev/null and b/src/similarity_model/__pycache__/hybrid_ranker.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/preprocessing.cpython-311.pyc b/src/similarity_model/__pycache__/preprocessing.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1f6b0f6ca38dc7ab63ef1beef501b99d3d257f3 Binary files /dev/null and b/src/similarity_model/__pycache__/preprocessing.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/preprocessing.cpython-313.pyc b/src/similarity_model/__pycache__/preprocessing.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1347dcf05025e19eeda99e64c8c5d66c85fa54d Binary files /dev/null and b/src/similarity_model/__pycache__/preprocessing.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/semantic_search.cpython-311.pyc b/src/similarity_model/__pycache__/semantic_search.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89def30aa40db0d1b74b5d06720faa94d0b14943 Binary files /dev/null and b/src/similarity_model/__pycache__/semantic_search.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/semantic_search.cpython-313.pyc b/src/similarity_model/__pycache__/semantic_search.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..094826a2c97ed2a428d9ecb3d9c2ee5b97236725 Binary files /dev/null and b/src/similarity_model/__pycache__/semantic_search.cpython-313.pyc differ diff --git a/src/similarity_model/__pycache__/similarity_engine.cpython-311.pyc b/src/similarity_model/__pycache__/similarity_engine.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a70b1d9021120a25708b831d064b12cebecc033 Binary files /dev/null and b/src/similarity_model/__pycache__/similarity_engine.cpython-311.pyc differ diff --git a/src/similarity_model/__pycache__/similarity_engine.cpython-313.pyc b/src/similarity_model/__pycache__/similarity_engine.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..56d5129bb5d34483282cc439381e96c1d0d0766f Binary files /dev/null and b/src/similarity_model/__pycache__/similarity_engine.cpython-313.pyc differ diff --git a/src/similarity_model/embedding_engine.py b/src/similarity_model/embedding_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..cf67a2066f7b81a946752795145ff84d3b410ea6 --- /dev/null +++ b/src/similarity_model/embedding_engine.py @@ -0,0 +1,290 @@ +# src/embedding_engine.py + +import re +import logging +from pathlib import Path +from typing import List + +import pandas as pd +import numpy as np +import faiss +from sentence_transformers import SentenceTransformer +from Data.database.sql_connector import ( + load_preprocessed_projects +) + +# ===================================================== +# Logging +# ===================================================== +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" +) +logger = logging.getLogger(__name__) + +# ===================================================== +# Config +# ===================================================== +DEFAULT_MODEL = "all-MiniLM-L6-v2" + +TEXT_COL = "clean_text" +TITLE_COL = "project_title" +TECH_COL = "technologies" + +MODEL_DIR = Path("models") +INDEX_PATH = MODEL_DIR / "faiss_index.bin" +META_PATH = MODEL_DIR / "metadata.parquet" + +TOP_K_DEFAULT = 10 +MIN_SCORE_THRESHOLD = 0.35 + +# ===================================================== +# Helpers +# ===================================================== +def normalize_text(text: str) -> str: + """ + Same cleaning logic used in preprocessing. + """ + if pd.isna(text): + return "" + + text = str(text).strip().lower() + + text = re.sub(r"http\S+|www\S+|\S+@\S+", " ", text) + text = re.sub(r"[^a-z0-9\s\+\#\./\-]", " ", text) + text = re.sub(r"\s+", " ", text).strip() + + return text + + +def tokenize(text: str) -> set: + """ + Simple tokenization for keyword boosting. + """ + text = normalize_text(text) + return set(text.split()) + + +# ===================================================== +# Core Engine +# ===================================================== +class ProjectEmbedder: + + def __init__(self, model_name: str = DEFAULT_MODEL): + logger.info(f"Loading embedding model: {model_name}") + + self.model = SentenceTransformer(model_name) + self.index = None + self.metadata = None + + # ------------------------------------------------- + # Embeddings + # ------------------------------------------------- + def generate_embeddings( + self, + texts: List[str], + batch_size: int = 64 + ) -> np.ndarray: + + logger.info(f"Generating embeddings for {len(texts)} projects...") + + vectors = self.model.encode( + texts, + batch_size=batch_size, + show_progress_bar=True, + convert_to_numpy=True, + normalize_embeddings=True + ) + + return vectors.astype("float32") + + # ------------------------------------------------- + # Build Index + # ------------------------------------------------- + def build_index(self, df: pd.DataFrame): + """ + Build FAISS cosine index. + """ + + self.metadata = df.copy() + + # preserve ids + self.metadata = self.metadata.reset_index(drop=True) + + # ensure needed columns exist + for col in [TITLE_COL, TEXT_COL]: + if col not in self.metadata.columns: + self.metadata[col] = "" + + if TECH_COL not in self.metadata.columns: + self.metadata[TECH_COL] = "" + + # weighted content: + # title repeated twice + rich_texts = ( + self.metadata[TITLE_COL].fillna("").astype(str) + " " + + self.metadata[TITLE_COL].fillna("").astype(str) + " " + + self.metadata[TEXT_COL].fillna("").astype(str) + ).tolist() + + embeddings = self.generate_embeddings(rich_texts) + + dim = embeddings.shape[1] + + base_index = faiss.IndexFlatIP(dim) + self.index = faiss.IndexIDMap(base_index) + + ids = np.arange(len(self.metadata)).astype("int64") + + self.index.add_with_ids(embeddings, ids) + + logger.info( + f"FAISS index built successfully with {self.index.ntotal} vectors." + ) + + # ------------------------------------------------- + # Save + # ------------------------------------------------- + def save_artifacts(self, folder: str = "models"): + + path = Path(folder) + path.mkdir(parents=True, exist_ok=True) + + faiss.write_index( + self.index, + str(path / "faiss_index.bin") + ) + + self.metadata.to_parquet( + path / "metadata.parquet", + index=False + ) + + logger.info(f"Artifacts saved to {folder}") + + # ------------------------------------------------- + # Load + # ------------------------------------------------- + def load_artifacts(self, folder: str = "models"): + + path = Path(folder) + + self.index = faiss.read_index( + str(path / "faiss_index.bin") + ) + + self.metadata = pd.read_parquet( + path / "metadata.parquet" + ) + + logger.info("Artifacts loaded successfully.") + + # ------------------------------------------------- + # Search + # ------------------------------------------------- + def search( + self, + query: str, + k: int = TOP_K_DEFAULT, + threshold: float = MIN_SCORE_THRESHOLD + ) -> pd.DataFrame: + + if self.index is None or self.metadata is None: + raise ValueError("Index or metadata not loaded.") + + # normalize query + query_clean = normalize_text(query) + + query_vec = self.model.encode( + [query_clean], + convert_to_numpy=True, + normalize_embeddings=True + ).astype("float32") + + scores, ids = self.index.search(query_vec, k) + + query_words = tokenize(query_clean) + + results = [] + + for idx, score in zip(ids[0], scores[0]): + + if idx == -1: + continue + + row = self.metadata.loc[idx] + + final_score = float(score) + + # keyword boost + title_words = tokenize(row[TITLE_COL]) + tech_words = tokenize(row[TECH_COL]) + + overlap = len(query_words & title_words) + overlap += len(query_words & tech_words) + + if overlap > 0: + final_score += 0.02 * overlap + + # cap score + final_score = min(final_score, 1.0) + + # threshold + if final_score < threshold: + continue + + results.append({ + "project_id": int(idx), + "title": row[TITLE_COL], + "technologies": row[TECH_COL], + "similarity_score": round(final_score, 4) + }) + + if not results: + return pd.DataFrame([{ + "message": "No similar projects found." + }]) + + return pd.DataFrame(results).sort_values( + by="similarity_score", + ascending=False + ).reset_index(drop=True) + +# ===================================================== +# Full Training Pipeline +# ===================================================== +def train_embedding_engine(): + + logger.info( + "Loading processed dataset from Azure SQL..." + ) + + df = load_preprocessed_projects() + + engine = ProjectEmbedder() + + engine.build_index(df) + + engine.save_artifacts() + + logger.info( + "Embedding engine completed successfully." + ) + + return engine + + +# ===================================================== +# Example Run +# ===================================================== +if __name__ == "__main__": + + engine = train_embedding_engine() + + query = "Build a mobile app for expense tracking using flutter and firebase" + + print(f"\nQuery: {query}\n") + + results = engine.search(query, k=5) + + print(results) \ No newline at end of file diff --git a/src/similarity_model/evaluation.py b/src/similarity_model/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..254eb566234d17ffb355f69090b1a47e2f56526d --- /dev/null +++ b/src/similarity_model/evaluation.py @@ -0,0 +1,214 @@ +# src/evaluation.py +# FINAL PROFESSIONAL EVALUATION VERSION +# Self Test + Real Query Test + Overall Judgment + +import logging +import pandas as pd + +from src.similarity_model import find_similar_projects +from src.similarity_model import load_metadata + + +# ===================================================== +# Logging +# ===================================================== +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" +) +logger = logging.getLogger(__name__) + +TOP_K = 5 +SELF_TEST_SAMPLES = 20 + +# ===================================================== +# SELF RETRIEVAL TEST +# ===================================================== +def run_self_test(): + + df = load_metadata() + + total = min(len(df), SELF_TEST_SAMPLES) + + success = 0 + + for i in range(total): + + row = df.loc[i] + + results = find_similar_projects( + title=row.get("project_title", ""), + abstract=row.get("abstract", ""), + description=row.get("description", ""), + features=row.get("features", []), + top_k=1 + ) + + if "project_id" in results.columns: + + pred = int(results.iloc[0]["project_id"]) + + if pred == i: + success += 1 + + score = success / total + + print("\n==============================") + print("SELF RETRIEVAL TEST") + print("==============================") + print(f"Projects Tested : {total}") + print(f"Top1 Accuracy : {score:.2%}") + print("==============================") + + return score + + +# ===================================================== +# REAL QUERY TEST +# ===================================================== +def run_real_queries(): + + queries = [ + + { + "title": + "AI Clinic Management System", + + "description": + """ + Smart clinic with booking, + chatbot, patient records, + doctor dashboard. + """ + }, + + { + "title": + "Smart Library Assistant", + + "description": + """ + Library app with chatbot, + recommendation system, + qr code borrowing. + """ + }, + + { + "title": + "Attendance Face Recognition", + + "description": + """ + Attendance system using + face recognition and reports. + """ + }, + + { + "title": + "E-commerce Recommendation Platform", + + "description": + """ + Online shopping website with + recommendation engine, + payments and dashboard. + """ + } + + ] + + print("\n==============================") + print("REAL QUERY TEST") + print("==============================") + + total_score = 0 + count = 0 + + for q in queries: + + results = find_similar_projects( + title=q["title"], + description=q["description"], + top_k=1 + ) + + if "hybrid_score" in results.columns: + + score = float( + results.iloc[0]["hybrid_score"] + ) + + risk = str( + results.iloc[0]["duplicate_risk"] + ) + + top_title = str( + results.iloc[0]["project_title"] + ) + + total_score += score + count += 1 + + print() + print("Query:", q["title"]) + print("Top Match:", top_title) + print("Score:", round(score, 4)) + print("Risk:", risk) + + avg = total_score / count if count else 0 + + print("\n==============================") + print(f"Average Query Score: {avg:.4f}") + print("==============================") + + return avg + + +# ===================================================== +# FINAL JUDGMENT +# ===================================================== +def final_status( + self_score, + query_score +): + + print("\n==============================") + print("FINAL MODEL STATUS") + print("==============================") + + final_score = ( + 0.60 * self_score + + 0.40 * query_score + ) + + if final_score >= 0.90: + print("EXCELLENT ✅") + + elif final_score >= 0.75: + print("VERY GOOD ✅") + + elif final_score >= 0.60: + print("GOOD ⚠️") + + else: + print("NEEDS IMPROVEMENT ❌") + + print("Overall Score:", round(final_score, 4)) + print("==============================\n") + + +# ===================================================== +# RUN +# ===================================================== +if __name__ == "__main__": + + self_score = run_self_test() + + query_score = run_real_queries() + + final_status( + self_score, + query_score + ) \ No newline at end of file diff --git a/src/similarity_model/feature_similarity.py b/src/similarity_model/feature_similarity.py new file mode 100644 index 0000000000000000000000000000000000000000..6a8bf8dd189dad592fde5ba47a8a05f9373e8667 --- /dev/null +++ b/src/similarity_model/feature_similarity.py @@ -0,0 +1,417 @@ +# src/feature_similarity.py + +import logging +import ast +from functools import lru_cache +from typing import List, Dict, Any + +import numpy as np +import pandas as pd + +from sentence_transformers import SentenceTransformer +from sklearn.metrics.pairwise import cosine_similarity +from scipy.optimize import linear_sum_assignment + +# ===================================================== +# Logging +# ===================================================== +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" +) + +logger = logging.getLogger(__name__) + +# ===================================================== +# Config +# ===================================================== +MODEL_NAME = "all-MiniLM-L6-v2" + +DEFAULT_THRESHOLD = 0.65 + +SIMILARITY_WEIGHT = 0.70 +COVERAGE_WEIGHT = 0.30 + +# ===================================================== +# Model Loader +# ===================================================== +@lru_cache(maxsize=1) +def load_feature_model(): + """ + Load feature embedding model once. + """ + logger.info(f"Loading feature model: {MODEL_NAME}") + return SentenceTransformer(MODEL_NAME) + + +# ===================================================== +# Helpers +# ===================================================== +def safe_feature_list(features): + """ + Convert any feature input into clean List[str] + Supports: + list, tuple, numpy array, string, NaN + """ + + import numpy as np + + # None + if features is None: + return [] + + # numpy nan scalar only + if isinstance(features, float): + if pd.isna(features): + return [] + + # numpy array + if isinstance(features, np.ndarray): + features = features.tolist() + + # tuple + if isinstance(features, tuple): + features = list(features) + + # string + if isinstance(features, str): + features = [features] + + # list + if isinstance(features, list): + + cleaned = [] + + for item in features: + val = str(item).strip().lower() + + if val and val != "nan": + cleaned.append(val) + + return list(dict.fromkeys(cleaned)) + + return [] + + +def remove_redundant_features(features): + + cleaned = [] + seen_words = [] + + for feat in features: + + feat_words = set(feat.split()) + + redundant = False + + for existing in seen_words: + + overlap = len( + feat_words & existing + ) / max(len(feat_words), 1) + + if overlap >= 0.90: + redundant = True + break + + if not redundant: + cleaned.append(feat) + seen_words.append(feat_words) + + return cleaned + + + +def empty_result( + unique_a=None, + unique_b=None +) -> Dict[str, Any]: + + return { + "score": 0.0, + "coverage": 0.0, + "shared_count": 0, + "matches": [], + "unique_a": unique_a or [], + "unique_b": unique_b or [] + } + + +def encode_features( + features: List[str], + model +) -> np.ndarray: + """ + Encode feature phrases into normalized vectors. + """ + + if not features: + return np.array([]) + + vectors = model.encode( + features, + convert_to_numpy=True, + normalize_embeddings=True + ) + + return vectors.astype("float32") + + +# ===================================================== +# Core Similarity Engine +# ===================================================== +def compute_feature_similarity( + features_a, + features_b, + model=None, + threshold: float = DEFAULT_THRESHOLD +) -> Dict[str, Any]: + """ + Compare two feature lists using: + + 1. Sentence embeddings + 2. Cosine similarity matrix + 3. Hungarian optimal matching + 4. Coverage-aware final score + """ + + if model is None: + model = load_feature_model() + + fa = remove_redundant_features( + safe_feature_list(features_a) + ) + + fb = remove_redundant_features( + safe_feature_list(features_b) + ) + + # empty cases + if not fa or not fb: + return empty_result( + unique_a=fa, + unique_b=fb + ) + + # ------------------------------------------------- + # Encode features + # ------------------------------------------------- + emb_a = encode_features(fa, model) + emb_b = encode_features(fb, model) + + # ------------------------------------------------- + # Similarity matrix + # ------------------------------------------------- + sim_matrix = cosine_similarity( + emb_a, + emb_b + ) + + # ------------------------------------------------- + # Hungarian Algorithm + # maximize similarity => minimize negative matrix + # ------------------------------------------------- + row_idx, col_idx = linear_sum_assignment( + -sim_matrix + ) + + matches = [] + + matched_a = set() + matched_b = set() + + for i, j in zip(row_idx, col_idx): + + sim = float(sim_matrix[i, j]) + + if sim >= threshold: + + matches.append({ + "feature_a": fa[i], + "feature_b": fb[j], + "score": round(sim, 3) + }) + + matched_a.add(i) + matched_b.add(j) + + # ------------------------------------------------- + # Final Metrics + # ------------------------------------------------- + shared_scores = [ + m["score"] for m in matches + ] + + mean_similarity = ( + float(np.mean(shared_scores)) + if shared_scores else 0.0 + ) + + max_len = max(len(fa), len(fb)) + + coverage = ( + len(matches) / max_len + if max_len > 0 else 0.0 + ) + + final_score = ( + (SIMILARITY_WEIGHT * mean_similarity) + + + (COVERAGE_WEIGHT * coverage) + ) + + final_score = min(final_score, 1.0) + + matched_text_a = " ".join( + [ + m["feature_a"] + for m in matches + ] + ).lower() + + matched_text_b = " ".join( + [ + m["feature_b"] + for m in matches + ] + ).lower() + + + def is_semantically_redundant( + feature, + matched_text + ): + words = set(feature.lower().split()) + + overlap = sum( + 1 for w in words + if w in matched_text + ) + + ratio = overlap / max(len(words), 1) + + return ratio >= 0.5 + + + unique_a = [ + fa[i] + for i in range(len(fa)) + if i not in matched_a + and not is_semantically_redundant( + fa[i], + matched_text_a + ) + ] + + unique_b = [ + fb[j] + for j in range(len(fb)) + if j not in matched_b + and not is_semantically_redundant( + fb[j], + matched_text_b + ) + ] + + return { + "score": round(final_score, 4), + "coverage": round(coverage, 4), + "shared_count": len(matches), + "matches": matches, + "unique_a": unique_a, + "unique_b": unique_b + } + + +# ===================================================== +# Compare Two Rows From DataFrame +# ===================================================== +def compare_projects( + df: pd.DataFrame, + idx1: int, + idx2: int, + model=None +) -> Dict[str, Any]: + """ + Compare two projects from dataset. + """ + + if model is None: + model = load_feature_model() + + f1 = df.loc[idx1, "features"] + f2 = df.loc[idx2, "features"] + + result = compute_feature_similarity( + f1, + f2, + model=model + ) + + result["project_a_id"] = int(idx1) + result["project_b_id"] = int(idx2) + + return result + + +# ===================================================== +# Compare One Against Many +# ===================================================== +def compare_project_against_many( + query_features, + candidate_feature_lists, + model=None, + threshold: float = DEFAULT_THRESHOLD +): + """ + Compare one project against many candidates. + """ + + if model is None: + model = load_feature_model() + + results = [] + + for idx, candidate in enumerate( + candidate_feature_lists + ): + + result = compute_feature_similarity( + query_features, + candidate, + model=model, + threshold=threshold + ) + + result["candidate_id"] = idx + + results.append(result) + + return results + + +# ===================================================== +# Example Run +# ===================================================== +if __name__ == "__main__": + + project_a = [ + "online reservation", + "ai chatbot", + "patient records", + "doctor dashboard" + ] + + project_b = [ + "appointment booking", + "chatbot assistant", + "medical records", + "analytics dashboard" + ] + + result = compute_feature_similarity( + project_a, + project_b + ) + + print(result) \ No newline at end of file diff --git a/src/similarity_model/hybrid_ranker.py b/src/similarity_model/hybrid_ranker.py new file mode 100644 index 0000000000000000000000000000000000000000..8c6f0648b623ec872d7650396b417eba4837a154 --- /dev/null +++ b/src/similarity_model/hybrid_ranker.py @@ -0,0 +1,370 @@ +# src/hybrid_ranker.py + +import logging +from typing import List, Dict, Any + +import pandas as pd + +from src.similarity_model import ( + compute_feature_similarity, + load_feature_model, + safe_feature_list +) + +# ===================================================== +# Logging +# ===================================================== +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" +) +logger = logging.getLogger(__name__) + +# ===================================================== +# Columns +# ===================================================== +TITLE_COL = "project_title" +FEATURE_COL = "features" + +# ===================================================== +# Weights / Thresholds +# ===================================================== +DEFAULT_SEMANTIC_WEIGHT = 0.65 +DEFAULT_FEATURE_WEIGHT = 0.35 + +HIGH_FEATURE_WEIGHT = 0.45 +LOW_FEATURE_WEIGHT = 0.20 + +BONUS_WEIGHT = 0.05 +MIN_HYBRID_SCORE = 0.35 + +# ===================================================== +# Helpers +# ===================================================== +def clamp(value: float) -> float: + """ + Keep score inside [0,1] + """ + return max(0.0, min(1.0, float(value))) + + +def get_dynamic_weights( + feature_count: int, + coverage: float +): + """ + Adaptive weights depending on + feature richness. + """ + + semantic_w = DEFAULT_SEMANTIC_WEIGHT + feature_w = DEFAULT_FEATURE_WEIGHT + + # many strong features + if feature_count >= 5 and coverage >= 0.60: + semantic_w = 0.30 + feature_w = HIGH_FEATURE_WEIGHT + + # weak features -> trust semantic more + elif feature_count <= 2: + semantic_w = 0.80 + feature_w = LOW_FEATURE_WEIGHT + + return semantic_w, feature_w + + +# ===================================================== +# Score Engines +# ===================================================== +def compute_hybrid_score( + semantic_score: float, + feature_score: float, + coverage: float, + feature_count: int +) -> float: + + semantic_score = clamp(semantic_score) + feature_score = clamp(feature_score) + coverage = clamp(coverage) + + # ========================================== + # Strong feature overlap case + # ========================================== + if coverage >= 0.90 and feature_score >= 0.65: + return round( + clamp( + 0.75 + + (0.15 * feature_score) + + (0.10 * semantic_score) + ), + 4 + ) + + # ========================================== + # Normal scoring + # ========================================== + score = ( + 0.25 * semantic_score + + 0.55 * feature_score + + 0.20 * coverage + ) + + return round(clamp(score), 4) + + +def compute_originality( + hybrid_score: float, + unique_query_features: int, + total_query_features: int +) -> float: + """ + Higher similarity => lower originality + More unique features => higher originality + """ + + hybrid_score = clamp(hybrid_score) + + inverse_similarity = 1.0 - hybrid_score + + uniqueness_ratio = ( + unique_query_features / total_query_features + if total_query_features > 0 else 0.0 + ) + + originality = 1 - hybrid_score + + return round(clamp(originality), 4) + + +def compute_confidence( + semantic_score: float, + feature_score: float, + coverage: float +) -> float: + """ + Trust score for ranking result. + """ + + confidence = ( + 0.40 * clamp(semantic_score) + + 0.40 * clamp(feature_score) + + 0.20 * clamp(coverage) + ) + + return round(clamp(confidence), 4) + + +def risk_label(score: float) -> str: + """ + Duplicate risk label. + """ + + if score >= 0.85: + return "Very High" + + if score >= 0.70: + return "High" + + if score >= 0.55: + return "Medium" + + if score >= 0.40: + return "Low" + + return "Very Low" + + +# ===================================================== +# Core Comparison +# ===================================================== +def compare_single_candidate( + query_row: Dict[str, Any], + candidate_row: Dict[str, Any], + semantic_score: float, + model=None +) -> Dict[str, Any]: + + if model is None: + model = load_feature_model() + + query_features = safe_feature_list( + query_row.get(FEATURE_COL, []) + ) + + candidate_features = safe_feature_list( + candidate_row.get(FEATURE_COL, []) + ) + + feature_result = compute_feature_similarity( + query_features, + candidate_features, + model=model + ) + + feature_score = feature_result["score"] + coverage = feature_result["coverage"] + + total_query_features = len(query_features) + unique_query_count = len( + feature_result["unique_a"] + ) + + hybrid_score = compute_hybrid_score( + semantic_score=semantic_score, + feature_score=feature_score, + coverage=coverage, + feature_count=total_query_features + ) + + originality_score = compute_originality( + hybrid_score=hybrid_score, + unique_query_features=unique_query_count, + total_query_features=total_query_features + ) + + confidence_score = compute_confidence( + semantic_score=semantic_score, + feature_score=feature_score, + coverage=coverage + ) + + return { + "project_title": + candidate_row.get(TITLE_COL, ""), + + "semantic_score": + round(float(semantic_score), 4), + + "feature_score": + feature_score, + + "coverage": + coverage, + + "hybrid_score": + hybrid_score, + + "originality_score": + originality_score, + + "confidence_score": + confidence_score, + + "duplicate_risk": + risk_label(hybrid_score), + + "shared_features_count": + feature_result["shared_count"], + + "matches": + feature_result["matches"], + + "unique_query_features": + feature_result["unique_a"], + + "unique_candidate_features": + feature_result["unique_b"] + } + + +# ===================================================== +# Rank Candidates +# ===================================================== +def rank_candidates( + query_row, + candidate_rows, + semantic_scores: List[float], + model=None, + min_score: float = MIN_HYBRID_SCORE +) -> pd.DataFrame: + """ + Re-rank semantic retrieval results + using hybrid logic. + """ + + if model is None: + model = load_feature_model() + + rows = [] + + for candidate_row, sem_score in zip( + candidate_rows, + semantic_scores + ): + + result = compare_single_candidate( + query_row=query_row, + candidate_row=candidate_row, + semantic_score=float(sem_score), + model=model + ) + + if result["hybrid_score"] >= min_score: + rows.append(result) + + if not rows: + return pd.DataFrame([{ + "message": "No strong similar projects found." + }]) + + results = pd.DataFrame(rows) + + results = results.sort_values( + by=[ + "hybrid_score", + "confidence_score", + "semantic_score" + ], + ascending=False + ).reset_index(drop=True) + + return results + + +# ===================================================== +# Example Run +# ===================================================== +if __name__ == "__main__": + + query_project = { + "project_title": "Clinic AI System", + "features": [ + "appointment booking", + "ai chatbot", + "patient records", + "doctor dashboard" + ] + } + + candidate_projects = [ + { + "project_title": + "Hospital Reservation Platform", + + "features": [ + "online booking", + "chatbot assistant", + "medical records", + "analytics dashboard" + ] + }, + { + "project_title": + "Smart Farming Monitor", + + "features": [ + "crop analysis", + "soil sensors" + ] + } + ] + + semantic_scores = [0.84, 0.33] + + ranked = rank_candidates( + query_row=query_project, + candidate_rows=candidate_projects, + semantic_scores=semantic_scores + ) + + print(ranked) \ No newline at end of file diff --git a/src/similarity_model/preprocessing.py b/src/similarity_model/preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..96d755636142d6d2c62218dcfb338ddb0f0c542b --- /dev/null +++ b/src/similarity_model/preprocessing.py @@ -0,0 +1,406 @@ +# src/preprocessing.py +# FINAL POLISHED VERSION +# Best Practical Feature Extraction for Graduation Project System + +import re +import logging +from pathlib import Path + +import pandas as pd +from keybert import KeyBERT +from sentence_transformers import SentenceTransformer + +# ===================================================== +# Logging +# ===================================================== +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" +) +logger = logging.getLogger(__name__) + +# ===================================================== +# Models +# ===================================================== +MODEL_NAME = "all-MiniLM-L6-v2" + +embed_model = SentenceTransformer( + MODEL_NAME +) + +kw_model = KeyBERT( + model=embed_model +) + +# ===================================================== +# Config +# ===================================================== +MIN_WORDS = 8 +MAX_WORDS = 4000 +MAX_FEATURES = 10 + +TECH_TERMS = { + "python", "java", "c++", "c#", "flutter", + "react", "node.js", "firebase", + "sql", "mysql", "mongodb", + "tensorflow", "pytorch", + "opencv", "arduino", + "chatbot", "cnn", "nlp", + "qr code", "api" +} + +BAD_WORDS = { + "government", + "facility", + "facilities", + "nationalization", + "content", + "keyword", + "keywords", + "presence", + "addition", + "appropriate", + "developed", + "system", + "application", + "project" +} + +# ===================================================== +# Helpers +# ===================================================== +def normalize_text(text): + """ + Clean raw text + """ + + if pd.isna(text): + return "" + + text = str(text).lower().strip() + + # remove urls/emails + text = re.sub( + r"http\S+|www\S+|\S+@\S+", + " ", + text + ) + + # keep useful chars + text = re.sub( + r"[^a-z0-9\+\#\./\- ]", + " ", + text + ) + + # remove spaces + text = re.sub( + r"\s+", + " ", + text + ) + + return text.strip() + + +def clean_phrase(text): + """ + Clean extracted phrase + """ + + text = normalize_text(text) + + # remove numbering + text = re.sub( + r"^\d+\-?\s*", + "", + text + ) + + # remove articles + text = re.sub( + r"^(a|an|the)\s+", + "", + text + ) + + return text.strip() + + +def is_valid_phrase(text): + """ + Validate phrase + """ + + if not text: + return False + + if len(text.split()) < 2: + return False + + if len(text.split()) > 4: + return False + + words = text.split() + + # reject if contains junk words + if any(word in BAD_WORDS for word in words): + return False + + return True + + +def detect_tech_terms(text): + """ + Detect important exact technical terms + """ + + found = [] + + for term in TECH_TERMS: + if term in text: + found.append(term) + + return found + + +# ===================================================== +# Main Feature Extraction +# ===================================================== +def extract_features(text): + """ + Final high-quality extractor + """ + + try: + + keywords = kw_model.extract_keywords( + text, + keyphrase_ngram_range=(1, 3), + stop_words="english", + top_n=12, + use_mmr=True, + diversity=0.5 + ) + + final = [] + seen = set() + + # ============================================= + # Keep KeyBERT original ranking + # ============================================= + for phrase, score in keywords: + + phrase = clean_phrase(phrase) + + if not is_valid_phrase(phrase): + continue + + if phrase not in seen: + seen.add(phrase) + final.append(phrase) + + # ============================================= + # Add missing exact tech terms + # ============================================= + for term in detect_tech_terms(text): + + if term not in seen: + seen.add(term) + final.append(term) + + return final[:MAX_FEATURES] + + except Exception as e: + + logger.warning( + f"Feature extraction failed: {e}" + ) + + return [] + + +# ===================================================== +# Main Pipeline +# ===================================================== +def preprocess_dataset(df): + """ + Full preprocessing pipeline + """ + + logger.info( + "Starting preprocessing..." + ) + + df = df.copy() + + # clean columns + df.columns = ( + df.columns + .str.strip() + .str.lower() + .str.replace( + r"\W+", + "_", + regex=True + ) + ) + + # ============================================= + # Column Mapping Fix + # ============================================= + column_mapping = { + "title": "project_title", + "ai_summary": "ai_summary", + "technologies": "technologies", + "keywords": "keywords", + "abstract": "abstract", + "description": "description", + "problem_statement": "problem_statement", + "proposed_solution": "proposed_solution", + "objectives": "objectives", + "category": "category" + } + + df = df.rename(columns=column_mapping) + + # ensure needed columns + for col in [ + "project_title", + "abstract", + "description" + ]: + + if col not in df.columns: + df[col] = "" + + df[col] = ( + df[col] + .fillna("") + .astype(str) + ) + + # ============================================= + # Smart weighted merge + # ============================================= + df["full_content"] = ( + df["project_title"] + ". " + + df["project_title"] + ". " + + df["abstract"] + ". " + + df["description"] + ) + + # normalize + df["clean_text"] = ( + df["full_content"] + .apply(normalize_text) + ) + + # remove duplicates + before = len(df) + + df = df.drop_duplicates( + subset=[ + "project_title", + "clean_text" + ] + ).copy() + + logger.info( + f"Removed duplicates: {before-len(df)}" + ) + + # word count filter + df["word_count"] = ( + df["clean_text"] + .str.split() + .str.len() + ) + + df = df[ + df["word_count"].between( + MIN_WORDS, + MAX_WORDS + ) + ].copy() + + df.reset_index( + drop=True, + inplace=True + ) + + # ============================================= + # Feature Extraction + # ============================================= + logger.info( + "Extracting features..." + ) + + df["features"] = ( + df["clean_text"] + .apply(extract_features) + ) + + # remove empty rows + df = df[ + df["features"] + .apply(len) > 0 + ].copy() + + df.reset_index( + drop=True, + inplace=True + ) + + logger.info( + f"Final rows: {len(df)}" + ) + + return df + + +# ===================================================== +# Save +# ===================================================== +def save_processed_data( + df, + output_dir="Data/processed" +): + + path = Path(output_dir) + + path.mkdir( + parents=True, + exist_ok=True + ) + + df.to_parquet( + path / "projects_clean.parquet", + index=False + ) + + df.to_csv( + path / "projects_clean.csv", + index=False + ) + + logger.info( + f"Saved to {path}" + ) + + +# ===================================================== +# Run +# ===================================================== +if __name__ == "__main__": + + file_path = "Data/raw/projects.xlsx" + + if file_path.endswith(".csv"): + raw_df = pd.read_csv(file_path) + else: + raw_df = pd.read_excel(file_path) + + clean_df = preprocess_dataset(raw_df) + + save_processed_data(clean_df) \ No newline at end of file diff --git a/src/similarity_model/semantic_search.py b/src/similarity_model/semantic_search.py new file mode 100644 index 0000000000000000000000000000000000000000..1b0a34b6bfe620009cf0e3f0ae55e62d96dfbecb --- /dev/null +++ b/src/similarity_model/semantic_search.py @@ -0,0 +1,321 @@ +# src/semantic_search.py + +import re +import ast +import logging +from pathlib import Path +from functools import lru_cache + +import numpy as np +import pandas as pd +import faiss +from sentence_transformers import SentenceTransformer + +from Data.database.sql_connector import ( + load_preprocessed_projects +) + +# ===================================================== +# Logging +# ===================================================== +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" +) +logger = logging.getLogger(__name__) + +# ===================================================== +# Config +# ===================================================== +DEFAULT_MODEL = "all-MiniLM-L6-v2" + +TITLE_COL = "project_title" +TECH_COL = "technologies" + +INDEX_PATH = "models/faiss_index.bin" +META_PATH = "models/metadata.parquet" +EMBED_PATH = "models/project_embeddings.npy" + +TOP_K = 10 +MIN_SCORE = 0.35 + +# ===================================================== +# Text Helpers +# ===================================================== +def normalize_text(text: str) -> str: + """ + Normalize user query to match preprocessing style. + """ + if pd.isna(text): + return "" + + text = str(text).strip().lower() + + text = re.sub(r"http\S+|www\S+|\S+@\S+", " ", text) + text = re.sub(r"[^a-z0-9\s\+\#\./\-]", " ", text) + text = re.sub(r"\s+", " ", text).strip() + + return text + + +def tokenize(text: str) -> set: + """ + Tokenize normalized text. + """ + return set(normalize_text(text).split()) + + +# ===================================================== +# Cached Loaders +# ===================================================== +@lru_cache(maxsize=1) +def load_model(): + logger.info(f"Loading model: {DEFAULT_MODEL}") + return SentenceTransformer(DEFAULT_MODEL) + + +@lru_cache(maxsize=1) +def load_faiss_index(): + if not Path(INDEX_PATH).exists(): + raise FileNotFoundError("FAISS index not found.") + + logger.info("Loading FAISS index...") + return faiss.read_index(INDEX_PATH) + + +@lru_cache(maxsize=1) +def load_metadata(): + + logger.info( + "Loading metadata from Azure SQL..." + ) + + df = load_preprocessed_projects() + + return df.reset_index(drop=True) + + +@lru_cache(maxsize=1) +def load_embeddings(): + if not Path(EMBED_PATH).exists(): + raise FileNotFoundError("Embeddings not found.") + + logger.info("Loading embeddings...") + return np.load(EMBED_PATH) + + +# ===================================================== +# Core Result Builder +# ===================================================== +def build_results( + df: pd.DataFrame, + ids, + scores, + query_text: str = "", + min_score: float = MIN_SCORE +) -> pd.DataFrame: + + rows = [] + + query_words = tokenize(query_text) + + for idx, score in zip(ids, scores): + + if idx == -1: + continue + + row = df.loc[idx] + + final_score = float(score) + + # --------------------------------------------- + # Keyword Boosting + # --------------------------------------------- + if query_words: + + title_words = tokenize(row[TITLE_COL]) + + tech_words = tokenize( + row.get(TECH_COL, "") + ) + + overlap = len(query_words & title_words) + overlap += len(query_words & tech_words) + + if overlap > 0: + final_score += 0.02 * overlap + + # cap score + final_score = min(final_score, 1.0) + + if final_score < min_score: + continue + + rows.append({ + "project_id": int(idx), + "project_title": row[TITLE_COL], + "technologies": row.get(TECH_COL, ""), + "score": round(final_score, 4) + }) + + if not rows: + return pd.DataFrame([{ + "message": "No similar projects found.", + "score": 0 + }]) + + return ( + pd.DataFrame(rows) + .sort_values("score", ascending=False) + .reset_index(drop=True) + ) + + +# ===================================================== +# Search by Free Text +# ===================================================== +def search_by_text( + query_text: str, + k: int = TOP_K, + min_score: float = MIN_SCORE +) -> pd.DataFrame: + + model = load_model() + index = load_faiss_index() + df = load_metadata() + + query_clean = normalize_text(query_text) + + query_vec = model.encode( + [query_clean], + convert_to_numpy=True, + normalize_embeddings=True + ).astype("float32") + + scores, ids = index.search(query_vec, k) + + return build_results( + df=df, + ids=ids[0], + scores=scores[0], + query_text=query_clean, + min_score=min_score + ) + + +# ===================================================== +# Search Similar to Existing Project +# ===================================================== +def search_by_project_id( + project_id: int, + k: int = TOP_K, + min_score: float = MIN_SCORE, + exclude_self: bool = True +) -> pd.DataFrame: + + df = load_metadata() + index = load_faiss_index() + embeddings = load_embeddings() + + if project_id < 0 or project_id >= len(df): + raise IndexError("Project ID out of range.") + + query_vec = embeddings[ + project_id + ].reshape(1, -1).astype("float32") + + extra_k = k + 1 if exclude_self else k + + scores, ids = index.search( + query_vec, + extra_k + ) + + rows = [] + + for idx, score in zip(ids[0], scores[0]): + + if idx == -1: + continue + + if exclude_self and idx == project_id: + continue + + final_score = min(float(score), 1.0) + + if final_score < min_score: + continue + + row = df.loc[idx] + + rows.append({ + "project_id": int(idx), + "project_title": row[TITLE_COL], + "technologies": row.get(TECH_COL, ""), + "score": round(final_score, 4) + }) + + if len(rows) == k: + break + + if not rows: + return pd.DataFrame([{ + "message": "No similar projects found.", + "score": 0 + }]) + + return pd.DataFrame(rows) + + +# ===================================================== +# Compare Two Ideas +# ===================================================== +def compare_two_ideas( + text_a: str, + text_b: str +) -> float: + + model = load_model() + + vecs = model.encode( + [ + normalize_text(text_a), + normalize_text(text_b) + ], + convert_to_numpy=True, + normalize_embeddings=True + ).astype("float32") + + score = float(np.dot(vecs[0], vecs[1])) + + return round(score, 4) + + +# ===================================================== +# Example Run +# ===================================================== +if __name__ == "__main__": + + print("\n=== Search by Text ===") + print( + search_by_text( + "mobile app for expense tracking using flutter", + k=5 + ) + ) + + print("\n=== Similar Projects ===") + print( + search_by_project_id( + project_id=0, + k=5 + ) + ) + + print("\n=== Compare Two Ideas ===") + print( + compare_two_ideas( + "AI Medical Chatbot", + "Healthcare Assistant with AI" + ) + ) \ No newline at end of file diff --git a/src/similarity_model/similarity_engine.py b/src/similarity_model/similarity_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..10e22642a13ee7a728480304dcc7d3d7b9c07294 --- /dev/null +++ b/src/similarity_model/similarity_engine.py @@ -0,0 +1,389 @@ +# src/similarity_engine.py +# FINAL VERSION +# Full Similarity Engine with Auto Feature Extraction + Hybrid Ranking + +import logging +from typing import Dict, Any, List, Optional + +import pandas as pd + +from src.similarity_model import ( + normalize_text, + extract_features, + load_model, + load_faiss_index, + load_metadata, + search_by_text, + load_feature_model, + safe_feature_list, + compute_feature_similarity, + compute_hybrid_score, + compute_originality, + compute_confidence, + risk_label +) + +# ===================================================== +# Logging +# ===================================================== +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s" +) +logger = logging.getLogger(__name__) + +# ===================================================== +# Config +# ===================================================== +TITLE_COL = "project_title" +FEATURE_COL = "features" + +DEFAULT_TOP_K = 5 +DEFAULT_SEARCH_POOL = 20 +DEFAULT_MIN_SEMANTIC_SCORE = 0.30 +MAX_QUERY_FEATURES = 12 + +# ===================================================== +# Query Builders +# ===================================================== +def build_raw_text( + title: str = "", + abstract: str = "", + description: str = "" +) -> str: + """ + Merge available text fields. + """ + parts = [ + str(title).strip(), + str(title).strip(), # title weighted + str(abstract).strip(), + str(description).strip() + ] + + return ". ".join( + [p for p in parts if p] + ).strip() + + +def merge_features( + title: str = "", + abstract: str = "", + description: str = "", + features: Optional[List[str]] = None +) -> List[str]: + """ + Use same extractor from preprocessing.py + + merge manual features + + remove duplicates + """ + + if features is None: + features = [] + + manual_features = safe_feature_list(features) + + raw_text = build_raw_text( + title=title, + abstract=abstract, + description=description + ) + + auto_features = extract_features( + normalize_text(raw_text) + ) + + final = [] + seen = set() + + # manual first + for feat in manual_features + auto_features: + + feat = str(feat).strip().lower() + + if feat and feat not in seen: + seen.add(feat) + final.append(feat) + + return final[:MAX_QUERY_FEATURES] + + +def build_query_project( + title: str, + abstract: str, + description: str, + features: List[str] +) -> Dict[str, Any]: + + return { + TITLE_COL: str(title).strip(), + "abstract": str(abstract).strip(), + "description": str(description).strip(), + FEATURE_COL: features + } + + +def build_query_text( + title: str, + abstract: str, + description: str, + features: List[str] +) -> str: + """ + Build semantic query text. + """ + + raw_text = build_raw_text( + title=title, + abstract=abstract, + description=description + ) + + feature_text = " ".join(features) + + text = f"{raw_text}. {feature_text}" + + return normalize_text(text) + + +# ===================================================== +# Compare Candidate +# ===================================================== +def compare_candidate( + query_project: Dict[str, Any], + candidate_row, + semantic_score: float, + feature_model +) -> Dict[str, Any]: + + feature_result = compute_feature_similarity( + query_project[FEATURE_COL], + candidate_row[FEATURE_COL], + model=feature_model + ) + + feature_score = feature_result["score"] + coverage = feature_result["coverage"] + + query_feature_count = len( + query_project[FEATURE_COL] + ) + + hybrid_score = compute_hybrid_score( + semantic_score=semantic_score, + feature_score=feature_score, + coverage=coverage, + feature_count=query_feature_count + ) + + originality_score = compute_originality( + hybrid_score=hybrid_score, + unique_query_features=len( + feature_result["unique_a"] + ), + total_query_features=query_feature_count + ) + + confidence_score = compute_confidence( + semantic_score=semantic_score, + feature_score=feature_score, + coverage=coverage + ) + + return { + "project_id": int(candidate_row.name), + "project_title": candidate_row[TITLE_COL], + + "semantic_score": + round(float(semantic_score), 4), + + "feature_score": + feature_score, + + "coverage": + coverage, + + "hybrid_score": + hybrid_score, + + "originality_score": + originality_score, + + "confidence_score": + confidence_score, + + "duplicate_risk": + risk_label(hybrid_score), + + "shared_features_count": + feature_result["shared_count"], + + "matched_features": + feature_result["matches"], + + "unique_query_features": + feature_result["unique_a"], + + "unique_candidate_features": + feature_result["unique_b"] + } + + +# ===================================================== +# Main Engine +# ===================================================== +def find_similar_projects( + title: str = "", + abstract: str = "", + description: str = "", + features: Optional[List[str]] = None, + top_k: int = DEFAULT_TOP_K, + search_pool: int = DEFAULT_SEARCH_POOL +) -> pd.DataFrame: + """ + Final Smart Pipeline + + Supports: + - title only + - title + description + - title + abstract + - title + abstract + description + - optional features + """ + + logger.info( + "Loading models and artifacts..." + ) + + load_model() + load_faiss_index() + + feature_model = load_feature_model() + df = load_metadata() + + # ============================================= + # Query preprocessing + # ============================================= + logger.info( + "Preparing query..." + ) + + final_features = merge_features( + title=title, + abstract=abstract, + description=description, + features=features + ) + + query_project = build_query_project( + title=title, + abstract=abstract, + description=description, + features=final_features + ) + + query_text = build_query_text( + title=title, + abstract=abstract, + description=description, + features=final_features + ) + + # ============================================= + # Stage 1 Semantic Retrieval + # ============================================= + logger.info( + "Running semantic retrieval..." + ) + + semantic_results = search_by_text( + query_text=query_text, + k=search_pool, + min_score=DEFAULT_MIN_SEMANTIC_SCORE + ) + + if "message" in semantic_results.columns: + return semantic_results + + # ============================================= + # Stage 2 Hybrid Ranking + # ============================================= + logger.info( + "Running hybrid ranking..." + ) + + rows = [] + + for _, row in semantic_results.iterrows(): + + candidate_id = int( + row["project_id"] + ) + + semantic_score = float( + row["score"] + ) + + candidate_row = df.loc[ + candidate_id + ] + + result = compare_candidate( + query_project=query_project, + candidate_row=candidate_row, + semantic_score=semantic_score, + feature_model=feature_model + ) + + rows.append(result) + + if not rows: + return pd.DataFrame([{ + "message": + "No strong similar projects found." + }]) + + final_df = pd.DataFrame(rows) + + final_df = final_df.sort_values( + by=[ + "hybrid_score", + "confidence_score", + "semantic_score" + ], + ascending=False + ).head(top_k).reset_index(drop=True) + + # add debug info + final_df["query_features_used"] = [ + final_features + ] * len(final_df) + + final_df["query_clean_text"] = [ + query_text + ] * len(final_df) + + return final_df + + +# ===================================================== +# Example Run +# ===================================================== +if __name__ == "__main__": + + results = find_similar_projects( + title="Smart Library", + abstract=""" + AI based digital library for students. + """, + description=""" + Includes chatbot, + recommendation system, + qr code scanner, + mobile application. + """, + features=["library"], + top_k=5 + ) + + print(results) \ No newline at end of file