import pandas as pd import openai from sentence_transformers import SentenceTransformer, util import numpy as np import torch import json import os # Load the Excel file and clean column names excel_file_path = 'cleaned_file.xlsx' df = pd.read_excel(excel_file_path) df.columns = [col.strip().lower() for col in df.columns] # Normalize column names to lowercase for consistency # Check and add missing columns if necessary if 'comment' not in df.columns: df['comment'] = "" if 'module' not in df.columns: df['module'] = "" # Extract data from DataFrame requirements = df['requirement'].tolist() responses = df['response'].tolist() comments = df['comment'].tolist() modules = df['module'].tolist() # Load prompts from the JSON file with open("corpus.json", "r") as file: prompts = json.load(file) PROMPT_SYSTEM_CONTENT = prompts["system_content"] # Function to generate a response using OpenAI API def generate_response(requirement, similar_texts, similar_responses, similar_comments, similar_products): product_description = similar_products # Initialize OpenAI client client = openai.OpenAI( api_key="2600a34cdd7a4cdf931c1e41955a122d", # Replace with your actual API key or load from a secure source base_url="https://api.aimlapi.com" ) # Construct the system content system_content = PROMPT_SYSTEM_CONTENT system_content += f"Product: {product_description}\nRequirement: {requirement}\n\n" # Add similar texts, responses, and comments to the system content for i, (text, response, comment, product) in enumerate(zip(similar_texts, similar_responses, similar_comments, similar_products)): system_content += ( f"Similar Requirement {i + 1}:\n" f"- Text: {text}\n" f"- Response: {response}\n" f"- Comment: {comment}\n" f"- Product: {product}\n\n" ) # Define the user prompt prompt = f"Generate the response and a concise comment with detailed functionality for {requirement}.The generation should be strictly of the type :Response \n Comment\n." # Generate a response using the OpenAI API chat_completion = client.chat.completions.create( model="meta-llama/Meta-Llama-3-8B-Instruct-Turbo", messages=[ {"role": "system", "content": system_content}, {"role": "user", "content": prompt} ], temperature=0.7, max_tokens=256, ) # Extract and return the response response = chat_completion.choices[0].message.content return response # Function to find the most similar texts using Sentence Transformers def find_most_similar(query, embeddings, texts, model): # Encode the query to get its embedding query_embedding = model.encode(query, convert_to_tensor=True).cpu() # Calculate cosine similarities between the query and stored embeddings similarities = util.pytorch_cos_sim(query_embedding, embeddings) # Get indices of the top 3 most similar texts most_similar_idx = similarities.argsort(descending=True)[0][:3].cpu().numpy() # Retrieve the most similar texts based on indices return [texts[idx] for idx in most_similar_idx], most_similar_idx