RRFP-HELPER-23 / utils.py
DXOR's picture
Update utils.py
852b165 verified
Raw History Blame Contribute Delete
3.24 kB
import pandas as pd
import openai
from sentence_transformers import SentenceTransformer, util
import numpy as np
import torch
import json
import os
# Load the Excel file and clean column names
excel_file_path = 'cleaned_file.xlsx'
df = pd.read_excel(excel_file_path)
df.columns = [col.strip().lower() for col in df.columns] # Normalize column names to lowercase for consistency
# Check and add missing columns if necessary
if 'comment' not in df.columns:
df['comment'] = ""
if 'module' not in df.columns:
df['module'] = ""
# Extract data from DataFrame
requirements = df['requirement'].tolist()
responses = df['response'].tolist()
comments = df['comment'].tolist()
modules = df['module'].tolist()
# Load prompts from the JSON file
with open("corpus.json", "r") as file:
prompts = json.load(file)
PROMPT_SYSTEM_CONTENT = prompts["system_content"]
# Function to generate a response using OpenAI API
def generate_response(requirement, similar_texts, similar_responses, similar_comments, similar_products):
product_description = similar_products
# Initialize OpenAI client
client = openai.OpenAI(
api_key="2600a34cdd7a4cdf931c1e41955a122d", # Replace with your actual API key or load from a secure source
base_url="https://api.aimlapi.com"
)
# Construct the system content
system_content = PROMPT_SYSTEM_CONTENT
system_content += f"Product: {product_description}\nRequirement: {requirement}\n\n"
# Add similar texts, responses, and comments to the system content
for i, (text, response, comment, product) in enumerate(zip(similar_texts, similar_responses, similar_comments, similar_products)):
system_content += (
f"Similar Requirement {i + 1}:\n"
f"- Text: {text}\n"
f"- Response: {response}\n"
f"- Comment: {comment}\n"
f"- Product: {product}\n\n"
)
# Define the user prompt
prompt = f"Generate the response and a concise comment with detailed functionality for {requirement}.The generation should be strictly of the type :Response \n Comment\n."
# Generate a response using the OpenAI API
chat_completion = client.chat.completions.create(
model="meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
messages=[
{"role": "system", "content": system_content},
{"role": "user", "content": prompt}
],
temperature=0.7,
max_tokens=256,
)
# Extract and return the response
response = chat_completion.choices[0].message.content
return response
# Function to find the most similar texts using Sentence Transformers
def find_most_similar(query, embeddings, texts, model):
# Encode the query to get its embedding
query_embedding = model.encode(query, convert_to_tensor=True).cpu()
# Calculate cosine similarities between the query and stored embeddings
similarities = util.pytorch_cos_sim(query_embedding, embeddings)
# Get indices of the top 3 most similar texts
most_similar_idx = similarities.argsort(descending=True)[0][:3].cpu().numpy()
# Retrieve the most similar texts based on indices
return [texts[idx] for idx in most_similar_idx], most_similar_idx