vmalapaka's picture
Update utils.py
92a9a04 verified
Raw History Blame Contribute Delete
3.17 kB
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Pinecone
from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
from pinecone import Pinecone as PineconeClient
from PyPDF2 import PdfReader
from langchain.document_loaders.sitemap import SitemapLoader
from langchain_openai import OpenAI
from langchain.chains.question_answering import load_qa_chain
from langchain_community.callbacks import get_openai_callback
import joblib
import pandas as pd
from sklearn.model_selection import train_test_split
def read_pdf_data(pdf):
pdf_read = PdfReader(pdf)
text = ""
for page in pdf_read.pages:
text +=page.extract_data()
return text
def split_data(docs):
text_splitter = RecursiveCharacterTextSplitter(
chunk_size = 1000,
chunk_overlap = 200,
length_function = len,
)
chunks = text_splitter.split_documents(docs)
return chunks
def create_embeddings_load_data():
embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
return embeddings
def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs):
PineconeClient(
api_key=pinecone_apikey,
environment=pinecone_environment
)
index_name = pinecone_index_name
index = Pinecone.from_documents(docs, embeddings, index_name=index_name)
return index
def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings):
PineconeClient(
api_key=pinecone_apikey,
environment=pinecone_environment
)
index_name = pinecone_index_name
index = Pinecone.from_existing_index(index_name, embeddings)
return index
def get_similar_docs(index,query,k=2):
similar_docs = index.similarity_search(query, k=k)
return similar_docs
def get_answer(docs,user_input):
chain = load_qa_chain(OpenAI(), chain_type="stuff")
with get_openai_callback() as cb:
response = chain.run(input_documents=docs, question=user_input)
return response
def predict(query_result):
Fitmodel = joblib.load('modelsvm.pk1')
result=Fitmodel.predict([query_result])
return result[0]
def read_data(data):
df = pd.read_csv(data,delimiter=',', header=None)
return df
#Create embeddings
def get_embeddings():
embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
return embeddings
#Generating embeddings for input dataset
def create_embeddings(df,embeddings):
df[2] = df[0].apply(lambda x: embeddings.embed_query(x))
return df
#Splitting the data into train & test
def split_train_test__data(df_sample):
# Split into training and testing sets
sentences_train, sentences_test, labels_train, labels_test = train_test_split(
list(df_sample[2]), list(df_sample[1]), test_size=0.25, random_state=0)
print(len(sentences_train))
return sentences_train, sentences_test, labels_train, labels_test
#Get the accuracy score on test data
def get_score(svm_classifier,sentences_test,labels_test):
score = svm_classifier.score(sentences_test, labels_test)
return score