from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.vectorstores import Pinecone from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings from pinecone import Pinecone as PineconeClient from PyPDF2 import PdfReader from langchain.document_loaders.sitemap import SitemapLoader from langchain_openai import OpenAI from langchain.chains.question_answering import load_qa_chain from langchain_community.callbacks import get_openai_callback import joblib import pandas as pd from sklearn.model_selection import train_test_split def read_pdf_data(pdf): pdf_read = PdfReader(pdf) text = "" for page in pdf_read.pages: text +=page.extract_data() return text def split_data(docs): text_splitter = RecursiveCharacterTextSplitter( chunk_size = 1000, chunk_overlap = 200, length_function = len, ) chunks = text_splitter.split_documents(docs) return chunks def create_embeddings_load_data(): embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") return embeddings def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs): PineconeClient( api_key=pinecone_apikey, environment=pinecone_environment ) index_name = pinecone_index_name index = Pinecone.from_documents(docs, embeddings, index_name=index_name) return index def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings): PineconeClient( api_key=pinecone_apikey, environment=pinecone_environment ) index_name = pinecone_index_name index = Pinecone.from_existing_index(index_name, embeddings) return index def get_similar_docs(index,query,k=2): similar_docs = index.similarity_search(query, k=k) return similar_docs def get_answer(docs,user_input): chain = load_qa_chain(OpenAI(), chain_type="stuff") with get_openai_callback() as cb: response = chain.run(input_documents=docs, question=user_input) return response def predict(query_result): Fitmodel = joblib.load('modelsvm.pk1') result=Fitmodel.predict([query_result]) return result[0] def read_data(data): df = pd.read_csv(data,delimiter=',', header=None) return df #Create embeddings def get_embeddings(): embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") return embeddings #Generating embeddings for input dataset def create_embeddings(df,embeddings): df[2] = df[0].apply(lambda x: embeddings.embed_query(x)) return df #Splitting the data into train & test def split_train_test__data(df_sample): # Split into training and testing sets sentences_train, sentences_test, labels_train, labels_test = train_test_split( list(df_sample[2]), list(df_sample[1]), test_size=0.25, random_state=0) print(len(sentences_train)) return sentences_train, sentences_test, labels_train, labels_test #Get the accuracy score on test data def get_score(svm_classifier,sentences_test,labels_test): score = svm_classifier.score(sentences_test, labels_test) return score