Spaces:
Build error
Build error
Download utils.py from vmalapaka/GrievanceTicketGeneration_ChatBot: direct link, hf CLI and curl.
- Browser
- Download file 3.17 kB
-
https://huggingface.co/spaces/vmalapaka/GrievanceTicketGeneration_ChatBot/resolve/main/utils.py
- Command line
-
hf download hf://spaces/vmalapaka/GrievanceTicketGeneration_ChatBot/utils.py
-
curl -L -o utils.py https://huggingface.co/spaces/vmalapaka/GrievanceTicketGeneration_ChatBot/resolve/main/utils.py
3.17 kB
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain_community.vectorstores import Pinecone | |
| from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings | |
| from pinecone import Pinecone as PineconeClient | |
| from PyPDF2 import PdfReader | |
| from langchain.document_loaders.sitemap import SitemapLoader | |
| from langchain_openai import OpenAI | |
| from langchain.chains.question_answering import load_qa_chain | |
| from langchain_community.callbacks import get_openai_callback | |
| import joblib | |
| import pandas as pd | |
| from sklearn.model_selection import train_test_split | |
| def read_pdf_data(pdf): | |
| pdf_read = PdfReader(pdf) | |
| text = "" | |
| for page in pdf_read.pages: | |
| text +=page.extract_data() | |
| return text | |
| def split_data(docs): | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size = 1000, | |
| chunk_overlap = 200, | |
| length_function = len, | |
| ) | |
| chunks = text_splitter.split_documents(docs) | |
| return chunks | |
| def create_embeddings_load_data(): | |
| embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") | |
| return embeddings | |
| def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs): | |
| PineconeClient( | |
| api_key=pinecone_apikey, | |
| environment=pinecone_environment | |
| ) | |
| index_name = pinecone_index_name | |
| index = Pinecone.from_documents(docs, embeddings, index_name=index_name) | |
| return index | |
| def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings): | |
| PineconeClient( | |
| api_key=pinecone_apikey, | |
| environment=pinecone_environment | |
| ) | |
| index_name = pinecone_index_name | |
| index = Pinecone.from_existing_index(index_name, embeddings) | |
| return index | |
| def get_similar_docs(index,query,k=2): | |
| similar_docs = index.similarity_search(query, k=k) | |
| return similar_docs | |
| def get_answer(docs,user_input): | |
| chain = load_qa_chain(OpenAI(), chain_type="stuff") | |
| with get_openai_callback() as cb: | |
| response = chain.run(input_documents=docs, question=user_input) | |
| return response | |
| def predict(query_result): | |
| Fitmodel = joblib.load('modelsvm.pk1') | |
| result=Fitmodel.predict([query_result]) | |
| return result[0] | |
| def read_data(data): | |
| df = pd.read_csv(data,delimiter=',', header=None) | |
| return df | |
| #Create embeddings | |
| def get_embeddings(): | |
| embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") | |
| return embeddings | |
| #Generating embeddings for input dataset | |
| def create_embeddings(df,embeddings): | |
| df[2] = df[0].apply(lambda x: embeddings.embed_query(x)) | |
| return df | |
| #Splitting the data into train & test | |
| def split_train_test__data(df_sample): | |
| # Split into training and testing sets | |
| sentences_train, sentences_test, labels_train, labels_test = train_test_split( | |
| list(df_sample[2]), list(df_sample[1]), test_size=0.25, random_state=0) | |
| print(len(sentences_train)) | |
| return sentences_train, sentences_test, labels_train, labels_test | |
| #Get the accuracy score on test data | |
| def get_score(svm_classifier,sentences_test,labels_test): | |
| score = svm_classifier.score(sentences_test, labels_test) | |
| return score | |