vmalapaka commited on
Commit
2d9cf9b
·
verified ·
1 Parent(s): 3bd0a10

Create utils.py

Browse files
Files changed (1) hide show
  1. pages/utils.py +73 -0
pages/utils.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
2
+ from langchain_community.vectorstores import Pinecone
3
+ from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
4
+ from pinecone import Pinecone as PineconeClient
5
+ from pypdf import PdfReader
6
+ from langchain.document_loaders.sitemap import SitemapLoader
7
+
8
+
9
+ def read_pdf_data(pdf):
10
+
11
+ pdf_read = PdfReader(pdf)
12
+ text = ""
13
+ for page in pdf_read.pages:
14
+ text +=page.extract_data()
15
+
16
+
17
+ return text
18
+
19
+ def split_data(docs):
20
+
21
+ text_splitter = RecursiveCharacterTextSplitter(
22
+ chunk_size = 1000,
23
+ chunk_overlap = 200,
24
+ length_function = len,
25
+ )
26
+
27
+ chunks = text_splitter.split_documents(docs)
28
+ return chunks
29
+
30
+
31
+ def create_embeddings_load_data():
32
+
33
+ embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
34
+ return embeddings
35
+
36
+ def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs):
37
+
38
+ PineconeClient(
39
+ api_key=pinecone_apikey,
40
+ environment=pinecone_environment
41
+ )
42
+
43
+ index_name = pinecone_index_name
44
+ index = Pinecone.from_documents(docs, embeddings, index_name=index_name)
45
+ return index
46
+
47
+ def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings):
48
+
49
+ PineconeClient(
50
+ api_key=pinecone_apikey,
51
+ environment=pinecone_environment
52
+ )
53
+
54
+ index_name = pinecone_index_name
55
+ index = Pinecone.from_existing_index(index_name, embeddings)
56
+ return index
57
+
58
+ def get_similar_docs(index,query,k=2):
59
+
60
+ similar_docs = index.similarity_search(query, k=k)
61
+ return similar_docs
62
+
63
+ def get_answer(docs,user_input):
64
+ chain = load_qa_chain(OpenAI(), chain_type="stuff")
65
+ with get_openai_callback() as cb:
66
+ response = chain.run(input_documents=docs, question=user_input)
67
+ return response
68
+
69
+
70
+ def predict(query_result):
71
+ Fitmodel = joblib.load('modelsvm.pk1')
72
+ result=Fitmodel.predict([query_result])
73
+ return result[0]