vmalapaka commited on
Commit
5ad63cd
·
verified ·
1 Parent(s): c6b393c

Create utils.py

Browse files
Files changed (1) hide show
  1. utils.py +89 -0
utils.py ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain.vectorstores import Pinecone
2
+ from langchain.llms import OpenAI
3
+ from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
4
+ from langchain.schema import Document
5
+ import pinecone
6
+ from pypdf import PdfReader
7
+ from langchain.llms.openai import OpenAI
8
+ from langchain.chains.summarize import load_summarize_chain
9
+ from langchain.llms import HuggingFaceHub
10
+
11
+ import time
12
+
13
+
14
+ def get_pdf_text(pdf_doc):
15
+ text = ""
16
+ pdf_reader = PdfReader(pdf_doc)
17
+ for page in pdf_reader.pages:
18
+ text += page.extract_text()
19
+ return text
20
+
21
+ def create_docs(user_pdf_list, unique_id):
22
+ docs=[]
23
+ for filename in user_pdf_list:
24
+
25
+ chunks=get_pdf_text(filename)
26
+
27
+ #Adding items to our list - Adding data & its metadata
28
+ docs.append(Document(
29
+ page_content=chunks,
30
+ metadata={"name": filename.name,"id":filename.file_id,"type=":filename.type,"size":filename.size,"unique_id":unique_id},
31
+ ))
32
+
33
+ return docs
34
+
35
+ def create_embeddings_load_data():
36
+ embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
37
+ return embeddings
38
+
39
+
40
+ #Function to push data to Vector Store - Pinecone here
41
+ def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs):
42
+
43
+ pinecone.init(
44
+ api_key=pinecone_apikey,
45
+ environment=pinecone_environment
46
+ )
47
+
48
+ Pinecone.from_documents(docs, embeddings, index_name=pinecone_index_name)
49
+
50
+
51
+
52
+ #Function to pull infrmation from Vector Store - Pinecone here
53
+ def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings):
54
+
55
+ time.sleep(20)
56
+ pinecone.init(
57
+ api_key=pinecone_apikey,
58
+ environment=pinecone_environment
59
+ )
60
+
61
+ index_name = pinecone_index_name
62
+
63
+ index = Pinecone.from_existing_index(index_name, embeddings)
64
+ return index
65
+
66
+
67
+
68
+ #Function to help us get relavant documents from vector store - based on user input
69
+ def similar_docs(query,k,pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,unique_id):
70
+
71
+ pinecone.init(
72
+ api_key=pinecone_apikey,
73
+ environment=pinecone_environment
74
+ )
75
+
76
+ index_name = pinecone_index_name
77
+
78
+ index = pull_from_pinecone(pinecone_apikey,pinecone_environment,index_name,embeddings)
79
+ similar_docs = index.similarity_search_with_score(query, int(k),{"unique_id":unique_id})
80
+ return similar_docs
81
+
82
+
83
+ # Summary of a document
84
+ def get_summary(current_doc):
85
+ llm = OpenAI(temperature=0)
86
+ chain = load_summarize_chain(llm, chain_type="map_reduce")
87
+ summary = chain.run([current_doc])
88
+
89
+ return summary