thomasjacob04 commited on
Commit
fc6fc96
·
verified ·
1 Parent(s): 43b8f26

update for 5 PDFs (#1)

Browse files

- update for 5 PDFs (d0a44ac13ec7e29db05be6e10ec6316db04223af)

Files changed (1) hide show
  1. app.py +38 -9
app.py CHANGED
@@ -9,21 +9,50 @@ from langchain.schema.runnable import RunnablePassthrough
9
  from langchain.schema.output_parser import StrOutputParser
10
  import gradio as gr
11
 
 
 
 
12
  # Load and split documents
13
- FILE_PATH = "vol1.pdf"
14
- loader = PyPDFLoader(FILE_PATH)
15
- docs = loader.load()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
 
17
- text_splitter = RecursiveCharacterTextSplitter(
18
- chunk_size=1000,
19
- chunk_overlap=200,
20
- )
21
- splits = text_splitter.split_documents(docs)
22
 
23
  # Set up embeddings and vector store
24
  HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
25
  embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
26
- vectorstore = FAISS.from_documents(splits, embeddings)
27
 
28
  # Set up LLM
29
  HF_API_KEY = os.environ.get("HF_API_KEY") # Ensure this is set in Hugging Face Space settings
 
9
  from langchain.schema.output_parser import StrOutputParser
10
  import gradio as gr
11
 
12
+
13
+ ############################## OLD CODE FOR 1 PDF ####################################
14
+
15
  # Load and split documents
16
+ # FILE_PATH = "vol1.pdf"
17
+ # loader = PyPDFLoader(FILE_PATH)
18
+ # docs = loader.load()
19
+
20
+ # text_splitter = RecursiveCharacterTextSplitter(
21
+ # chunk_size=1000,
22
+ # chunk_overlap=200,
23
+ # )
24
+ # splits = text_splitter.split_documents(docs)
25
+
26
+ # # Set up embeddings and vector store
27
+ # HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
28
+ # embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
29
+ # vectorstore = FAISS.from_documents(splits, embeddings)
30
+
31
+ ############################## NEW CODE FOR 5 PDFs ####################################
32
+ # Load and split documents
33
+ FILE_PATHS = ["vol1.pdf", "vol2.pdf", "vol3.pdf", "vol4.pdf", "vol5.pdf"]
34
+
35
+ # Initialize an empty list to store all documents
36
+ all_docs = []
37
+
38
+ # Iterate through each file path in FILE_PATHS
39
+ for file_path in FILE_PATHS:
40
+ # Load the PDF using PyPDFLoader
41
+ loader = PyPDFLoader(file_path)
42
+ # Split the loaded document using RecursiveCharacterTextSplitter
43
+ text_splitter = RecursiveCharacterTextSplitter(
44
+ chunk_size=1000,
45
+ chunk_overlap=200,
46
+ )
47
+ # Load and split the current document and append to all_docs
48
+ documents = loader.load_and_split(text_splitter)
49
+ all_docs.extend(documents)
50
 
 
 
 
 
 
51
 
52
  # Set up embeddings and vector store
53
  HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
54
  embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
55
+ vectorstore = FAISS.from_documents(all_docs, embeddings)
56
 
57
  # Set up LLM
58
  HF_API_KEY = os.environ.get("HF_API_KEY") # Ensure this is set in Hugging Face Space settings