thomasjacob04 commited on
Commit
4116c43
·
verified ·
1 Parent(s): 3a5ab43

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +31 -31
app.py CHANGED
@@ -13,46 +13,46 @@ import gradio as gr
13
  ############################## OLD CODE FOR 1 PDF ####################################
14
 
15
  # Load and split documents
16
- # FILE_PATH = "vol1.pdf"
17
- # loader = PyPDFLoader(FILE_PATH)
18
- # docs = loader.load()
19
 
20
- # text_splitter = RecursiveCharacterTextSplitter(
21
- # chunk_size=1000,
22
- # chunk_overlap=200,
23
- # )
24
- # splits = text_splitter.split_documents(docs)
25
 
26
- # # Set up embeddings and vector store
27
- # HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
28
- # embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
29
- # vectorstore = FAISS.from_documents(splits, embeddings)
30
 
31
  ############################## NEW CODE FOR 5 PDFs ####################################
32
  # Load and split documents
33
- FILE_PATHS = ["vol1.pdf", "vol2.pdf"]
34
 
35
- # Initialize an empty list to store all documents
36
- all_docs = []
37
 
38
- # Iterate through each file path in FILE_PATHS
39
- for file_path in FILE_PATHS:
40
- # Load the PDF using PyPDFLoader
41
- loader = PyPDFLoader(file_path)
42
- # Split the loaded document using RecursiveCharacterTextSplitter
43
- text_splitter = RecursiveCharacterTextSplitter(
44
- chunk_size=1000,
45
- chunk_overlap=200,
46
- )
47
- # Load and split the current document and append to all_docs
48
- documents = loader.load_and_split(text_splitter)
49
- all_docs.extend(documents)
50
 
51
 
52
- # Set up embeddings and vector store
53
- HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
54
- embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
55
- vectorstore = FAISS.from_documents(all_docs, embeddings)
56
 
57
  # Set up LLM
58
  HF_API_KEY = os.environ.get("HF_API_KEY") # Ensure this is set in Hugging Face Space settings
 
13
  ############################## OLD CODE FOR 1 PDF ####################################
14
 
15
  # Load and split documents
16
+ FILE_PATH = "vol1.pdf"
17
+ loader = PyPDFLoader(FILE_PATH)
18
+ docs = loader.load()
19
 
20
+ text_splitter = RecursiveCharacterTextSplitter(
21
+ chunk_size=1000,
22
+ chunk_overlap=200,
23
+ )
24
+ splits = text_splitter.split_documents(docs)
25
 
26
+ # Set up embeddings and vector store
27
+ HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
28
+ embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
29
+ vectorstore = FAISS.from_documents(splits, embeddings)
30
 
31
  ############################## NEW CODE FOR 5 PDFs ####################################
32
  # Load and split documents
33
+ # FILE_PATHS = ["vol1.pdf", "vol2.pdf"]
34
 
35
+ # # Initialize an empty list to store all documents
36
+ # all_docs = []
37
 
38
+ # # Iterate through each file path in FILE_PATHS
39
+ # for file_path in FILE_PATHS:
40
+ # # Load the PDF using PyPDFLoader
41
+ # loader = PyPDFLoader(file_path)
42
+ # # Split the loaded document using RecursiveCharacterTextSplitter
43
+ # text_splitter = RecursiveCharacterTextSplitter(
44
+ # chunk_size=1000,
45
+ # chunk_overlap=200,
46
+ # )
47
+ # # Load and split the current document and append to all_docs
48
+ # documents = loader.load_and_split(text_splitter)
49
+ # all_docs.extend(documents)
50
 
51
 
52
+ # # Set up embeddings and vector store
53
+ # HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
54
+ # embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
55
+ # vectorstore = FAISS.from_documents(all_docs, embeddings)
56
 
57
  # Set up LLM
58
  HF_API_KEY = os.environ.get("HF_API_KEY") # Ensure this is set in Hugging Face Space settings