Spaces:
Build error
Build error
update for 5 PDFs (#1)
Browse files- update for 5 PDFs (d0a44ac13ec7e29db05be6e10ec6316db04223af)
app.py
CHANGED
|
@@ -9,21 +9,50 @@ from langchain.schema.runnable import RunnablePassthrough
|
|
| 9 |
from langchain.schema.output_parser import StrOutputParser
|
| 10 |
import gradio as gr
|
| 11 |
|
|
|
|
|
|
|
|
|
|
| 12 |
# Load and split documents
|
| 13 |
-
FILE_PATH = "vol1.pdf"
|
| 14 |
-
loader = PyPDFLoader(FILE_PATH)
|
| 15 |
-
docs = loader.load()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
-
text_splitter = RecursiveCharacterTextSplitter(
|
| 18 |
-
chunk_size=1000,
|
| 19 |
-
chunk_overlap=200,
|
| 20 |
-
)
|
| 21 |
-
splits = text_splitter.split_documents(docs)
|
| 22 |
|
| 23 |
# Set up embeddings and vector store
|
| 24 |
HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
|
| 25 |
embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
|
| 26 |
-
vectorstore = FAISS.from_documents(
|
| 27 |
|
| 28 |
# Set up LLM
|
| 29 |
HF_API_KEY = os.environ.get("HF_API_KEY") # Ensure this is set in Hugging Face Space settings
|
|
|
|
| 9 |
from langchain.schema.output_parser import StrOutputParser
|
| 10 |
import gradio as gr
|
| 11 |
|
| 12 |
+
|
| 13 |
+
############################## OLD CODE FOR 1 PDF ####################################
|
| 14 |
+
|
| 15 |
# Load and split documents
|
| 16 |
+
# FILE_PATH = "vol1.pdf"
|
| 17 |
+
# loader = PyPDFLoader(FILE_PATH)
|
| 18 |
+
# docs = loader.load()
|
| 19 |
+
|
| 20 |
+
# text_splitter = RecursiveCharacterTextSplitter(
|
| 21 |
+
# chunk_size=1000,
|
| 22 |
+
# chunk_overlap=200,
|
| 23 |
+
# )
|
| 24 |
+
# splits = text_splitter.split_documents(docs)
|
| 25 |
+
|
| 26 |
+
# # Set up embeddings and vector store
|
| 27 |
+
# HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
|
| 28 |
+
# embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
|
| 29 |
+
# vectorstore = FAISS.from_documents(splits, embeddings)
|
| 30 |
+
|
| 31 |
+
############################## NEW CODE FOR 5 PDFs ####################################
|
| 32 |
+
# Load and split documents
|
| 33 |
+
FILE_PATHS = ["vol1.pdf", "vol2.pdf", "vol3.pdf", "vol4.pdf", "vol5.pdf"]
|
| 34 |
+
|
| 35 |
+
# Initialize an empty list to store all documents
|
| 36 |
+
all_docs = []
|
| 37 |
+
|
| 38 |
+
# Iterate through each file path in FILE_PATHS
|
| 39 |
+
for file_path in FILE_PATHS:
|
| 40 |
+
# Load the PDF using PyPDFLoader
|
| 41 |
+
loader = PyPDFLoader(file_path)
|
| 42 |
+
# Split the loaded document using RecursiveCharacterTextSplitter
|
| 43 |
+
text_splitter = RecursiveCharacterTextSplitter(
|
| 44 |
+
chunk_size=1000,
|
| 45 |
+
chunk_overlap=200,
|
| 46 |
+
)
|
| 47 |
+
# Load and split the current document and append to all_docs
|
| 48 |
+
documents = loader.load_and_split(text_splitter)
|
| 49 |
+
all_docs.extend(documents)
|
| 50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
|
| 52 |
# Set up embeddings and vector store
|
| 53 |
HF_EMBED_MODEL_ID = "BAAI/bge-small-en-v1.5"
|
| 54 |
embeddings = HuggingFaceEmbeddings(model_name=HF_EMBED_MODEL_ID)
|
| 55 |
+
vectorstore = FAISS.from_documents(all_docs, embeddings)
|
| 56 |
|
| 57 |
# Set up LLM
|
| 58 |
HF_API_KEY = os.environ.get("HF_API_KEY") # Ensure this is set in Hugging Face Space settings
|