Spaces:
Sleeping
Sleeping
File size: 1,107 Bytes
7745c91 bcb3532 91e7385 7745c91 05086e1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | import os
import PyPDF2
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import Chroma
# Load and process the book
def extract_text_from_pdf(pdf_path):
with open(pdf_path, "rb") as f:
reader = PyPDF2.PdfReader(f)
text = " ".join([page.extract_text() for page in reader.pages if page.extract_text()])
return text
# Convert text into searchable chunks
def store_book_embeddings(pdf_path, db_path="embeddings"):
book_text = extract_text_from_pdf(pdf_path)
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
chunks = splitter.split_text(book_text)
# Convert text into embeddings
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
vectorstore = Chroma.from_texts(chunks, embeddings, persist_directory=db_path)
print("✅ Book successfully stored in ChromaDB!")
# Run this script to process the book
if __name__ == "__main__":
store_book_embeddings("data/Adventures_of_Sherlock_Holmes.pdf")
|