pharma_analytics / process_book.py
scorpio84's picture
Update process_book.py
bcb3532 verified
Raw History Blame Contribute Delete
1.11 kB
import os
import PyPDF2
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import Chroma
# Load and process the book
def extract_text_from_pdf(pdf_path):
with open(pdf_path, "rb") as f:
reader = PyPDF2.PdfReader(f)
text = " ".join([page.extract_text() for page in reader.pages if page.extract_text()])
return text
# Convert text into searchable chunks
def store_book_embeddings(pdf_path, db_path="embeddings"):
book_text = extract_text_from_pdf(pdf_path)
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
chunks = splitter.split_text(book_text)
# Convert text into embeddings
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
vectorstore = Chroma.from_texts(chunks, embeddings, persist_directory=db_path)
print("✅ Book successfully stored in ChromaDB!")
# Run this script to process the book
if __name__ == "__main__":
store_book_embeddings("data/Adventures_of_Sherlock_Holmes.pdf")