Spaces:
Sleeping
Sleeping
Download process_book.py from scorpio84/pharma_analytics: direct link, hf CLI and curl.
- Browser
- Download file 1.11 kB
-
https://huggingface.co/spaces/scorpio84/pharma_analytics/resolve/main/process_book.py
- Command line
-
hf download hf://spaces/scorpio84/pharma_analytics/process_book.py
-
curl -L -o process_book.py https://huggingface.co/spaces/scorpio84/pharma_analytics/resolve/main/process_book.py
1.11 kB
| import os | |
| import PyPDF2 | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.embeddings import HuggingFaceEmbeddings | |
| from langchain.vectorstores import Chroma | |
| # Load and process the book | |
| def extract_text_from_pdf(pdf_path): | |
| with open(pdf_path, "rb") as f: | |
| reader = PyPDF2.PdfReader(f) | |
| text = " ".join([page.extract_text() for page in reader.pages if page.extract_text()]) | |
| return text | |
| # Convert text into searchable chunks | |
| def store_book_embeddings(pdf_path, db_path="embeddings"): | |
| book_text = extract_text_from_pdf(pdf_path) | |
| splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) | |
| chunks = splitter.split_text(book_text) | |
| # Convert text into embeddings | |
| embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") | |
| vectorstore = Chroma.from_texts(chunks, embeddings, persist_directory=db_path) | |
| print("✅ Book successfully stored in ChromaDB!") | |
| # Run this script to process the book | |
| if __name__ == "__main__": | |
| store_book_embeddings("data/Adventures_of_Sherlock_Holmes.pdf") | |