File size: 1,107 Bytes
7745c91
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bcb3532
91e7385
7745c91
 
 
 
 
05086e1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
import os
import PyPDF2
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import Chroma

# Load and process the book
def extract_text_from_pdf(pdf_path):
    with open(pdf_path, "rb") as f:
        reader = PyPDF2.PdfReader(f)
        text = " ".join([page.extract_text() for page in reader.pages if page.extract_text()])
    return text

# Convert text into searchable chunks
def store_book_embeddings(pdf_path, db_path="embeddings"):
    book_text = extract_text_from_pdf(pdf_path)
    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
    chunks = splitter.split_text(book_text)

    # Convert text into embeddings
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
vectorstore = Chroma.from_texts(chunks, embeddings, persist_directory=db_path)
    
    print("✅ Book successfully stored in ChromaDB!")

# Run this script to process the book
if __name__ == "__main__":
    store_book_embeddings("data/Adventures_of_Sherlock_Holmes.pdf")