Spaces:
Sleeping
Sleeping
Download Src/ingestion/data_loader.py from kanhacoderx/Documind-AI: direct link, hf CLI and curl.
- Browser
- Download file 1.03 kB
-
https://huggingface.co/spaces/kanhacoderx/Documind-AI/resolve/main/Src/ingestion/data_loader.py
- Command line
-
hf download hf://spaces/kanhacoderx/Documind-AI/Src/ingestion/data_loader.py
-
curl -L -o data_loader.py https://huggingface.co/spaces/kanhacoderx/Documind-AI/resolve/main/Src/ingestion/data_loader.py
1.03 kB
| from langchain_community.document_loaders import PyPDFLoader | |
| from langchain_community.document_loaders import PyPDFLoader | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from dataclasses import dataclass | |
| #Create DataIngestionConfig | |
| class DataIngestion: | |
| def __init__(self,file_path:str): | |
| self.file_path=file_path | |
| def load_documents(self): | |
| ''' | |
| load pdf and return document object | |
| ''' | |
| loader=PyPDFLoader(self.file_path) | |
| documents=loader.load() | |
| return documents | |
| def split_documents(self,documents): | |
| ''' | |
| Split Document into chunks | |
| ''' | |
| text_splitter=RecursiveCharacterTextSplitter( | |
| chunk_size=500, | |
| chunk_overlap=50 | |
| ) | |
| chunks=text_splitter.split_documents(documents) | |
| return chunks | |
| def ingests(self): | |
| '''Pipeline''' | |
| docs=self.load_documents() | |
| chunks=self.split_documents(docs) | |
| return chunks |