Spaces:
Sleeping
Sleeping
Download project/document_chunker.py from devank2000/agentic-rag: direct link, hf CLI and curl.
- Browser
- Download file 8.1 kB
-
https://huggingface.co/spaces/devank2000/agentic-rag/resolve/main/project/document_chunker.py
- Command line
-
hf download hf://spaces/devank2000/agentic-rag/project/document_chunker.py
-
curl -L -o document_chunker.py https://huggingface.co/spaces/devank2000/agentic-rag/resolve/main/project/document_chunker.py
8.1 kB
| import os | |
| import glob | |
| import config | |
| from pathlib import Path | |
| from langchain_text_splitters import MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter | |
| class DocumentChunker: | |
| def __init__(self): | |
| if config.MIN_PARENT_SIZE <= 0 or config.MAX_PARENT_SIZE < config.MIN_PARENT_SIZE: | |
| raise ValueError("Parent chunk sizes must be positive and MIN_PARENT_SIZE <= MAX_PARENT_SIZE.") | |
| if not 0 <= config.CHILD_CHUNK_OVERLAP < config.CHILD_CHUNK_SIZE: | |
| raise ValueError("CHILD_CHUNK_OVERLAP must be smaller than CHILD_CHUNK_SIZE.") | |
| if config.CHILD_CHUNK_OVERLAP >= config.MAX_PARENT_SIZE: | |
| raise ValueError("CHILD_CHUNK_OVERLAP must be smaller than MAX_PARENT_SIZE.") | |
| self.__parent_splitter = MarkdownHeaderTextSplitter( | |
| headers_to_split_on=config.HEADERS_TO_SPLIT_ON, | |
| strip_headers=False | |
| ) | |
| self.__child_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=config.CHILD_CHUNK_SIZE, | |
| chunk_overlap=config.CHILD_CHUNK_OVERLAP | |
| ) | |
| self.__min_parent_size = config.MIN_PARENT_SIZE | |
| self.__max_parent_size = config.MAX_PARENT_SIZE | |
| def __merge_metadata(target, source, prepend=False): | |
| for key, value in source.items(): | |
| if key not in target: | |
| target[key] = value | |
| else: | |
| first, second = (value, target[key]) if prepend else (target[key], value) | |
| values = [ | |
| item.strip() | |
| for raw in (first, second) | |
| for item in str(raw).split(" -> ") | |
| if item.strip() | |
| ] | |
| target[key] = " -> ".join(dict.fromkeys(values)) | |
| def create_chunks(self, path_dir=config.MARKDOWN_DIR): | |
| all_parent_chunks, all_child_chunks = [], [] | |
| for doc_path_str in sorted(glob.glob(os.path.join(path_dir, "*.md"))): | |
| doc_path = Path(doc_path_str) | |
| parent_chunks, child_chunks = self.create_chunks_single(doc_path) | |
| all_parent_chunks.extend(parent_chunks) | |
| all_child_chunks.extend(child_chunks) | |
| return all_parent_chunks, all_child_chunks | |
| def create_chunks_single(self, md_path, source_name=None): | |
| doc_path = Path(md_path) | |
| source_name = source_name or f"{doc_path.stem}.pdf" | |
| with open(doc_path, "r", encoding="utf-8") as f: | |
| parent_chunks = self.__parent_splitter.split_text(f.read()) | |
| merged_parents = self.__merge_small_parents(parent_chunks) | |
| split_parents = self.__split_large_parents(merged_parents) | |
| cleaned_parents = self.__clean_small_chunks(split_parents) | |
| if any(len(chunk.page_content) > self.__max_parent_size for chunk in cleaned_parents): | |
| raise ValueError("Parent chunking produced a chunk larger than MAX_PARENT_SIZE.") | |
| all_parent_chunks, all_child_chunks = [], [] | |
| self.__create_child_chunks( | |
| all_parent_chunks, | |
| all_child_chunks, | |
| cleaned_parents, | |
| doc_path, | |
| source_name, | |
| ) | |
| return all_parent_chunks, all_child_chunks | |
| def __merge_small_parents(self, chunks): | |
| if not chunks: | |
| return [] | |
| merged, current = [], None | |
| for chunk in chunks: | |
| if current is None: | |
| current = chunk | |
| else: | |
| current.page_content += "\n\n" + chunk.page_content | |
| self.__merge_metadata(current.metadata, chunk.metadata) | |
| if len(current.page_content) >= self.__min_parent_size: | |
| merged.append(current) | |
| current = None | |
| if current: | |
| if merged: | |
| merged[-1].page_content += "\n\n" + current.page_content | |
| self.__merge_metadata(merged[-1].metadata, current.metadata) | |
| else: | |
| merged.append(current) | |
| return merged | |
| def __split_large_parents(self, chunks): | |
| split_chunks = [] | |
| for chunk in chunks: | |
| if len(chunk.page_content) <= self.__max_parent_size: | |
| split_chunks.append(chunk) | |
| else: | |
| splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=self.__max_parent_size, | |
| chunk_overlap=config.CHILD_CHUNK_OVERLAP | |
| ) | |
| sub_chunks = splitter.split_documents([chunk]) | |
| split_chunks.extend(sub_chunks) | |
| return split_chunks | |
| def __rebalance_pair(self, first, second): | |
| combined = first.page_content.rstrip() + "\n\n" + second.page_content.lstrip() | |
| lower = max(1, len(combined) - self.__max_parent_size) | |
| upper = min(self.__max_parent_size, len(combined) - 1) | |
| if len(combined) >= 2 * self.__min_parent_size: | |
| lower = max(lower, self.__min_parent_size) | |
| upper = min(upper, len(combined) - self.__min_parent_size) | |
| preferred = min(max(len(combined) // 2, lower), upper) | |
| split_at = preferred | |
| for separator in ("\n\n", "\n", " "): | |
| before = combined.rfind(separator, lower, preferred + 1) | |
| after = combined.find(separator, preferred, upper + 1) | |
| if before >= lower: | |
| split_at = before | |
| break | |
| if after != -1: | |
| split_at = after | |
| break | |
| left_text = combined[:split_at].rstrip() | |
| right_text = combined[split_at:].lstrip() | |
| if len(combined) >= 2 * self.__min_parent_size and ( | |
| len(left_text) < self.__min_parent_size | |
| or len(right_text) < self.__min_parent_size | |
| ): | |
| split_at = preferred | |
| left_text, right_text = combined[:split_at], combined[split_at:] | |
| if not left_text or not right_text: | |
| return first, second | |
| metadata = dict(first.metadata) | |
| self.__merge_metadata(metadata, second.metadata) | |
| first.page_content, first.metadata = left_text, dict(metadata) | |
| second.page_content, second.metadata = right_text, dict(metadata) | |
| return first, second | |
| def __clean_small_chunks(self, chunks): | |
| cleaned = [] | |
| for i, chunk in enumerate(chunks): | |
| if len(chunk.page_content) < self.__min_parent_size: | |
| if cleaned and len(cleaned[-1].page_content) + 2 + len(chunk.page_content) <= self.__max_parent_size: | |
| cleaned[-1].page_content += "\n\n" + chunk.page_content | |
| self.__merge_metadata(cleaned[-1].metadata, chunk.metadata) | |
| elif ( | |
| i < len(chunks) - 1 | |
| and len(chunk.page_content) + 2 + len(chunks[i + 1].page_content) <= self.__max_parent_size | |
| ): | |
| chunks[i + 1].page_content = chunk.page_content + "\n\n" + chunks[i + 1].page_content | |
| self.__merge_metadata(chunks[i + 1].metadata, chunk.metadata, prepend=True) | |
| else: | |
| cleaned.append(chunk) | |
| else: | |
| cleaned.append(chunk) | |
| for i, chunk in enumerate(cleaned): | |
| if len(chunk.page_content) >= self.__min_parent_size or len(cleaned) == 1: | |
| continue | |
| if i < len(cleaned) - 1: | |
| cleaned[i], cleaned[i + 1] = self.__rebalance_pair(chunk, cleaned[i + 1]) | |
| else: | |
| cleaned[i - 1], cleaned[i] = self.__rebalance_pair(cleaned[i - 1], chunk) | |
| return cleaned | |
| def __create_child_chunks(self, all_parent_pairs, all_child_chunks, parent_chunks, doc_path, source_name): | |
| for i, p_chunk in enumerate(parent_chunks): | |
| parent_id = f"{doc_path.stem}_p{i}" | |
| p_chunk.metadata.update({"source": source_name, "parent_id": parent_id}) | |
| all_parent_pairs.append((parent_id, p_chunk)) | |
| all_child_chunks.extend(self.__child_splitter.split_documents([p_chunk])) | |