agentic-rag / project /document_chunker.py
devank2000's picture
initial commit
b3b0ccd verified
Raw History Blame Contribute Delete
8.1 kB
import os
import glob
import config
from pathlib import Path
from langchain_text_splitters import MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter
class DocumentChunker:
def __init__(self):
if config.MIN_PARENT_SIZE <= 0 or config.MAX_PARENT_SIZE < config.MIN_PARENT_SIZE:
raise ValueError("Parent chunk sizes must be positive and MIN_PARENT_SIZE <= MAX_PARENT_SIZE.")
if not 0 <= config.CHILD_CHUNK_OVERLAP < config.CHILD_CHUNK_SIZE:
raise ValueError("CHILD_CHUNK_OVERLAP must be smaller than CHILD_CHUNK_SIZE.")
if config.CHILD_CHUNK_OVERLAP >= config.MAX_PARENT_SIZE:
raise ValueError("CHILD_CHUNK_OVERLAP must be smaller than MAX_PARENT_SIZE.")
self.__parent_splitter = MarkdownHeaderTextSplitter(
headers_to_split_on=config.HEADERS_TO_SPLIT_ON,
strip_headers=False
)
self.__child_splitter = RecursiveCharacterTextSplitter(
chunk_size=config.CHILD_CHUNK_SIZE,
chunk_overlap=config.CHILD_CHUNK_OVERLAP
)
self.__min_parent_size = config.MIN_PARENT_SIZE
self.__max_parent_size = config.MAX_PARENT_SIZE
@staticmethod
def __merge_metadata(target, source, prepend=False):
for key, value in source.items():
if key not in target:
target[key] = value
else:
first, second = (value, target[key]) if prepend else (target[key], value)
values = [
item.strip()
for raw in (first, second)
for item in str(raw).split(" -> ")
if item.strip()
]
target[key] = " -> ".join(dict.fromkeys(values))
def create_chunks(self, path_dir=config.MARKDOWN_DIR):
all_parent_chunks, all_child_chunks = [], []
for doc_path_str in sorted(glob.glob(os.path.join(path_dir, "*.md"))):
doc_path = Path(doc_path_str)
parent_chunks, child_chunks = self.create_chunks_single(doc_path)
all_parent_chunks.extend(parent_chunks)
all_child_chunks.extend(child_chunks)
return all_parent_chunks, all_child_chunks
def create_chunks_single(self, md_path, source_name=None):
doc_path = Path(md_path)
source_name = source_name or f"{doc_path.stem}.pdf"
with open(doc_path, "r", encoding="utf-8") as f:
parent_chunks = self.__parent_splitter.split_text(f.read())
merged_parents = self.__merge_small_parents(parent_chunks)
split_parents = self.__split_large_parents(merged_parents)
cleaned_parents = self.__clean_small_chunks(split_parents)
if any(len(chunk.page_content) > self.__max_parent_size for chunk in cleaned_parents):
raise ValueError("Parent chunking produced a chunk larger than MAX_PARENT_SIZE.")
all_parent_chunks, all_child_chunks = [], []
self.__create_child_chunks(
all_parent_chunks,
all_child_chunks,
cleaned_parents,
doc_path,
source_name,
)
return all_parent_chunks, all_child_chunks
def __merge_small_parents(self, chunks):
if not chunks:
return []
merged, current = [], None
for chunk in chunks:
if current is None:
current = chunk
else:
current.page_content += "\n\n" + chunk.page_content
self.__merge_metadata(current.metadata, chunk.metadata)
if len(current.page_content) >= self.__min_parent_size:
merged.append(current)
current = None
if current:
if merged:
merged[-1].page_content += "\n\n" + current.page_content
self.__merge_metadata(merged[-1].metadata, current.metadata)
else:
merged.append(current)
return merged
def __split_large_parents(self, chunks):
split_chunks = []
for chunk in chunks:
if len(chunk.page_content) <= self.__max_parent_size:
split_chunks.append(chunk)
else:
splitter = RecursiveCharacterTextSplitter(
chunk_size=self.__max_parent_size,
chunk_overlap=config.CHILD_CHUNK_OVERLAP
)
sub_chunks = splitter.split_documents([chunk])
split_chunks.extend(sub_chunks)
return split_chunks
def __rebalance_pair(self, first, second):
combined = first.page_content.rstrip() + "\n\n" + second.page_content.lstrip()
lower = max(1, len(combined) - self.__max_parent_size)
upper = min(self.__max_parent_size, len(combined) - 1)
if len(combined) >= 2 * self.__min_parent_size:
lower = max(lower, self.__min_parent_size)
upper = min(upper, len(combined) - self.__min_parent_size)
preferred = min(max(len(combined) // 2, lower), upper)
split_at = preferred
for separator in ("\n\n", "\n", " "):
before = combined.rfind(separator, lower, preferred + 1)
after = combined.find(separator, preferred, upper + 1)
if before >= lower:
split_at = before
break
if after != -1:
split_at = after
break
left_text = combined[:split_at].rstrip()
right_text = combined[split_at:].lstrip()
if len(combined) >= 2 * self.__min_parent_size and (
len(left_text) < self.__min_parent_size
or len(right_text) < self.__min_parent_size
):
split_at = preferred
left_text, right_text = combined[:split_at], combined[split_at:]
if not left_text or not right_text:
return first, second
metadata = dict(first.metadata)
self.__merge_metadata(metadata, second.metadata)
first.page_content, first.metadata = left_text, dict(metadata)
second.page_content, second.metadata = right_text, dict(metadata)
return first, second
def __clean_small_chunks(self, chunks):
cleaned = []
for i, chunk in enumerate(chunks):
if len(chunk.page_content) < self.__min_parent_size:
if cleaned and len(cleaned[-1].page_content) + 2 + len(chunk.page_content) <= self.__max_parent_size:
cleaned[-1].page_content += "\n\n" + chunk.page_content
self.__merge_metadata(cleaned[-1].metadata, chunk.metadata)
elif (
i < len(chunks) - 1
and len(chunk.page_content) + 2 + len(chunks[i + 1].page_content) <= self.__max_parent_size
):
chunks[i + 1].page_content = chunk.page_content + "\n\n" + chunks[i + 1].page_content
self.__merge_metadata(chunks[i + 1].metadata, chunk.metadata, prepend=True)
else:
cleaned.append(chunk)
else:
cleaned.append(chunk)
for i, chunk in enumerate(cleaned):
if len(chunk.page_content) >= self.__min_parent_size or len(cleaned) == 1:
continue
if i < len(cleaned) - 1:
cleaned[i], cleaned[i + 1] = self.__rebalance_pair(chunk, cleaned[i + 1])
else:
cleaned[i - 1], cleaned[i] = self.__rebalance_pair(cleaned[i - 1], chunk)
return cleaned
def __create_child_chunks(self, all_parent_pairs, all_child_chunks, parent_chunks, doc_path, source_name):
for i, p_chunk in enumerate(parent_chunks):
parent_id = f"{doc_path.stem}_p{i}"
p_chunk.metadata.update({"source": source_name, "parent_id": parent_id})
all_parent_pairs.append((parent_id, p_chunk))
all_child_chunks.extend(self.__child_splitter.split_documents([p_chunk]))