app/utils/chunking.py

import markdown
import re
from typing import List, Dict, Any

def strip_frontmatter(content: str) -> str:
    """Remove YAML frontmatter (---...---) from markdown content."""
    return re.sub(r'^---\s*\n.*?\n---\s*\n', '', content, flags=re.DOTALL)

def semantic_chunking(markdown_content: str, min_tokens: int = 100, max_tokens: int = 512, overlap_tokens: int = 50) -> List[Dict[str, Any]]:
    content = strip_frontmatter(markdown_content)
    chunks = []
    paragraphs = content.split('\n\n')

    current_chunk_content = ""
    current_heading = "Introduction"

    for paragraph in paragraphs:
        if paragraph.strip().startswith('#'):
            current_heading = paragraph.strip().lstrip('#').strip()

        paragraph_tokens = len(paragraph.split())

        if len(current_chunk_content.split()) + paragraph_tokens > max_tokens:
            if current_chunk_content.strip():
                chunks.append({
                    "content": current_chunk_content.strip(),
                    "metadata": {"source": "unknown", "heading": current_heading}
                })
            current_chunk_content = paragraph
        else:
            current_chunk_content += "\n\n" + paragraph

    if current_chunk_content.strip():
        chunks.append({
            "content": current_chunk_content.strip(),
            "metadata": {"source": "unknown", "heading": current_heading}
        })

    return chunks