Spaces:
Runtime error
Runtime error
| import markdown | |
| import re | |
| from typing import List, Dict, Any | |
| def strip_frontmatter(content: str) -> str: | |
| """Remove YAML frontmatter (---...---) from markdown content.""" | |
| return re.sub(r'^---\s*\n.*?\n---\s*\n', '', content, flags=re.DOTALL) | |
| def semantic_chunking(markdown_content: str, min_tokens: int = 100, max_tokens: int = 512, overlap_tokens: int = 50) -> List[Dict[str, Any]]: | |
| content = strip_frontmatter(markdown_content) | |
| chunks = [] | |
| paragraphs = content.split('\n\n') | |
| current_chunk_content = "" | |
| current_heading = "Introduction" | |
| for paragraph in paragraphs: | |
| if paragraph.strip().startswith('#'): | |
| current_heading = paragraph.strip().lstrip('#').strip() | |
| paragraph_tokens = len(paragraph.split()) | |
| if len(current_chunk_content.split()) + paragraph_tokens > max_tokens: | |
| if current_chunk_content.strip(): | |
| chunks.append({ | |
| "content": current_chunk_content.strip(), | |
| "metadata": {"source": "unknown", "heading": current_heading} | |
| }) | |
| current_chunk_content = paragraph | |
| else: | |
| current_chunk_content += "\n\n" + paragraph | |
| if current_chunk_content.strip(): | |
| chunks.append({ | |
| "content": current_chunk_content.strip(), | |
| "metadata": {"source": "unknown", "heading": current_heading} | |
| }) | |
| return chunks | |