import os import shutil import config import pymupdf.layout import pymupdf4llm from pathlib import Path import glob import tiktoken from functools import lru_cache def clear_directory_contents(directory: Path) -> None: """Delete everything under directory but not the directory itself (safe for Docker volume / bind mount roots).""" directory = Path(directory) if not directory.is_dir(): return for child in directory.iterdir(): if child.is_dir(): shutil.rmtree(child) else: child.unlink() os.environ["TOKENIZERS_PARALLELISM"] = "false" def pdf_to_markdown(pdf_path, output_dir): doc = pymupdf.open(pdf_path) md = pymupdf4llm.to_markdown(doc, header=False, footer=False, page_separators=True, ignore_images=True, write_images=False, image_path=None) md_cleaned = md.encode('utf-8', errors='surrogatepass').decode('utf-8', errors='ignore') output_path = Path(output_dir) / Path(doc.name).stem Path(output_path).with_suffix(".md").write_bytes(md_cleaned.encode('utf-8')) def pdfs_to_markdowns(path_pattern, overwrite: bool = False): output_dir = Path(config.MARKDOWN_DIR) output_dir.mkdir(parents=True, exist_ok=True) for pdf_path in map(Path, glob.glob(path_pattern)): md_path = (output_dir / pdf_path.stem).with_suffix(".md") if overwrite or not md_path.exists(): pdf_to_markdown(pdf_path, output_dir) @lru_cache(maxsize=1) def _get_token_encoding(): try: return tiktoken.encoding_for_model("gpt-4") except Exception: try: return tiktoken.get_encoding("cl100k_base") except Exception: return None def estimate_context_tokens(messages: list) -> int: contents = [ str(msg.content) for msg in messages if hasattr(msg, "content") and msg.content ] encoding = _get_token_encoding() if encoding is None: return sum(max(1, len(content) // 4) for content in contents) return sum(len(encoding.encode(content)) for content in contents)