agentic-rag / project /utils.py
devank2000's picture
initial commit
b3b0ccd verified
Raw History Blame Contribute Delete
2.07 kB
import os
import shutil
import config
import pymupdf.layout
import pymupdf4llm
from pathlib import Path
import glob
import tiktoken
from functools import lru_cache
def clear_directory_contents(directory: Path) -> None:
"""Delete everything under directory but not the directory itself (safe for Docker volume / bind mount roots)."""
directory = Path(directory)
if not directory.is_dir():
return
for child in directory.iterdir():
if child.is_dir():
shutil.rmtree(child)
else:
child.unlink()
os.environ["TOKENIZERS_PARALLELISM"] = "false"
def pdf_to_markdown(pdf_path, output_dir):
doc = pymupdf.open(pdf_path)
md = pymupdf4llm.to_markdown(doc, header=False, footer=False, page_separators=True, ignore_images=True, write_images=False, image_path=None)
md_cleaned = md.encode('utf-8', errors='surrogatepass').decode('utf-8', errors='ignore')
output_path = Path(output_dir) / Path(doc.name).stem
Path(output_path).with_suffix(".md").write_bytes(md_cleaned.encode('utf-8'))
def pdfs_to_markdowns(path_pattern, overwrite: bool = False):
output_dir = Path(config.MARKDOWN_DIR)
output_dir.mkdir(parents=True, exist_ok=True)
for pdf_path in map(Path, glob.glob(path_pattern)):
md_path = (output_dir / pdf_path.stem).with_suffix(".md")
if overwrite or not md_path.exists():
pdf_to_markdown(pdf_path, output_dir)
@lru_cache(maxsize=1)
def _get_token_encoding():
try:
return tiktoken.encoding_for_model("gpt-4")
except Exception:
try:
return tiktoken.get_encoding("cl100k_base")
except Exception:
return None
def estimate_context_tokens(messages: list) -> int:
contents = [
str(msg.content)
for msg in messages
if hasattr(msg, "content") and msg.content
]
encoding = _get_token_encoding()
if encoding is None:
return sum(max(1, len(content) // 4) for content in contents)
return sum(len(encoding.encode(content)) for content in contents)