Spaces:
Sleeping
Sleeping
Download project/utils.py from devank2000/agentic-rag: direct link, hf CLI and curl.
- Browser
- Download file 2.07 kB
-
https://huggingface.co/spaces/devank2000/agentic-rag/resolve/main/project/utils.py
- Command line
-
hf download hf://spaces/devank2000/agentic-rag/project/utils.py
-
curl -L -o utils.py https://huggingface.co/spaces/devank2000/agentic-rag/resolve/main/project/utils.py
2.07 kB
| import os | |
| import shutil | |
| import config | |
| import pymupdf.layout | |
| import pymupdf4llm | |
| from pathlib import Path | |
| import glob | |
| import tiktoken | |
| from functools import lru_cache | |
| def clear_directory_contents(directory: Path) -> None: | |
| """Delete everything under directory but not the directory itself (safe for Docker volume / bind mount roots).""" | |
| directory = Path(directory) | |
| if not directory.is_dir(): | |
| return | |
| for child in directory.iterdir(): | |
| if child.is_dir(): | |
| shutil.rmtree(child) | |
| else: | |
| child.unlink() | |
| os.environ["TOKENIZERS_PARALLELISM"] = "false" | |
| def pdf_to_markdown(pdf_path, output_dir): | |
| doc = pymupdf.open(pdf_path) | |
| md = pymupdf4llm.to_markdown(doc, header=False, footer=False, page_separators=True, ignore_images=True, write_images=False, image_path=None) | |
| md_cleaned = md.encode('utf-8', errors='surrogatepass').decode('utf-8', errors='ignore') | |
| output_path = Path(output_dir) / Path(doc.name).stem | |
| Path(output_path).with_suffix(".md").write_bytes(md_cleaned.encode('utf-8')) | |
| def pdfs_to_markdowns(path_pattern, overwrite: bool = False): | |
| output_dir = Path(config.MARKDOWN_DIR) | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| for pdf_path in map(Path, glob.glob(path_pattern)): | |
| md_path = (output_dir / pdf_path.stem).with_suffix(".md") | |
| if overwrite or not md_path.exists(): | |
| pdf_to_markdown(pdf_path, output_dir) | |
| def _get_token_encoding(): | |
| try: | |
| return tiktoken.encoding_for_model("gpt-4") | |
| except Exception: | |
| try: | |
| return tiktoken.get_encoding("cl100k_base") | |
| except Exception: | |
| return None | |
| def estimate_context_tokens(messages: list) -> int: | |
| contents = [ | |
| str(msg.content) | |
| for msg in messages | |
| if hasattr(msg, "content") and msg.content | |
| ] | |
| encoding = _get_token_encoding() | |
| if encoding is None: | |
| return sum(max(1, len(content) // 4) for content in contents) | |
| return sum(len(encoding.encode(content)) for content in contents) | |