Spaces:
Sleeping
Sleeping
File size: 2,068 Bytes
b3b0ccd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | import os
import shutil
import config
import pymupdf.layout
import pymupdf4llm
from pathlib import Path
import glob
import tiktoken
from functools import lru_cache
def clear_directory_contents(directory: Path) -> None:
"""Delete everything under directory but not the directory itself (safe for Docker volume / bind mount roots)."""
directory = Path(directory)
if not directory.is_dir():
return
for child in directory.iterdir():
if child.is_dir():
shutil.rmtree(child)
else:
child.unlink()
os.environ["TOKENIZERS_PARALLELISM"] = "false"
def pdf_to_markdown(pdf_path, output_dir):
doc = pymupdf.open(pdf_path)
md = pymupdf4llm.to_markdown(doc, header=False, footer=False, page_separators=True, ignore_images=True, write_images=False, image_path=None)
md_cleaned = md.encode('utf-8', errors='surrogatepass').decode('utf-8', errors='ignore')
output_path = Path(output_dir) / Path(doc.name).stem
Path(output_path).with_suffix(".md").write_bytes(md_cleaned.encode('utf-8'))
def pdfs_to_markdowns(path_pattern, overwrite: bool = False):
output_dir = Path(config.MARKDOWN_DIR)
output_dir.mkdir(parents=True, exist_ok=True)
for pdf_path in map(Path, glob.glob(path_pattern)):
md_path = (output_dir / pdf_path.stem).with_suffix(".md")
if overwrite or not md_path.exists():
pdf_to_markdown(pdf_path, output_dir)
@lru_cache(maxsize=1)
def _get_token_encoding():
try:
return tiktoken.encoding_for_model("gpt-4")
except Exception:
try:
return tiktoken.get_encoding("cl100k_base")
except Exception:
return None
def estimate_context_tokens(messages: list) -> int:
contents = [
str(msg.content)
for msg in messages
if hasattr(msg, "content") and msg.content
]
encoding = _get_token_encoding()
if encoding is None:
return sum(max(1, len(content) // 4) for content in contents)
return sum(len(encoding.encode(content)) for content in contents)
|