Buckets:

download
raw
1.87 kB
#!/usr/bin/env python3
import os
from datetime import datetime
from pypdf import PdfReader
import resource
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, ".."))
PDF_ROOT = os.path.join(PROJECT_ROOT, "additional_llm_data")
def log(msg):
print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}")
def mem(tag=""):
usage = resource.getrusage(resource.RUSAGE_SELF)
if tag:
log(f"[MEM:{tag}] ru_maxrss={usage.ru_maxrss}")
else:
log(f"[MEM] ru_maxrss={usage.ru_maxrss}")
def find_first_pdf(root):
for dirpath, _, filenames in os.walk(root):
for name in filenames:
if name.lower().endswith(".pdf"):
return os.path.join(dirpath, name)
return None
log("Starting test_pdf_chunk.py")
log(f"CWD: {os.getcwd()}")
log(f"Project root:{PROJECT_ROOT}")
log(f"PDF root: {PDF_ROOT}")
mem("startup")
pdf_path = find_first_pdf(PDF_ROOT)
if not pdf_path:
log("No PDFs found under additional_llm_data/. Exiting.")
exit(0)
log(f"Using first PDF found: {pdf_path}")
mem("before_open")
reader = PdfReader(pdf_path)
text = "\n".join((page.extract_text() or "") for page in reader.pages)
log(f"Extracted {len(text)} chars")
mem("after_read")
MAX_CHARS = 1200
OVERLAP_CHARS = 200
chunks = []
start = 0
n = len(text)
log("Beginning chunk loop")
mem("before_loop")
MAX_CHARS = 1200
OVERLAP_CHARS = 200
chunks = []
start = 0
n = len(text)
log("Beginning chunk loop")
mem("before_loop")
while start < n:
end = min(n, start + MAX_CHARS)
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
if end >= n:
break
next_start = end - OVERLAP_CHARS
if next_start <= start:
next_start = start + 1
start = next_start
log(f"Finished chunking, got {len(chunks)} chunks")
mem("after_loop")

Xet Storage Details

Size:
1.87 kB
·
Xet hash:
723b546f25a7e2017f9422f16479c75c64b5c78b14080c19ea838d9a6ef917c3

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.