File size: 1,483 Bytes
7aab86a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | import hashlib
from pathlib import Path
from .config import MAX_PDF_BYTES
from .schema import Document, Page
def read_document(name, data, models):
if Path(name).suffix.lower() != '.pdf':
raise ValueError('Upload a PDF. Save other document formats as PDF first.')
if not data or len(data) > MAX_PDF_BYTES:
raise ValueError('Upload a non-empty PDF smaller than 40 MB.')
if b'%PDF-' not in data[:1024]:
raise ValueError('This file does not appear to be a PDF.')
extracted = models.extract_pdf(name, data)
pages = [Page(number=p.number, original=p.original, english=p.english,
language=p.language, method='Gemini PDF extraction', reviewed=False,
warnings=p.warnings) for p in extracted]
if not pages or not any(p.english.strip() for p in pages):
raise ValueError('No readable text was found. Upload a clearer, unlocked PDF.')
return Document(id=hashlib.sha256(data + b'.pdf').hexdigest()[:12],
name=Path(name).name, pages=pages)
def split_text(text, limit=1000):
"""Retained for Sarvam translation; never discards characters."""
while text:
end = min(len(text), limit)
if end < len(text):
cut = max(text.rfind('\n', limit // 2, end), text.rfind('. ', limit // 2, end),
text.rfind(' ', limit // 2, end))
if cut > 0:
end = cut + 1
yield text[:end]
text = text[end:]
|