""" extract.py — pull plain text out of an uploaded file ==================================================================== WHAT THIS FILE IS: a small helper that reads an uploaded document and returns its text, so the admin form can fill body_text automatically instead of you copy-pasting. WHAT IT HANDLES: • PDF — text-based PDFs (fast, free). If a PDF turns out to be scanned (an image of the page, no selectable text), it falls back to OCR using Claude vision — reads the page images and transcribes. • Word — .docx files • Plain text, Markdown and CSV — read as they are • PowerPoint / images — not supported yet; returns a clear message. HOW OCR WORKS HERE: scanned PDF → each page rendered to an image (pdf2image + poppler) → images sent to Claude via the existing Anthropic key → Claude transcribes the text. No new vendor. Costs Anthropic tokens per page, so OCR only runs when normal extraction fails, never by default. FOR WHOEVER TAKES OVER: OCR needs the poppler-utils system package (see packages.txt) and the ANTHROPIC_API_KEY secret. If OCR is ever too slow/expensive, the MAX_OCR_PAGES cap below limits how many pages get sent to Claude per document. ==================================================================== """ import io import os import base64 # Safety cap: don't OCR more than this many pages in one go (cost/time). MAX_OCR_PAGES = 15 def _clean(text: str) -> str: """ Remove null bytes and other control characters that Postgres can't store (they cause 'unsupported Unicode escape sequence' errors). Keeps normal whitespace (newlines, tabs). """ if not text: return text # Drop null bytes outright, then strip other non-printable control chars # except newline (\n), carriage return (\r), and tab (\t). text = text.replace("\x00", "") return "".join( ch for ch in text if ch in "\n\r\t" or ord(ch) >= 32 ) def extract_text(uploaded_file) -> tuple[str, str]: """ Takes a Streamlit uploaded file. Returns (text, note). text — the extracted plain text (may be empty) note — a short status message to show the admin (empty if all fine) """ name = (uploaded_file.name or "").lower() data = uploaded_file.read() if name.endswith(".pdf"): text, note = _extract_pdf(data) elif name.endswith(".docx"): text, note = _extract_docx(data) elif name.endswith((".txt", ".md", ".markdown", ".csv")): text, note = data.decode("utf-8", errors="replace"), "" elif name.endswith((".ppt", ".pptx")): return "", "PowerPoint isn't supported yet — please paste the text manually." elif name.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp")): text, note = _ocr_images([data], [name]) # single image → OCR else: return "", f"Unsupported file type: {name}. Please paste the text manually." return _clean(text), note def _extract_pdf(data: bytes) -> tuple[str, str]: try: from pypdf import PdfReader except Exception: # noqa: BLE001 return "", "PDF reader isn't installed. Check requirements.txt includes pypdf." try: reader = PdfReader(io.BytesIO(data)) parts = [] for page in reader.pages: parts.append(page.extract_text() or "") text = "\n\n".join(p for p in parts if p.strip()).strip() except Exception as e: # noqa: BLE001 return "", f"Couldn't read that PDF: {e}" # If normal extraction produced real text, we're done — cheap path. # Threshold is deliberately low: even a short real document should # skip OCR. Scanned PDFs typically extract zero or a few stray chars. if len(text) >= 20: return text, "" # Otherwise it's almost certainly a scanned PDF → fall back to OCR. ocr_text, ocr_note = _ocr_pdf(data) if ocr_text: return ocr_text, ocr_note or "Read via OCR (scanned PDF)." return text, ( ocr_note or "This looks scanned and OCR couldn't read it — please paste the text manually." ) def _extract_docx(data: bytes) -> tuple[str, str]: try: import docx # python-docx except Exception: # noqa: BLE001 return "", "Word reader isn't installed. Check requirements.txt includes python-docx." try: doc = docx.Document(io.BytesIO(data)) text = "\n\n".join(p.text for p in doc.paragraphs if p.text.strip()).strip() except Exception as e: # noqa: BLE001 return "", f"Couldn't read that Word file: {e}" if len(text) < 50: return text, "Very little text came out of that file — double-check it, or paste manually." return text, "" def _ocr_pdf(data: bytes) -> tuple[str, str]: """Render each PDF page to an image, then OCR the images with Claude.""" try: from pdf2image import convert_from_bytes except Exception: # noqa: BLE001 return "", "OCR needs pdf2image + poppler-utils (see requirements.txt and packages.txt)." try: images = convert_from_bytes(data, dpi=150) except Exception as e: # noqa: BLE001 return "", f"Couldn't render the PDF for OCR: {e}" if not images: return "", "No pages found to OCR." capped_note = "" if len(images) > MAX_OCR_PAGES: images = images[:MAX_OCR_PAGES] capped_note = f" (only the first {MAX_OCR_PAGES} pages were read)" # Convert each PIL image to PNG bytes. page_bytes, names = [], [] for i, img in enumerate(images): buf = io.BytesIO() img.save(buf, format="PNG") page_bytes.append(buf.getvalue()) names.append(f"page-{i+1}.png") text, note = _ocr_images(page_bytes, names) return text, (note + capped_note).strip() def _ocr_images(images: list[bytes], names: list[str]) -> tuple[str, str]: """Send page images to Claude and ask it to transcribe the text.""" api_key = os.environ.get("ANTHROPIC_API_KEY") if not api_key: return "", "OCR needs the ANTHROPIC_API_KEY secret to be set." try: from anthropic import Anthropic except Exception: # noqa: BLE001 return "", "Anthropic library isn't installed." client = Anthropic(api_key=api_key) # Build one message containing all page images plus a transcription ask. content = [] for img in images: content.append( { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": base64.b64encode(img).decode("utf-8"), }, } ) content.append( { "type": "text", "text": ( "Transcribe all the readable text from these document pages, " "in reading order. Return only the transcribed text — no " "commentary, no summary. Preserve paragraph breaks. If a page " "is blank or unreadable, skip it." ), } ) try: msg = client.messages.create( model="claude-sonnet-4-5", max_tokens=4000, messages=[{"role": "user", "content": content}], ) text = "".join(b.text for b in msg.content if b.type == "text").strip() except Exception as e: # noqa: BLE001 return "", f"OCR via Claude failed: {e}" if len(text) < 20: return text, "OCR returned very little — the scan may be too low quality." return text, ""