TC-Brain-26 / extract.py
aaliyahhasna's picture
Update extract.py
045df0f verified
Raw History Blame Contribute Delete
7.57 kB
"""
extract.py β€” pull plain text out of an uploaded file
====================================================================
WHAT THIS FILE IS: a small helper that reads an uploaded document and
returns its text, so the admin form can fill body_text automatically
instead of you copy-pasting.
WHAT IT HANDLES:
β€’ PDF β€” text-based PDFs (fast, free). If a PDF turns out to be
scanned (an image of the page, no selectable text), it falls back
to OCR using Claude vision β€” reads the page images and transcribes.
β€’ Word β€” .docx files
β€’ Plain text, Markdown and CSV β€” read as they are
β€’ PowerPoint / images β€” not supported yet; returns a clear message.
HOW OCR WORKS HERE: scanned PDF β†’ each page rendered to an image
(pdf2image + poppler) β†’ images sent to Claude via the existing
Anthropic key β†’ Claude transcribes the text. No new vendor. Costs
Anthropic tokens per page, so OCR only runs when normal extraction
fails, never by default.
FOR WHOEVER TAKES OVER: OCR needs the poppler-utils system package
(see packages.txt) and the ANTHROPIC_API_KEY secret. If OCR is ever
too slow/expensive, the MAX_OCR_PAGES cap below limits how many pages
get sent to Claude per document.
====================================================================
"""
import io
import os
import base64
# Safety cap: don't OCR more than this many pages in one go (cost/time).
MAX_OCR_PAGES = 15
def _clean(text: str) -> str:
"""
Remove null bytes and other control characters that Postgres can't
store (they cause 'unsupported Unicode escape sequence' errors).
Keeps normal whitespace (newlines, tabs).
"""
if not text:
return text
# Drop null bytes outright, then strip other non-printable control chars
# except newline (\n), carriage return (\r), and tab (\t).
text = text.replace("\x00", "")
return "".join(
ch for ch in text
if ch in "\n\r\t" or ord(ch) >= 32
)
def extract_text(uploaded_file) -> tuple[str, str]:
"""
Takes a Streamlit uploaded file. Returns (text, note).
text β€” the extracted plain text (may be empty)
note β€” a short status message to show the admin (empty if all fine)
"""
name = (uploaded_file.name or "").lower()
data = uploaded_file.read()
if name.endswith(".pdf"):
text, note = _extract_pdf(data)
elif name.endswith(".docx"):
text, note = _extract_docx(data)
elif name.endswith((".txt", ".md", ".markdown", ".csv")):
text, note = data.decode("utf-8", errors="replace"), ""
elif name.endswith((".ppt", ".pptx")):
return "", "PowerPoint isn't supported yet β€” please paste the text manually."
elif name.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp")):
text, note = _ocr_images([data], [name]) # single image β†’ OCR
else:
return "", f"Unsupported file type: {name}. Please paste the text manually."
return _clean(text), note
def _extract_pdf(data: bytes) -> tuple[str, str]:
try:
from pypdf import PdfReader
except Exception: # noqa: BLE001
return "", "PDF reader isn't installed. Check requirements.txt includes pypdf."
try:
reader = PdfReader(io.BytesIO(data))
parts = []
for page in reader.pages:
parts.append(page.extract_text() or "")
text = "\n\n".join(p for p in parts if p.strip()).strip()
except Exception as e: # noqa: BLE001
return "", f"Couldn't read that PDF: {e}"
# If normal extraction produced real text, we're done β€” cheap path.
# Threshold is deliberately low: even a short real document should
# skip OCR. Scanned PDFs typically extract zero or a few stray chars.
if len(text) >= 20:
return text, ""
# Otherwise it's almost certainly a scanned PDF β†’ fall back to OCR.
ocr_text, ocr_note = _ocr_pdf(data)
if ocr_text:
return ocr_text, ocr_note or "Read via OCR (scanned PDF)."
return text, (
ocr_note
or "This looks scanned and OCR couldn't read it β€” please paste the text manually."
)
def _extract_docx(data: bytes) -> tuple[str, str]:
try:
import docx # python-docx
except Exception: # noqa: BLE001
return "", "Word reader isn't installed. Check requirements.txt includes python-docx."
try:
doc = docx.Document(io.BytesIO(data))
text = "\n\n".join(p.text for p in doc.paragraphs if p.text.strip()).strip()
except Exception as e: # noqa: BLE001
return "", f"Couldn't read that Word file: {e}"
if len(text) < 50:
return text, "Very little text came out of that file β€” double-check it, or paste manually."
return text, ""
def _ocr_pdf(data: bytes) -> tuple[str, str]:
"""Render each PDF page to an image, then OCR the images with Claude."""
try:
from pdf2image import convert_from_bytes
except Exception: # noqa: BLE001
return "", "OCR needs pdf2image + poppler-utils (see requirements.txt and packages.txt)."
try:
images = convert_from_bytes(data, dpi=150)
except Exception as e: # noqa: BLE001
return "", f"Couldn't render the PDF for OCR: {e}"
if not images:
return "", "No pages found to OCR."
capped_note = ""
if len(images) > MAX_OCR_PAGES:
images = images[:MAX_OCR_PAGES]
capped_note = f" (only the first {MAX_OCR_PAGES} pages were read)"
# Convert each PIL image to PNG bytes.
page_bytes, names = [], []
for i, img in enumerate(images):
buf = io.BytesIO()
img.save(buf, format="PNG")
page_bytes.append(buf.getvalue())
names.append(f"page-{i+1}.png")
text, note = _ocr_images(page_bytes, names)
return text, (note + capped_note).strip()
def _ocr_images(images: list[bytes], names: list[str]) -> tuple[str, str]:
"""Send page images to Claude and ask it to transcribe the text."""
api_key = os.environ.get("ANTHROPIC_API_KEY")
if not api_key:
return "", "OCR needs the ANTHROPIC_API_KEY secret to be set."
try:
from anthropic import Anthropic
except Exception: # noqa: BLE001
return "", "Anthropic library isn't installed."
client = Anthropic(api_key=api_key)
# Build one message containing all page images plus a transcription ask.
content = []
for img in images:
content.append(
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": base64.b64encode(img).decode("utf-8"),
},
}
)
content.append(
{
"type": "text",
"text": (
"Transcribe all the readable text from these document pages, "
"in reading order. Return only the transcribed text β€” no "
"commentary, no summary. Preserve paragraph breaks. If a page "
"is blank or unreadable, skip it."
),
}
)
try:
msg = client.messages.create(
model="claude-sonnet-4-5",
max_tokens=4000,
messages=[{"role": "user", "content": content}],
)
text = "".join(b.text for b in msg.content if b.type == "text").strip()
except Exception as e: # noqa: BLE001
return "", f"OCR via Claude failed: {e}"
if len(text) < 20:
return text, "OCR returned very little β€” the scan may be too low quality."
return text, ""