Spaces:
Sleeping
Sleeping
Download extract.py from ThinkCity/TC-Brain-26: direct link, hf CLI and curl.
- Browser
- Download file 7.57 kB
-
https://huggingface.co/spaces/ThinkCity/TC-Brain-26/resolve/main/extract.py
- Command line
-
hf download hf://spaces/ThinkCity/TC-Brain-26/extract.py
-
curl -L -o extract.py https://huggingface.co/spaces/ThinkCity/TC-Brain-26/resolve/main/extract.py
7.57 kB
| """ | |
| extract.py β pull plain text out of an uploaded file | |
| ==================================================================== | |
| WHAT THIS FILE IS: a small helper that reads an uploaded document and | |
| returns its text, so the admin form can fill body_text automatically | |
| instead of you copy-pasting. | |
| WHAT IT HANDLES: | |
| β’ PDF β text-based PDFs (fast, free). If a PDF turns out to be | |
| scanned (an image of the page, no selectable text), it falls back | |
| to OCR using Claude vision β reads the page images and transcribes. | |
| β’ Word β .docx files | |
| β’ Plain text, Markdown and CSV β read as they are | |
| β’ PowerPoint / images β not supported yet; returns a clear message. | |
| HOW OCR WORKS HERE: scanned PDF β each page rendered to an image | |
| (pdf2image + poppler) β images sent to Claude via the existing | |
| Anthropic key β Claude transcribes the text. No new vendor. Costs | |
| Anthropic tokens per page, so OCR only runs when normal extraction | |
| fails, never by default. | |
| FOR WHOEVER TAKES OVER: OCR needs the poppler-utils system package | |
| (see packages.txt) and the ANTHROPIC_API_KEY secret. If OCR is ever | |
| too slow/expensive, the MAX_OCR_PAGES cap below limits how many pages | |
| get sent to Claude per document. | |
| ==================================================================== | |
| """ | |
| import io | |
| import os | |
| import base64 | |
| # Safety cap: don't OCR more than this many pages in one go (cost/time). | |
| MAX_OCR_PAGES = 15 | |
| def _clean(text: str) -> str: | |
| """ | |
| Remove null bytes and other control characters that Postgres can't | |
| store (they cause 'unsupported Unicode escape sequence' errors). | |
| Keeps normal whitespace (newlines, tabs). | |
| """ | |
| if not text: | |
| return text | |
| # Drop null bytes outright, then strip other non-printable control chars | |
| # except newline (\n), carriage return (\r), and tab (\t). | |
| text = text.replace("\x00", "") | |
| return "".join( | |
| ch for ch in text | |
| if ch in "\n\r\t" or ord(ch) >= 32 | |
| ) | |
| def extract_text(uploaded_file) -> tuple[str, str]: | |
| """ | |
| Takes a Streamlit uploaded file. Returns (text, note). | |
| text β the extracted plain text (may be empty) | |
| note β a short status message to show the admin (empty if all fine) | |
| """ | |
| name = (uploaded_file.name or "").lower() | |
| data = uploaded_file.read() | |
| if name.endswith(".pdf"): | |
| text, note = _extract_pdf(data) | |
| elif name.endswith(".docx"): | |
| text, note = _extract_docx(data) | |
| elif name.endswith((".txt", ".md", ".markdown", ".csv")): | |
| text, note = data.decode("utf-8", errors="replace"), "" | |
| elif name.endswith((".ppt", ".pptx")): | |
| return "", "PowerPoint isn't supported yet β please paste the text manually." | |
| elif name.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp")): | |
| text, note = _ocr_images([data], [name]) # single image β OCR | |
| else: | |
| return "", f"Unsupported file type: {name}. Please paste the text manually." | |
| return _clean(text), note | |
| def _extract_pdf(data: bytes) -> tuple[str, str]: | |
| try: | |
| from pypdf import PdfReader | |
| except Exception: # noqa: BLE001 | |
| return "", "PDF reader isn't installed. Check requirements.txt includes pypdf." | |
| try: | |
| reader = PdfReader(io.BytesIO(data)) | |
| parts = [] | |
| for page in reader.pages: | |
| parts.append(page.extract_text() or "") | |
| text = "\n\n".join(p for p in parts if p.strip()).strip() | |
| except Exception as e: # noqa: BLE001 | |
| return "", f"Couldn't read that PDF: {e}" | |
| # If normal extraction produced real text, we're done β cheap path. | |
| # Threshold is deliberately low: even a short real document should | |
| # skip OCR. Scanned PDFs typically extract zero or a few stray chars. | |
| if len(text) >= 20: | |
| return text, "" | |
| # Otherwise it's almost certainly a scanned PDF β fall back to OCR. | |
| ocr_text, ocr_note = _ocr_pdf(data) | |
| if ocr_text: | |
| return ocr_text, ocr_note or "Read via OCR (scanned PDF)." | |
| return text, ( | |
| ocr_note | |
| or "This looks scanned and OCR couldn't read it β please paste the text manually." | |
| ) | |
| def _extract_docx(data: bytes) -> tuple[str, str]: | |
| try: | |
| import docx # python-docx | |
| except Exception: # noqa: BLE001 | |
| return "", "Word reader isn't installed. Check requirements.txt includes python-docx." | |
| try: | |
| doc = docx.Document(io.BytesIO(data)) | |
| text = "\n\n".join(p.text for p in doc.paragraphs if p.text.strip()).strip() | |
| except Exception as e: # noqa: BLE001 | |
| return "", f"Couldn't read that Word file: {e}" | |
| if len(text) < 50: | |
| return text, "Very little text came out of that file β double-check it, or paste manually." | |
| return text, "" | |
| def _ocr_pdf(data: bytes) -> tuple[str, str]: | |
| """Render each PDF page to an image, then OCR the images with Claude.""" | |
| try: | |
| from pdf2image import convert_from_bytes | |
| except Exception: # noqa: BLE001 | |
| return "", "OCR needs pdf2image + poppler-utils (see requirements.txt and packages.txt)." | |
| try: | |
| images = convert_from_bytes(data, dpi=150) | |
| except Exception as e: # noqa: BLE001 | |
| return "", f"Couldn't render the PDF for OCR: {e}" | |
| if not images: | |
| return "", "No pages found to OCR." | |
| capped_note = "" | |
| if len(images) > MAX_OCR_PAGES: | |
| images = images[:MAX_OCR_PAGES] | |
| capped_note = f" (only the first {MAX_OCR_PAGES} pages were read)" | |
| # Convert each PIL image to PNG bytes. | |
| page_bytes, names = [], [] | |
| for i, img in enumerate(images): | |
| buf = io.BytesIO() | |
| img.save(buf, format="PNG") | |
| page_bytes.append(buf.getvalue()) | |
| names.append(f"page-{i+1}.png") | |
| text, note = _ocr_images(page_bytes, names) | |
| return text, (note + capped_note).strip() | |
| def _ocr_images(images: list[bytes], names: list[str]) -> tuple[str, str]: | |
| """Send page images to Claude and ask it to transcribe the text.""" | |
| api_key = os.environ.get("ANTHROPIC_API_KEY") | |
| if not api_key: | |
| return "", "OCR needs the ANTHROPIC_API_KEY secret to be set." | |
| try: | |
| from anthropic import Anthropic | |
| except Exception: # noqa: BLE001 | |
| return "", "Anthropic library isn't installed." | |
| client = Anthropic(api_key=api_key) | |
| # Build one message containing all page images plus a transcription ask. | |
| content = [] | |
| for img in images: | |
| content.append( | |
| { | |
| "type": "image", | |
| "source": { | |
| "type": "base64", | |
| "media_type": "image/png", | |
| "data": base64.b64encode(img).decode("utf-8"), | |
| }, | |
| } | |
| ) | |
| content.append( | |
| { | |
| "type": "text", | |
| "text": ( | |
| "Transcribe all the readable text from these document pages, " | |
| "in reading order. Return only the transcribed text β no " | |
| "commentary, no summary. Preserve paragraph breaks. If a page " | |
| "is blank or unreadable, skip it." | |
| ), | |
| } | |
| ) | |
| try: | |
| msg = client.messages.create( | |
| model="claude-sonnet-4-5", | |
| max_tokens=4000, | |
| messages=[{"role": "user", "content": content}], | |
| ) | |
| text = "".join(b.text for b in msg.content if b.type == "text").strip() | |
| except Exception as e: # noqa: BLE001 | |
| return "", f"OCR via Claude failed: {e}" | |
| if len(text) < 20: | |
| return text, "OCR returned very little β the scan may be too low quality." | |
| return text, "" |