Spaces:
Paused
Paused
Malina
Claude Fable 5.1
Scoped tokens, bounded archives and frames, build deadline, circuit breaker, honest note caps
da9de03 Download extract.py from NextNeural/LUNA: direct link, hf CLI and curl.
- Browser
- Download file 19.1 kB
-
https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/extract.py
- Command line
-
hf download hf://spaces/NextNeural/LUNA/extract.py
-
curl -L -o extract.py https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/extract.py
19.1 kB
| # File categorization + content extraction. Each file resolves to one of: | |
| # kind="text" → payload is extracted plain text (docs, sheets, code, data, html) | |
| # kind="image" → payload is a base64 data URL (read later by the VLM) | |
| # kind="media" → payload is a filesystem path (transcribed later by Whisper) | |
| # kind="error" → payload is a short human reason it couldn't be read | |
| # Everything here is best-effort and never raises: a single bad file shouldn't sink | |
| # the whole task. | |
| import base64 | |
| import io | |
| import os | |
| import re | |
| import shutil | |
| import tempfile | |
| import zipfile | |
| from config import (EXTRACT_MAX_CHARS, PDF_MAX_PAGES, PDF_SCAN_MAX_PAGES, | |
| ZIP_MAX_MEMBERS, ZIP_MEMBER_MAX_BYTES, ZIP_TOTAL_MAX_BYTES) | |
| # Formats LUNA has no parser for. They used to fall through to a raw text read, which | |
| # for a .doc or .xls is a binary blob: the file was reported as "binary, not readable" | |
| # and the user was left to guess why their Word document was unreadable. | |
| UNSUPPORTED = { | |
| "doc": "an old Word file (.doc): save it as .docx or PDF", | |
| "ppt": "an old PowerPoint file (.ppt): save it as .pptx or PDF", | |
| "rtf": "a Rich Text file (.rtf): save it as .docx, .txt or PDF", | |
| "xls": "an old Excel file (.xls): save it as .xlsx or .csv", | |
| "ods": "an OpenDocument spreadsheet (.ods): save it as .xlsx or .csv", | |
| } | |
| # ── Extension → category map (category is also what the UI shows) ────────────── | |
| _EXT = { | |
| # documents | |
| "pdf": "document", "docx": "document", "doc": "document", "pptx": "document", | |
| "ppt": "document", "rtf": "document", "odt": "document", "epub": "document", | |
| "html": "document", "htm": "document", | |
| # spreadsheets | |
| "xlsx": "spreadsheet", "xls": "spreadsheet", "ods": "spreadsheet", | |
| # plain text / data | |
| "txt": "text", "text": "text", "log": "text", "md": "text", "markdown": "text", | |
| "csv": "data", "tsv": "data", "json": "data", "yaml": "data", "yml": "data", | |
| "xml": "data", "ini": "data", "toml": "data", | |
| # code | |
| "py": "code", "js": "code", "ts": "code", "jsx": "code", "tsx": "code", | |
| "java": "code", "c": "code", "h": "code", "cpp": "code", "cc": "code", | |
| "cs": "code", "go": "code", "rs": "code", "rb": "code", "php": "code", | |
| "swift": "code", "kt": "code", "sh": "code", "bash": "code", "sql": "code", | |
| "css": "code", "scss": "code", "r": "code", "m": "code", "lua": "code", | |
| # images | |
| "png": "image", "jpg": "image", "jpeg": "image", "gif": "image", | |
| "webp": "image", "bmp": "image", "tif": "image", "tiff": "image", | |
| # audio / video | |
| "mp3": "audio", "wav": "audio", "m4a": "audio", "flac": "audio", | |
| "ogg": "audio", "aac": "audio", "opus": "audio", "wma": "audio", | |
| "mp4": "video", "mov": "video", "mkv": "video", "webm": "video", | |
| "avi": "video", "m4v": "video", "wmv": "video", "flv": "video", | |
| # vector graphics | |
| "svg": "vector", | |
| # archives | |
| "zip": "archive", | |
| } | |
| _TEXTLIKE = {"text", "data", "code"} | |
| def category_of(name: str) -> str: | |
| ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" | |
| return _EXT.get(ext, "other") | |
| def _truncate(s: str) -> str: | |
| """Cap one file's text at what a build may read of it. | |
| This used to cap at PER_FILE_CHARS, the size of ONE model call, so a long document | |
| existed only as far as its first 16000 characters: a 143-page report (420968 | |
| characters) was read to page 7 and no chapter of it ever reached a model. The cap | |
| is now the whole-file budget; pipeline splits what is over one call's worth into | |
| parts.""" | |
| s = (s or "").strip() | |
| return s[:EXTRACT_MAX_CHARS] if len(s) > EXTRACT_MAX_CHARS else s | |
| class NotText(Exception): | |
| """Raised when a file is binary, so the caller can say so instead of guessing.""" | |
| def _looks_binary(raw: bytes) -> bool: | |
| """A NUL byte, or a lot of bytes no text encoding produces, means this is not text. | |
| Without this check an unknown extension was read as UTF-8 with errors="replace" and | |
| the resulting mojibake was sent to the model as if it were content: tokens paid for | |
| noise, and a real risk the model reads meaning into it. Measured on a test file of | |
| raw bytes: 1192 of the first 2000 characters were replacement or control codes.""" | |
| if not raw: | |
| return False | |
| if b"\x00" in raw: | |
| return True | |
| odd = sum(1 for b in raw if b < 9 or 13 < b < 32 or b == 127) | |
| return odd / len(raw) > 0.06 | |
| _B64_BLOB = re.compile(r"[A-Za-z0-9+/]{200,}={0,2}") | |
| _WORDISH = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿĄ-ſ]{3,}") | |
| def _word_density(s: str) -> float: | |
| """Share of the text made of word-like runs. Prose sits above 0.6; a column of | |
| numbers sits near 0.""" | |
| if not s: | |
| return 0.0 | |
| return sum(len(w) for w in _WORDISH.findall(s)) / len(s) | |
| def _read_svg(path: str) -> str: | |
| """An SVG is text, so the plain reader shipped the whole thing: 16000 characters | |
| of path data and base64 for one logo in the test corpus. Tokens were paid for | |
| noise, and a stray letter run inside a base64 blob even scored as a keyword hit. | |
| Only the parts a person would actually read are kept.""" | |
| import xml.etree.ElementTree as ET | |
| with open(path, "r", encoding="utf-8", errors="replace") as f: | |
| raw = _B64_BLOB.sub(" ", f.read(4_000_000)) | |
| seen, size = [], "? x ?" | |
| try: | |
| root = ET.fromstring(raw) | |
| for el in root.iter(): | |
| if el.tag.rsplit("}", 1)[-1].lower() in ("title", "desc", "text", "tspan"): | |
| t = (el.text or "").strip() | |
| if t and t not in seen: | |
| seen.append(t) | |
| size = f"{root.get('width') or '?'} x {root.get('height') or '?'}" | |
| except Exception: | |
| # Exported SVGs are not always well formed - an undeclared namespace prefix | |
| # is enough to stop the parser dead. Falling back to a scan keeps the words | |
| # rather than reporting the whole file as unreadable. | |
| for m in re.finditer(r"<(?:title|desc|text|tspan)\b[^>]*>(.*?)</", raw, | |
| re.S | re.I): | |
| t = re.sub(r"<[^>]*>", " ", m.group(1)).strip() | |
| if t and t not in seen: | |
| seen.append(t) | |
| d = re.search(r'\bwidth\s*=\s*"([^"]+)"', raw) | |
| h = re.search(r'\bheight\s*=\s*"([^"]+)"', raw) | |
| if d or h: | |
| size = f"{d.group(1) if d else '?'} x {h.group(1) if h else '?'}" | |
| body = " ".join(seen).strip() | |
| return (f"SVG vector graphic, {size}. Text it contains: {body}" if body | |
| else f"SVG vector graphic, {size}. It contains no readable text.") | |
| def _read_textfile(path: str) -> str: | |
| with open(path, "rb") as f: | |
| head = f.read(4096) | |
| if _looks_binary(head): | |
| raise NotText() | |
| with open(path, "r", encoding="utf-8", errors="replace") as f: | |
| return f.read(EXTRACT_MAX_CHARS + 1) | |
| def _read_pdf(path: str) -> str: | |
| from pypdf import PdfReader | |
| reader = PdfReader(path) | |
| n_pages = len(reader.pages) | |
| out, total = [], 0 | |
| for i, page in enumerate(reader.pages): | |
| if i >= PDF_MAX_PAGES: | |
| out.append(f"[Only the first {PDF_MAX_PAGES} of {n_pages} pages were read.]") | |
| break | |
| t = page.extract_text() or "" | |
| out.append(t) | |
| total += len(t) | |
| if total > EXTRACT_MAX_CHARS: | |
| break | |
| return "\n".join(out) | |
| def _read_docx(path: str) -> str: | |
| import docx | |
| d = docx.Document(path) | |
| parts = [p.text for p in d.paragraphs if p.text.strip()] | |
| for tbl in d.tables: | |
| for row in tbl.rows: | |
| cells = [c.text.strip() for c in row.cells] | |
| if any(cells): | |
| parts.append(" | ".join(cells)) | |
| return "\n".join(parts) | |
| def _read_pptx(path: str) -> str: | |
| from pptx import Presentation | |
| prs = Presentation(path) | |
| parts = [] | |
| for i, slide in enumerate(prs.slides, 1): | |
| parts.append(f"[Slide {i}]") | |
| for shape in slide.shapes: | |
| if shape.has_text_frame and shape.text_frame.text.strip(): | |
| parts.append(shape.text_frame.text) | |
| return "\n".join(parts) | |
| def _read_xlsx(path: str) -> str: | |
| from openpyxl import load_workbook | |
| wb = load_workbook(path, read_only=True, data_only=True) | |
| parts, total = [], 0 | |
| for ws in wb.worksheets: | |
| parts.append(f"[Sheet: {ws.title}]") | |
| for row in ws.iter_rows(values_only=True): | |
| cells = [str(c) for c in row if c is not None] | |
| if cells: | |
| line = " | ".join(cells) | |
| parts.append(line) | |
| total += len(line) | |
| if total > EXTRACT_MAX_CHARS: | |
| break | |
| if total > EXTRACT_MAX_CHARS: | |
| break | |
| wb.close() | |
| return "\n".join(parts) | |
| def _read_html(path: str) -> str: | |
| from bs4 import BeautifulSoup | |
| with open(path, "r", encoding="utf-8", errors="replace") as f: | |
| soup = BeautifulSoup(f.read(), "html.parser") | |
| for tag in soup(["script", "style", "noscript"]): | |
| tag.decompose() | |
| return soup.get_text(separator="\n") | |
| _ZIP_BY_EXT = { | |
| "pdf": "_read_pdf", "docx": "_read_docx", "pptx": "_read_pptx", "xlsx": "_read_xlsx", | |
| "html": "_read_html", "htm": "_read_html", "odt": "_read_opendoc", "epub": "_read_opendoc", | |
| "svg": "_read_svg", | |
| } | |
| def _read_zip(path: str) -> str: | |
| """Members through the SAME readers a loose upload gets, within hard bounds. | |
| Before: every text-like or document member was read as UTF-8 regardless of what it | |
| was, so a .docx or .pdf inside an archive arrived as binary mojibake, a member's | |
| declared size was never checked (a 25 MB upload can declare gigabytes), and media | |
| inside were dropped without a word. Now binaries go through their extractors via a | |
| temporary file, sizes are capped per member and in total, and what was not read is | |
| listed so the Note can account for it.""" | |
| parts, total, read_bytes, listed = [], 0, 0, [] | |
| not_read: list[str] = [] | |
| tmpdir = tempfile.mkdtemp(prefix="luna_zip_") | |
| try: | |
| with zipfile.ZipFile(path) as z: | |
| infos = [i for i in z.infolist() if not i.is_dir() and i.file_size > 0] | |
| if len(infos) > ZIP_MAX_MEMBERS: | |
| not_read.append(f"{len(infos) - ZIP_MAX_MEMBERS} further members (limit {ZIP_MAX_MEMBERS})") | |
| infos = infos[:ZIP_MAX_MEMBERS] | |
| for info in infos: | |
| member = info.filename | |
| base = os.path.basename(member) | |
| if base.startswith("._") or "__MACOSX/" in member: | |
| continue | |
| cat = category_of(member) | |
| ext = member.rsplit(".", 1)[-1].lower() if "." in member else "" | |
| if info.file_size > ZIP_MEMBER_MAX_BYTES: | |
| not_read.append(f"{member} (too large)") | |
| continue | |
| if read_bytes + info.file_size > ZIP_TOTAL_MAX_BYTES: | |
| not_read.append(f"{member} (archive budget used up)") | |
| continue | |
| if cat in ("audio", "video", "image", "archive") or ext in UNSUPPORTED: | |
| not_read.append(f"{member} ({cat if cat != 'other' else ext})") | |
| continue | |
| if cat not in _TEXTLIKE and cat not in ("document", "spreadsheet", "vector", "other"): | |
| not_read.append(member) | |
| continue | |
| try: | |
| reader = _ZIP_BY_EXT.get(ext) | |
| if reader: | |
| # Real parser: needs a path. Extracted under a bounded read. | |
| out = os.path.join(tmpdir, f"m{len(listed)}.{ext}") | |
| with z.open(info) as src, open(out, "wb") as dst: | |
| shutil.copyfileobj(src, dst, 1 << 20) | |
| read_bytes += info.file_size | |
| text = globals()[reader](out) | |
| try: | |
| os.remove(out) | |
| except OSError: | |
| pass | |
| else: | |
| with z.open(info) as fh: | |
| raw = fh.read(min(info.file_size, EXTRACT_MAX_CHARS * 4)) | |
| read_bytes += len(raw) | |
| if _looks_binary(raw[:4096]): | |
| not_read.append(f"{member} (binary)") | |
| continue | |
| text = raw.decode("utf-8", errors="replace") | |
| except Exception as ex: | |
| not_read.append(f"{member} (couldn't read: {type(ex).__name__})") | |
| continue | |
| text = (text or "").strip() | |
| if not text: | |
| not_read.append(f"{member} (no readable text)") | |
| continue | |
| listed.append(member) | |
| parts.append(f"\n=== {member} ===\n{text}") | |
| total += len(text) | |
| if total > EXTRACT_MAX_CHARS: | |
| break | |
| finally: | |
| shutil.rmtree(tmpdir, ignore_errors=True) | |
| if not_read: | |
| parts.append("\n=== Members of this archive that were NOT read ===\n" | |
| + "\n".join(f"- {m}" for m in not_read[:50]) | |
| + (f"\n- and {len(not_read) - 50} more" if len(not_read) > 50 else "")) | |
| return "\n".join(parts) | |
| def _pdf_page_images(path: str) -> list: | |
| """Page images out of a PDF that has no text layer, i.e. a scan. | |
| Scanned PDFs used to come back as "no readable text found" and were set aside, which | |
| for an invoice or a signed contract is the whole content of the file. Most scans | |
| embed each page as one image object, so pypdf can hand it over directly and no | |
| rasterising dependency is needed.""" | |
| from pypdf import PdfReader | |
| out = [] | |
| reader = PdfReader(path) | |
| for i, page in enumerate(reader.pages[:PDF_SCAN_MAX_PAGES], 1): | |
| try: | |
| for im in page.images: | |
| out.append((i, _pil_data_url(im.image))) | |
| break # one image per page is the scan itself | |
| except Exception: | |
| continue | |
| return out | |
| def _read_opendoc(path: str) -> str: | |
| """.odt and .epub are ZIP containers of XML. Reading them as plain text produced | |
| markup and binary noise; unpacking the XML and stripping tags gives the real text.""" | |
| import re as _re | |
| parts, total = [], 0 | |
| with zipfile.ZipFile(path) as z: | |
| members = [m for m in z.namelist() | |
| if m.endswith((".xml", ".xhtml", ".html", ".htm")) | |
| and not m.startswith("META-INF/")] | |
| for m in members: | |
| try: | |
| raw = z.read(m).decode("utf-8", errors="replace") | |
| except Exception: | |
| continue | |
| txt = _re.sub(r"<[^>]+>", " ", raw) | |
| txt = _re.sub(r"\s+", " ", txt).strip() | |
| if len(txt) < 20: | |
| continue | |
| parts.append(txt) | |
| total += len(txt) | |
| if total > EXTRACT_MAX_CHARS: | |
| break | |
| return "\n".join(parts) | |
| def _pil_data_url(img) -> str: | |
| """One PIL image, downscaled and encoded the way the vision model wants it.""" | |
| from config import VLM_FRAME_MAX_PX | |
| img = img.convert("RGB") | |
| w, h = img.size | |
| scale = VLM_FRAME_MAX_PX / max(w, h) | |
| if scale < 1: | |
| img = img.resize((max(1, int(w * scale)), max(1, int(h * scale)))) | |
| buf = io.BytesIO() | |
| img.save(buf, format="JPEG", quality=85) | |
| return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode() | |
| def _image_data_url(path: str) -> str: | |
| from PIL import Image, ImageOps | |
| img = Image.open(path) | |
| # Phone photos carry their rotation in EXIF rather than in the pixels. Without this | |
| # the vision model was handed sideways images and read the text in them badly. | |
| img = ImageOps.exif_transpose(img) | |
| return _pil_data_url(img) | |
| # Dispatch by category → (kind, extractor) | |
| _TEXT_EXTRACTORS = { | |
| "document": None, # resolved below per-extension | |
| "spreadsheet": _read_xlsx, | |
| "text": _read_textfile, | |
| "data": _read_textfile, | |
| "code": _read_textfile, | |
| "archive": _read_zip, | |
| } | |
| def extract(path: str, name: str | None = None) -> dict: | |
| """Extract one file. Returns {name, category, kind, payload}.""" | |
| name = name or os.path.basename(path) | |
| cat = category_of(name) | |
| ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" | |
| try: | |
| if ext in UNSUPPORTED: | |
| return {"name": name, "category": cat, "kind": "error", | |
| "payload": f"unsupported format: {UNSUPPORTED[ext]}"} | |
| if cat == "image": | |
| return {"name": name, "category": cat, "kind": "image", | |
| "payload": _image_data_url(path)} | |
| if cat in ("audio", "video"): | |
| return {"name": name, "category": cat, "kind": "media", "payload": path} | |
| if cat == "document": | |
| if ext == "pdf": | |
| text = _read_pdf(path) | |
| if not text.strip(): | |
| pages = _pdf_page_images(path) | |
| if pages: | |
| return {"name": name, "category": cat, "kind": "pages", | |
| "payload": pages} | |
| elif ext in ("docx",): | |
| text = _read_docx(path) | |
| elif ext in ("pptx",): | |
| text = _read_pptx(path) | |
| elif ext in ("html", "htm"): | |
| text = _read_html(path) | |
| elif ext in ("odt", "epub"): | |
| text = _read_opendoc(path) | |
| else: | |
| text = _read_textfile(path) | |
| elif cat == "vector": | |
| text = _read_svg(path) | |
| elif cat in _TEXT_EXTRACTORS: | |
| text = _TEXT_EXTRACTORS[cat](path) | |
| else: # "other" - try reading it as text; many unknowns are plain text | |
| text = _read_textfile(path) | |
| # A file with an extension we do not know is read as text on the chance that | |
| # it is prose. When it is not - a column of numbers, a dump, a blob - the | |
| # plain read used to hand the model thousands of characters of nothing. One | |
| # in the test corpus was 9316 characters of digits. Say what it looks like | |
| # and show a sample instead, so REDUCE can still list it honestly. | |
| if cat == "other" and text and _word_density(text) < 0.15: | |
| sample = " ".join(text.split())[:300] | |
| text = ("This is a data file, not prose: almost none of it is words. " | |
| f"It starts: {sample}") | |
| text = _truncate(text) | |
| if not text.strip(): | |
| return {"name": name, "category": cat, "kind": "error", | |
| "payload": "no readable text found"} | |
| return {"name": name, "category": cat, "kind": "text", "payload": text} | |
| except NotText: | |
| return {"name": name, "category": cat, "kind": "error", | |
| "payload": "this is a binary file, not something with readable text"} | |
| except Exception as e: | |
| return {"name": name, "category": cat, "kind": "error", | |
| "payload": f"couldn't read ({type(e).__name__})"} | |