# File categorization + content extraction. Each file resolves to one of: # kind="text" → payload is extracted plain text (docs, sheets, code, data, html) # kind="image" → payload is a base64 data URL (read later by the VLM) # kind="media" → payload is a filesystem path (transcribed later by Whisper) # kind="error" → payload is a short human reason it couldn't be read # Everything here is best-effort and never raises: a single bad file shouldn't sink # the whole task. import base64 import io import os import re import shutil import tempfile import zipfile from config import (EXTRACT_MAX_CHARS, PDF_MAX_PAGES, PDF_SCAN_MAX_PAGES, ZIP_MAX_MEMBERS, ZIP_MEMBER_MAX_BYTES, ZIP_TOTAL_MAX_BYTES) # Formats LUNA has no parser for. They used to fall through to a raw text read, which # for a .doc or .xls is a binary blob: the file was reported as "binary, not readable" # and the user was left to guess why their Word document was unreadable. UNSUPPORTED = { "doc": "an old Word file (.doc): save it as .docx or PDF", "ppt": "an old PowerPoint file (.ppt): save it as .pptx or PDF", "rtf": "a Rich Text file (.rtf): save it as .docx, .txt or PDF", "xls": "an old Excel file (.xls): save it as .xlsx or .csv", "ods": "an OpenDocument spreadsheet (.ods): save it as .xlsx or .csv", } # ── Extension → category map (category is also what the UI shows) ────────────── _EXT = { # documents "pdf": "document", "docx": "document", "doc": "document", "pptx": "document", "ppt": "document", "rtf": "document", "odt": "document", "epub": "document", "html": "document", "htm": "document", # spreadsheets "xlsx": "spreadsheet", "xls": "spreadsheet", "ods": "spreadsheet", # plain text / data "txt": "text", "text": "text", "log": "text", "md": "text", "markdown": "text", "csv": "data", "tsv": "data", "json": "data", "yaml": "data", "yml": "data", "xml": "data", "ini": "data", "toml": "data", # code "py": "code", "js": "code", "ts": "code", "jsx": "code", "tsx": "code", "java": "code", "c": "code", "h": "code", "cpp": "code", "cc": "code", "cs": "code", "go": "code", "rs": "code", "rb": "code", "php": "code", "swift": "code", "kt": "code", "sh": "code", "bash": "code", "sql": "code", "css": "code", "scss": "code", "r": "code", "m": "code", "lua": "code", # images "png": "image", "jpg": "image", "jpeg": "image", "gif": "image", "webp": "image", "bmp": "image", "tif": "image", "tiff": "image", # audio / video "mp3": "audio", "wav": "audio", "m4a": "audio", "flac": "audio", "ogg": "audio", "aac": "audio", "opus": "audio", "wma": "audio", "mp4": "video", "mov": "video", "mkv": "video", "webm": "video", "avi": "video", "m4v": "video", "wmv": "video", "flv": "video", # vector graphics "svg": "vector", # archives "zip": "archive", } _TEXTLIKE = {"text", "data", "code"} def category_of(name: str) -> str: ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" return _EXT.get(ext, "other") def _truncate(s: str) -> str: """Cap one file's text at what a build may read of it. This used to cap at PER_FILE_CHARS, the size of ONE model call, so a long document existed only as far as its first 16000 characters: a 143-page report (420968 characters) was read to page 7 and no chapter of it ever reached a model. The cap is now the whole-file budget; pipeline splits what is over one call's worth into parts.""" s = (s or "").strip() return s[:EXTRACT_MAX_CHARS] if len(s) > EXTRACT_MAX_CHARS else s class NotText(Exception): """Raised when a file is binary, so the caller can say so instead of guessing.""" def _looks_binary(raw: bytes) -> bool: """A NUL byte, or a lot of bytes no text encoding produces, means this is not text. Without this check an unknown extension was read as UTF-8 with errors="replace" and the resulting mojibake was sent to the model as if it were content: tokens paid for noise, and a real risk the model reads meaning into it. Measured on a test file of raw bytes: 1192 of the first 2000 characters were replacement or control codes.""" if not raw: return False if b"\x00" in raw: return True odd = sum(1 for b in raw if b < 9 or 13 < b < 32 or b == 127) return odd / len(raw) > 0.06 _B64_BLOB = re.compile(r"[A-Za-z0-9+/]{200,}={0,2}") _WORDISH = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿĄ-ſ]{3,}") def _word_density(s: str) -> float: """Share of the text made of word-like runs. Prose sits above 0.6; a column of numbers sits near 0.""" if not s: return 0.0 return sum(len(w) for w in _WORDISH.findall(s)) / len(s) def _read_svg(path: str) -> str: """An SVG is text, so the plain reader shipped the whole thing: 16000 characters of path data and base64 for one logo in the test corpus. Tokens were paid for noise, and a stray letter run inside a base64 blob even scored as a keyword hit. Only the parts a person would actually read are kept.""" import xml.etree.ElementTree as ET with open(path, "r", encoding="utf-8", errors="replace") as f: raw = _B64_BLOB.sub(" ", f.read(4_000_000)) seen, size = [], "? x ?" try: root = ET.fromstring(raw) for el in root.iter(): if el.tag.rsplit("}", 1)[-1].lower() in ("title", "desc", "text", "tspan"): t = (el.text or "").strip() if t and t not in seen: seen.append(t) size = f"{root.get('width') or '?'} x {root.get('height') or '?'}" except Exception: # Exported SVGs are not always well formed - an undeclared namespace prefix # is enough to stop the parser dead. Falling back to a scan keeps the words # rather than reporting the whole file as unreadable. for m in re.finditer(r"<(?:title|desc|text|tspan)\b[^>]*>(.*?)]*>", " ", m.group(1)).strip() if t and t not in seen: seen.append(t) d = re.search(r'\bwidth\s*=\s*"([^"]+)"', raw) h = re.search(r'\bheight\s*=\s*"([^"]+)"', raw) if d or h: size = f"{d.group(1) if d else '?'} x {h.group(1) if h else '?'}" body = " ".join(seen).strip() return (f"SVG vector graphic, {size}. Text it contains: {body}" if body else f"SVG vector graphic, {size}. It contains no readable text.") def _read_textfile(path: str) -> str: with open(path, "rb") as f: head = f.read(4096) if _looks_binary(head): raise NotText() with open(path, "r", encoding="utf-8", errors="replace") as f: return f.read(EXTRACT_MAX_CHARS + 1) def _read_pdf(path: str) -> str: from pypdf import PdfReader reader = PdfReader(path) n_pages = len(reader.pages) out, total = [], 0 for i, page in enumerate(reader.pages): if i >= PDF_MAX_PAGES: out.append(f"[Only the first {PDF_MAX_PAGES} of {n_pages} pages were read.]") break t = page.extract_text() or "" out.append(t) total += len(t) if total > EXTRACT_MAX_CHARS: break return "\n".join(out) def _read_docx(path: str) -> str: import docx d = docx.Document(path) parts = [p.text for p in d.paragraphs if p.text.strip()] for tbl in d.tables: for row in tbl.rows: cells = [c.text.strip() for c in row.cells] if any(cells): parts.append(" | ".join(cells)) return "\n".join(parts) def _read_pptx(path: str) -> str: from pptx import Presentation prs = Presentation(path) parts = [] for i, slide in enumerate(prs.slides, 1): parts.append(f"[Slide {i}]") for shape in slide.shapes: if shape.has_text_frame and shape.text_frame.text.strip(): parts.append(shape.text_frame.text) return "\n".join(parts) def _read_xlsx(path: str) -> str: from openpyxl import load_workbook wb = load_workbook(path, read_only=True, data_only=True) parts, total = [], 0 for ws in wb.worksheets: parts.append(f"[Sheet: {ws.title}]") for row in ws.iter_rows(values_only=True): cells = [str(c) for c in row if c is not None] if cells: line = " | ".join(cells) parts.append(line) total += len(line) if total > EXTRACT_MAX_CHARS: break if total > EXTRACT_MAX_CHARS: break wb.close() return "\n".join(parts) def _read_html(path: str) -> str: from bs4 import BeautifulSoup with open(path, "r", encoding="utf-8", errors="replace") as f: soup = BeautifulSoup(f.read(), "html.parser") for tag in soup(["script", "style", "noscript"]): tag.decompose() return soup.get_text(separator="\n") _ZIP_BY_EXT = { "pdf": "_read_pdf", "docx": "_read_docx", "pptx": "_read_pptx", "xlsx": "_read_xlsx", "html": "_read_html", "htm": "_read_html", "odt": "_read_opendoc", "epub": "_read_opendoc", "svg": "_read_svg", } def _read_zip(path: str) -> str: """Members through the SAME readers a loose upload gets, within hard bounds. Before: every text-like or document member was read as UTF-8 regardless of what it was, so a .docx or .pdf inside an archive arrived as binary mojibake, a member's declared size was never checked (a 25 MB upload can declare gigabytes), and media inside were dropped without a word. Now binaries go through their extractors via a temporary file, sizes are capped per member and in total, and what was not read is listed so the Note can account for it.""" parts, total, read_bytes, listed = [], 0, 0, [] not_read: list[str] = [] tmpdir = tempfile.mkdtemp(prefix="luna_zip_") try: with zipfile.ZipFile(path) as z: infos = [i for i in z.infolist() if not i.is_dir() and i.file_size > 0] if len(infos) > ZIP_MAX_MEMBERS: not_read.append(f"{len(infos) - ZIP_MAX_MEMBERS} further members (limit {ZIP_MAX_MEMBERS})") infos = infos[:ZIP_MAX_MEMBERS] for info in infos: member = info.filename base = os.path.basename(member) if base.startswith("._") or "__MACOSX/" in member: continue cat = category_of(member) ext = member.rsplit(".", 1)[-1].lower() if "." in member else "" if info.file_size > ZIP_MEMBER_MAX_BYTES: not_read.append(f"{member} (too large)") continue if read_bytes + info.file_size > ZIP_TOTAL_MAX_BYTES: not_read.append(f"{member} (archive budget used up)") continue if cat in ("audio", "video", "image", "archive") or ext in UNSUPPORTED: not_read.append(f"{member} ({cat if cat != 'other' else ext})") continue if cat not in _TEXTLIKE and cat not in ("document", "spreadsheet", "vector", "other"): not_read.append(member) continue try: reader = _ZIP_BY_EXT.get(ext) if reader: # Real parser: needs a path. Extracted under a bounded read. out = os.path.join(tmpdir, f"m{len(listed)}.{ext}") with z.open(info) as src, open(out, "wb") as dst: shutil.copyfileobj(src, dst, 1 << 20) read_bytes += info.file_size text = globals()[reader](out) try: os.remove(out) except OSError: pass else: with z.open(info) as fh: raw = fh.read(min(info.file_size, EXTRACT_MAX_CHARS * 4)) read_bytes += len(raw) if _looks_binary(raw[:4096]): not_read.append(f"{member} (binary)") continue text = raw.decode("utf-8", errors="replace") except Exception as ex: not_read.append(f"{member} (couldn't read: {type(ex).__name__})") continue text = (text or "").strip() if not text: not_read.append(f"{member} (no readable text)") continue listed.append(member) parts.append(f"\n=== {member} ===\n{text}") total += len(text) if total > EXTRACT_MAX_CHARS: break finally: shutil.rmtree(tmpdir, ignore_errors=True) if not_read: parts.append("\n=== Members of this archive that were NOT read ===\n" + "\n".join(f"- {m}" for m in not_read[:50]) + (f"\n- and {len(not_read) - 50} more" if len(not_read) > 50 else "")) return "\n".join(parts) def _pdf_page_images(path: str) -> list: """Page images out of a PDF that has no text layer, i.e. a scan. Scanned PDFs used to come back as "no readable text found" and were set aside, which for an invoice or a signed contract is the whole content of the file. Most scans embed each page as one image object, so pypdf can hand it over directly and no rasterising dependency is needed.""" from pypdf import PdfReader out = [] reader = PdfReader(path) for i, page in enumerate(reader.pages[:PDF_SCAN_MAX_PAGES], 1): try: for im in page.images: out.append((i, _pil_data_url(im.image))) break # one image per page is the scan itself except Exception: continue return out def _read_opendoc(path: str) -> str: """.odt and .epub are ZIP containers of XML. Reading them as plain text produced markup and binary noise; unpacking the XML and stripping tags gives the real text.""" import re as _re parts, total = [], 0 with zipfile.ZipFile(path) as z: members = [m for m in z.namelist() if m.endswith((".xml", ".xhtml", ".html", ".htm")) and not m.startswith("META-INF/")] for m in members: try: raw = z.read(m).decode("utf-8", errors="replace") except Exception: continue txt = _re.sub(r"<[^>]+>", " ", raw) txt = _re.sub(r"\s+", " ", txt).strip() if len(txt) < 20: continue parts.append(txt) total += len(txt) if total > EXTRACT_MAX_CHARS: break return "\n".join(parts) def _pil_data_url(img) -> str: """One PIL image, downscaled and encoded the way the vision model wants it.""" from config import VLM_FRAME_MAX_PX img = img.convert("RGB") w, h = img.size scale = VLM_FRAME_MAX_PX / max(w, h) if scale < 1: img = img.resize((max(1, int(w * scale)), max(1, int(h * scale)))) buf = io.BytesIO() img.save(buf, format="JPEG", quality=85) return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode() def _image_data_url(path: str) -> str: from PIL import Image, ImageOps img = Image.open(path) # Phone photos carry their rotation in EXIF rather than in the pixels. Without this # the vision model was handed sideways images and read the text in them badly. img = ImageOps.exif_transpose(img) return _pil_data_url(img) # Dispatch by category → (kind, extractor) _TEXT_EXTRACTORS = { "document": None, # resolved below per-extension "spreadsheet": _read_xlsx, "text": _read_textfile, "data": _read_textfile, "code": _read_textfile, "archive": _read_zip, } def extract(path: str, name: str | None = None) -> dict: """Extract one file. Returns {name, category, kind, payload}.""" name = name or os.path.basename(path) cat = category_of(name) ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" try: if ext in UNSUPPORTED: return {"name": name, "category": cat, "kind": "error", "payload": f"unsupported format: {UNSUPPORTED[ext]}"} if cat == "image": return {"name": name, "category": cat, "kind": "image", "payload": _image_data_url(path)} if cat in ("audio", "video"): return {"name": name, "category": cat, "kind": "media", "payload": path} if cat == "document": if ext == "pdf": text = _read_pdf(path) if not text.strip(): pages = _pdf_page_images(path) if pages: return {"name": name, "category": cat, "kind": "pages", "payload": pages} elif ext in ("docx",): text = _read_docx(path) elif ext in ("pptx",): text = _read_pptx(path) elif ext in ("html", "htm"): text = _read_html(path) elif ext in ("odt", "epub"): text = _read_opendoc(path) else: text = _read_textfile(path) elif cat == "vector": text = _read_svg(path) elif cat in _TEXT_EXTRACTORS: text = _TEXT_EXTRACTORS[cat](path) else: # "other" - try reading it as text; many unknowns are plain text text = _read_textfile(path) # A file with an extension we do not know is read as text on the chance that # it is prose. When it is not - a column of numbers, a dump, a blob - the # plain read used to hand the model thousands of characters of nothing. One # in the test corpus was 9316 characters of digits. Say what it looks like # and show a sample instead, so REDUCE can still list it honestly. if cat == "other" and text and _word_density(text) < 0.15: sample = " ".join(text.split())[:300] text = ("This is a data file, not prose: almost none of it is words. " f"It starts: {sample}") text = _truncate(text) if not text.strip(): return {"name": name, "category": cat, "kind": "error", "payload": "no readable text found"} return {"name": name, "category": cat, "kind": "text", "payload": text} except NotText: return {"name": name, "category": cat, "kind": "error", "payload": "this is a binary file, not something with readable text"} except Exception as e: return {"name": name, "category": cat, "kind": "error", "payload": f"couldn't read ({type(e).__name__})"}