Spaces:
Paused
Paused
File size: 19,110 Bytes
281f5b1 8413646 da9de03 281f5b1 da9de03 281f5b1 8413646 281f5b1 a483b94 281f5b1 a483b94 281f5b1 602f9e5 8413646 281f5b1 602f9e5 281f5b1 a483b94 281f5b1 da9de03 281f5b1 da9de03 281f5b1 a483b94 281f5b1 a483b94 281f5b1 a483b94 281f5b1 da9de03 281f5b1 da9de03 281f5b1 602f9e5 a483b94 602f9e5 281f5b1 602f9e5 281f5b1 da9de03 281f5b1 602f9e5 281f5b1 602f9e5 281f5b1 8413646 281f5b1 a416f24 281f5b1 8413646 281f5b1 602f9e5 281f5b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 | # File categorization + content extraction. Each file resolves to one of:
# kind="text" β payload is extracted plain text (docs, sheets, code, data, html)
# kind="image" β payload is a base64 data URL (read later by the VLM)
# kind="media" β payload is a filesystem path (transcribed later by Whisper)
# kind="error" β payload is a short human reason it couldn't be read
# Everything here is best-effort and never raises: a single bad file shouldn't sink
# the whole task.
import base64
import io
import os
import re
import shutil
import tempfile
import zipfile
from config import (EXTRACT_MAX_CHARS, PDF_MAX_PAGES, PDF_SCAN_MAX_PAGES,
ZIP_MAX_MEMBERS, ZIP_MEMBER_MAX_BYTES, ZIP_TOTAL_MAX_BYTES)
# Formats LUNA has no parser for. They used to fall through to a raw text read, which
# for a .doc or .xls is a binary blob: the file was reported as "binary, not readable"
# and the user was left to guess why their Word document was unreadable.
UNSUPPORTED = {
"doc": "an old Word file (.doc): save it as .docx or PDF",
"ppt": "an old PowerPoint file (.ppt): save it as .pptx or PDF",
"rtf": "a Rich Text file (.rtf): save it as .docx, .txt or PDF",
"xls": "an old Excel file (.xls): save it as .xlsx or .csv",
"ods": "an OpenDocument spreadsheet (.ods): save it as .xlsx or .csv",
}
# ββ Extension β category map (category is also what the UI shows) ββββββββββββββ
_EXT = {
# documents
"pdf": "document", "docx": "document", "doc": "document", "pptx": "document",
"ppt": "document", "rtf": "document", "odt": "document", "epub": "document",
"html": "document", "htm": "document",
# spreadsheets
"xlsx": "spreadsheet", "xls": "spreadsheet", "ods": "spreadsheet",
# plain text / data
"txt": "text", "text": "text", "log": "text", "md": "text", "markdown": "text",
"csv": "data", "tsv": "data", "json": "data", "yaml": "data", "yml": "data",
"xml": "data", "ini": "data", "toml": "data",
# code
"py": "code", "js": "code", "ts": "code", "jsx": "code", "tsx": "code",
"java": "code", "c": "code", "h": "code", "cpp": "code", "cc": "code",
"cs": "code", "go": "code", "rs": "code", "rb": "code", "php": "code",
"swift": "code", "kt": "code", "sh": "code", "bash": "code", "sql": "code",
"css": "code", "scss": "code", "r": "code", "m": "code", "lua": "code",
# images
"png": "image", "jpg": "image", "jpeg": "image", "gif": "image",
"webp": "image", "bmp": "image", "tif": "image", "tiff": "image",
# audio / video
"mp3": "audio", "wav": "audio", "m4a": "audio", "flac": "audio",
"ogg": "audio", "aac": "audio", "opus": "audio", "wma": "audio",
"mp4": "video", "mov": "video", "mkv": "video", "webm": "video",
"avi": "video", "m4v": "video", "wmv": "video", "flv": "video",
# vector graphics
"svg": "vector",
# archives
"zip": "archive",
}
_TEXTLIKE = {"text", "data", "code"}
def category_of(name: str) -> str:
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
return _EXT.get(ext, "other")
def _truncate(s: str) -> str:
"""Cap one file's text at what a build may read of it.
This used to cap at PER_FILE_CHARS, the size of ONE model call, so a long document
existed only as far as its first 16000 characters: a 143-page report (420968
characters) was read to page 7 and no chapter of it ever reached a model. The cap
is now the whole-file budget; pipeline splits what is over one call's worth into
parts."""
s = (s or "").strip()
return s[:EXTRACT_MAX_CHARS] if len(s) > EXTRACT_MAX_CHARS else s
class NotText(Exception):
"""Raised when a file is binary, so the caller can say so instead of guessing."""
def _looks_binary(raw: bytes) -> bool:
"""A NUL byte, or a lot of bytes no text encoding produces, means this is not text.
Without this check an unknown extension was read as UTF-8 with errors="replace" and
the resulting mojibake was sent to the model as if it were content: tokens paid for
noise, and a real risk the model reads meaning into it. Measured on a test file of
raw bytes: 1192 of the first 2000 characters were replacement or control codes."""
if not raw:
return False
if b"\x00" in raw:
return True
odd = sum(1 for b in raw if b < 9 or 13 < b < 32 or b == 127)
return odd / len(raw) > 0.06
_B64_BLOB = re.compile(r"[A-Za-z0-9+/]{200,}={0,2}")
_WORDISH = re.compile(r"[A-Za-zΓ-ΓΓ-ΓΆΓΈ-ΓΏΔ-ΕΏ]{3,}")
def _word_density(s: str) -> float:
"""Share of the text made of word-like runs. Prose sits above 0.6; a column of
numbers sits near 0."""
if not s:
return 0.0
return sum(len(w) for w in _WORDISH.findall(s)) / len(s)
def _read_svg(path: str) -> str:
"""An SVG is text, so the plain reader shipped the whole thing: 16000 characters
of path data and base64 for one logo in the test corpus. Tokens were paid for
noise, and a stray letter run inside a base64 blob even scored as a keyword hit.
Only the parts a person would actually read are kept."""
import xml.etree.ElementTree as ET
with open(path, "r", encoding="utf-8", errors="replace") as f:
raw = _B64_BLOB.sub(" ", f.read(4_000_000))
seen, size = [], "? x ?"
try:
root = ET.fromstring(raw)
for el in root.iter():
if el.tag.rsplit("}", 1)[-1].lower() in ("title", "desc", "text", "tspan"):
t = (el.text or "").strip()
if t and t not in seen:
seen.append(t)
size = f"{root.get('width') or '?'} x {root.get('height') or '?'}"
except Exception:
# Exported SVGs are not always well formed - an undeclared namespace prefix
# is enough to stop the parser dead. Falling back to a scan keeps the words
# rather than reporting the whole file as unreadable.
for m in re.finditer(r"<(?:title|desc|text|tspan)\b[^>]*>(.*?)</", raw,
re.S | re.I):
t = re.sub(r"<[^>]*>", " ", m.group(1)).strip()
if t and t not in seen:
seen.append(t)
d = re.search(r'\bwidth\s*=\s*"([^"]+)"', raw)
h = re.search(r'\bheight\s*=\s*"([^"]+)"', raw)
if d or h:
size = f"{d.group(1) if d else '?'} x {h.group(1) if h else '?'}"
body = " ".join(seen).strip()
return (f"SVG vector graphic, {size}. Text it contains: {body}" if body
else f"SVG vector graphic, {size}. It contains no readable text.")
def _read_textfile(path: str) -> str:
with open(path, "rb") as f:
head = f.read(4096)
if _looks_binary(head):
raise NotText()
with open(path, "r", encoding="utf-8", errors="replace") as f:
return f.read(EXTRACT_MAX_CHARS + 1)
def _read_pdf(path: str) -> str:
from pypdf import PdfReader
reader = PdfReader(path)
n_pages = len(reader.pages)
out, total = [], 0
for i, page in enumerate(reader.pages):
if i >= PDF_MAX_PAGES:
out.append(f"[Only the first {PDF_MAX_PAGES} of {n_pages} pages were read.]")
break
t = page.extract_text() or ""
out.append(t)
total += len(t)
if total > EXTRACT_MAX_CHARS:
break
return "\n".join(out)
def _read_docx(path: str) -> str:
import docx
d = docx.Document(path)
parts = [p.text for p in d.paragraphs if p.text.strip()]
for tbl in d.tables:
for row in tbl.rows:
cells = [c.text.strip() for c in row.cells]
if any(cells):
parts.append(" | ".join(cells))
return "\n".join(parts)
def _read_pptx(path: str) -> str:
from pptx import Presentation
prs = Presentation(path)
parts = []
for i, slide in enumerate(prs.slides, 1):
parts.append(f"[Slide {i}]")
for shape in slide.shapes:
if shape.has_text_frame and shape.text_frame.text.strip():
parts.append(shape.text_frame.text)
return "\n".join(parts)
def _read_xlsx(path: str) -> str:
from openpyxl import load_workbook
wb = load_workbook(path, read_only=True, data_only=True)
parts, total = [], 0
for ws in wb.worksheets:
parts.append(f"[Sheet: {ws.title}]")
for row in ws.iter_rows(values_only=True):
cells = [str(c) for c in row if c is not None]
if cells:
line = " | ".join(cells)
parts.append(line)
total += len(line)
if total > EXTRACT_MAX_CHARS:
break
if total > EXTRACT_MAX_CHARS:
break
wb.close()
return "\n".join(parts)
def _read_html(path: str) -> str:
from bs4 import BeautifulSoup
with open(path, "r", encoding="utf-8", errors="replace") as f:
soup = BeautifulSoup(f.read(), "html.parser")
for tag in soup(["script", "style", "noscript"]):
tag.decompose()
return soup.get_text(separator="\n")
_ZIP_BY_EXT = {
"pdf": "_read_pdf", "docx": "_read_docx", "pptx": "_read_pptx", "xlsx": "_read_xlsx",
"html": "_read_html", "htm": "_read_html", "odt": "_read_opendoc", "epub": "_read_opendoc",
"svg": "_read_svg",
}
def _read_zip(path: str) -> str:
"""Members through the SAME readers a loose upload gets, within hard bounds.
Before: every text-like or document member was read as UTF-8 regardless of what it
was, so a .docx or .pdf inside an archive arrived as binary mojibake, a member's
declared size was never checked (a 25 MB upload can declare gigabytes), and media
inside were dropped without a word. Now binaries go through their extractors via a
temporary file, sizes are capped per member and in total, and what was not read is
listed so the Note can account for it."""
parts, total, read_bytes, listed = [], 0, 0, []
not_read: list[str] = []
tmpdir = tempfile.mkdtemp(prefix="luna_zip_")
try:
with zipfile.ZipFile(path) as z:
infos = [i for i in z.infolist() if not i.is_dir() and i.file_size > 0]
if len(infos) > ZIP_MAX_MEMBERS:
not_read.append(f"{len(infos) - ZIP_MAX_MEMBERS} further members (limit {ZIP_MAX_MEMBERS})")
infos = infos[:ZIP_MAX_MEMBERS]
for info in infos:
member = info.filename
base = os.path.basename(member)
if base.startswith("._") or "__MACOSX/" in member:
continue
cat = category_of(member)
ext = member.rsplit(".", 1)[-1].lower() if "." in member else ""
if info.file_size > ZIP_MEMBER_MAX_BYTES:
not_read.append(f"{member} (too large)")
continue
if read_bytes + info.file_size > ZIP_TOTAL_MAX_BYTES:
not_read.append(f"{member} (archive budget used up)")
continue
if cat in ("audio", "video", "image", "archive") or ext in UNSUPPORTED:
not_read.append(f"{member} ({cat if cat != 'other' else ext})")
continue
if cat not in _TEXTLIKE and cat not in ("document", "spreadsheet", "vector", "other"):
not_read.append(member)
continue
try:
reader = _ZIP_BY_EXT.get(ext)
if reader:
# Real parser: needs a path. Extracted under a bounded read.
out = os.path.join(tmpdir, f"m{len(listed)}.{ext}")
with z.open(info) as src, open(out, "wb") as dst:
shutil.copyfileobj(src, dst, 1 << 20)
read_bytes += info.file_size
text = globals()[reader](out)
try:
os.remove(out)
except OSError:
pass
else:
with z.open(info) as fh:
raw = fh.read(min(info.file_size, EXTRACT_MAX_CHARS * 4))
read_bytes += len(raw)
if _looks_binary(raw[:4096]):
not_read.append(f"{member} (binary)")
continue
text = raw.decode("utf-8", errors="replace")
except Exception as ex:
not_read.append(f"{member} (couldn't read: {type(ex).__name__})")
continue
text = (text or "").strip()
if not text:
not_read.append(f"{member} (no readable text)")
continue
listed.append(member)
parts.append(f"\n=== {member} ===\n{text}")
total += len(text)
if total > EXTRACT_MAX_CHARS:
break
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
if not_read:
parts.append("\n=== Members of this archive that were NOT read ===\n"
+ "\n".join(f"- {m}" for m in not_read[:50])
+ (f"\n- and {len(not_read) - 50} more" if len(not_read) > 50 else ""))
return "\n".join(parts)
def _pdf_page_images(path: str) -> list:
"""Page images out of a PDF that has no text layer, i.e. a scan.
Scanned PDFs used to come back as "no readable text found" and were set aside, which
for an invoice or a signed contract is the whole content of the file. Most scans
embed each page as one image object, so pypdf can hand it over directly and no
rasterising dependency is needed."""
from pypdf import PdfReader
out = []
reader = PdfReader(path)
for i, page in enumerate(reader.pages[:PDF_SCAN_MAX_PAGES], 1):
try:
for im in page.images:
out.append((i, _pil_data_url(im.image)))
break # one image per page is the scan itself
except Exception:
continue
return out
def _read_opendoc(path: str) -> str:
""".odt and .epub are ZIP containers of XML. Reading them as plain text produced
markup and binary noise; unpacking the XML and stripping tags gives the real text."""
import re as _re
parts, total = [], 0
with zipfile.ZipFile(path) as z:
members = [m for m in z.namelist()
if m.endswith((".xml", ".xhtml", ".html", ".htm"))
and not m.startswith("META-INF/")]
for m in members:
try:
raw = z.read(m).decode("utf-8", errors="replace")
except Exception:
continue
txt = _re.sub(r"<[^>]+>", " ", raw)
txt = _re.sub(r"\s+", " ", txt).strip()
if len(txt) < 20:
continue
parts.append(txt)
total += len(txt)
if total > EXTRACT_MAX_CHARS:
break
return "\n".join(parts)
def _pil_data_url(img) -> str:
"""One PIL image, downscaled and encoded the way the vision model wants it."""
from config import VLM_FRAME_MAX_PX
img = img.convert("RGB")
w, h = img.size
scale = VLM_FRAME_MAX_PX / max(w, h)
if scale < 1:
img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85)
return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
def _image_data_url(path: str) -> str:
from PIL import Image, ImageOps
img = Image.open(path)
# Phone photos carry their rotation in EXIF rather than in the pixels. Without this
# the vision model was handed sideways images and read the text in them badly.
img = ImageOps.exif_transpose(img)
return _pil_data_url(img)
# Dispatch by category β (kind, extractor)
_TEXT_EXTRACTORS = {
"document": None, # resolved below per-extension
"spreadsheet": _read_xlsx,
"text": _read_textfile,
"data": _read_textfile,
"code": _read_textfile,
"archive": _read_zip,
}
def extract(path: str, name: str | None = None) -> dict:
"""Extract one file. Returns {name, category, kind, payload}."""
name = name or os.path.basename(path)
cat = category_of(name)
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
try:
if ext in UNSUPPORTED:
return {"name": name, "category": cat, "kind": "error",
"payload": f"unsupported format: {UNSUPPORTED[ext]}"}
if cat == "image":
return {"name": name, "category": cat, "kind": "image",
"payload": _image_data_url(path)}
if cat in ("audio", "video"):
return {"name": name, "category": cat, "kind": "media", "payload": path}
if cat == "document":
if ext == "pdf":
text = _read_pdf(path)
if not text.strip():
pages = _pdf_page_images(path)
if pages:
return {"name": name, "category": cat, "kind": "pages",
"payload": pages}
elif ext in ("docx",):
text = _read_docx(path)
elif ext in ("pptx",):
text = _read_pptx(path)
elif ext in ("html", "htm"):
text = _read_html(path)
elif ext in ("odt", "epub"):
text = _read_opendoc(path)
else:
text = _read_textfile(path)
elif cat == "vector":
text = _read_svg(path)
elif cat in _TEXT_EXTRACTORS:
text = _TEXT_EXTRACTORS[cat](path)
else: # "other" - try reading it as text; many unknowns are plain text
text = _read_textfile(path)
# A file with an extension we do not know is read as text on the chance that
# it is prose. When it is not - a column of numbers, a dump, a blob - the
# plain read used to hand the model thousands of characters of nothing. One
# in the test corpus was 9316 characters of digits. Say what it looks like
# and show a sample instead, so REDUCE can still list it honestly.
if cat == "other" and text and _word_density(text) < 0.15:
sample = " ".join(text.split())[:300]
text = ("This is a data file, not prose: almost none of it is words. "
f"It starts: {sample}")
text = _truncate(text)
if not text.strip():
return {"name": name, "category": cat, "kind": "error",
"payload": "no readable text found"}
return {"name": name, "category": cat, "kind": "text", "payload": text}
except NotText:
return {"name": name, "category": cat, "kind": "error",
"payload": "this is a binary file, not something with readable text"}
except Exception as e:
return {"name": name, "category": cat, "kind": "error",
"payload": f"couldn't read ({type(e).__name__})"}
|