File size: 19,110 Bytes
281f5b1
 
 
 
 
 
 
 
 
 
8413646
da9de03
 
281f5b1
 
da9de03
 
 
 
 
 
 
 
 
 
 
 
 
281f5b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8413646
 
281f5b1
 
 
 
 
 
 
 
 
 
 
 
 
a483b94
 
 
 
 
 
 
281f5b1
a483b94
281f5b1
 
602f9e5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8413646
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
281f5b1
602f9e5
 
 
 
281f5b1
a483b94
281f5b1
 
 
 
 
da9de03
281f5b1
da9de03
 
 
 
281f5b1
 
 
a483b94
281f5b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a483b94
281f5b1
a483b94
281f5b1
 
 
 
 
 
 
 
 
 
 
 
 
 
da9de03
 
 
 
 
 
 
281f5b1
da9de03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
281f5b1
 
 
602f9e5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a483b94
602f9e5
 
 
 
 
 
281f5b1
 
 
 
 
 
 
 
602f9e5
 
 
 
 
 
 
 
 
 
281f5b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
da9de03
 
 
281f5b1
 
 
 
 
 
 
 
 
602f9e5
 
 
 
 
281f5b1
 
 
 
 
 
602f9e5
 
281f5b1
 
8413646
 
281f5b1
 
a416f24
281f5b1
 
8413646
 
 
 
 
 
 
 
 
281f5b1
 
 
 
 
602f9e5
 
 
281f5b1
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
# File categorization + content extraction. Each file resolves to one of:
#   kind="text"  β†’ payload is extracted plain text (docs, sheets, code, data, html)
#   kind="image" β†’ payload is a base64 data URL (read later by the VLM)
#   kind="media" β†’ payload is a filesystem path (transcribed later by Whisper)
#   kind="error" β†’ payload is a short human reason it couldn't be read
# Everything here is best-effort and never raises: a single bad file shouldn't sink
# the whole task.
import base64
import io
import os
import re
import shutil
import tempfile
import zipfile

from config import (EXTRACT_MAX_CHARS, PDF_MAX_PAGES, PDF_SCAN_MAX_PAGES,
                    ZIP_MAX_MEMBERS, ZIP_MEMBER_MAX_BYTES, ZIP_TOTAL_MAX_BYTES)

# Formats LUNA has no parser for. They used to fall through to a raw text read, which
# for a .doc or .xls is a binary blob: the file was reported as "binary, not readable"
# and the user was left to guess why their Word document was unreadable.
UNSUPPORTED = {
    "doc": "an old Word file (.doc): save it as .docx or PDF",
    "ppt": "an old PowerPoint file (.ppt): save it as .pptx or PDF",
    "rtf": "a Rich Text file (.rtf): save it as .docx, .txt or PDF",
    "xls": "an old Excel file (.xls): save it as .xlsx or .csv",
    "ods": "an OpenDocument spreadsheet (.ods): save it as .xlsx or .csv",
}

# ── Extension β†’ category map (category is also what the UI shows) ──────────────
_EXT = {
    # documents
    "pdf": "document", "docx": "document", "doc": "document", "pptx": "document",
    "ppt": "document", "rtf": "document", "odt": "document", "epub": "document",
    "html": "document", "htm": "document",
    # spreadsheets
    "xlsx": "spreadsheet", "xls": "spreadsheet", "ods": "spreadsheet",
    # plain text / data
    "txt": "text", "text": "text", "log": "text", "md": "text", "markdown": "text",
    "csv": "data", "tsv": "data", "json": "data", "yaml": "data", "yml": "data",
    "xml": "data", "ini": "data", "toml": "data",
    # code
    "py": "code", "js": "code", "ts": "code", "jsx": "code", "tsx": "code",
    "java": "code", "c": "code", "h": "code", "cpp": "code", "cc": "code",
    "cs": "code", "go": "code", "rs": "code", "rb": "code", "php": "code",
    "swift": "code", "kt": "code", "sh": "code", "bash": "code", "sql": "code",
    "css": "code", "scss": "code", "r": "code", "m": "code", "lua": "code",
    # images
    "png": "image", "jpg": "image", "jpeg": "image", "gif": "image",
    "webp": "image", "bmp": "image", "tif": "image", "tiff": "image",
    # audio / video
    "mp3": "audio", "wav": "audio", "m4a": "audio", "flac": "audio",
    "ogg": "audio", "aac": "audio", "opus": "audio", "wma": "audio",
    "mp4": "video", "mov": "video", "mkv": "video", "webm": "video",
    "avi": "video", "m4v": "video", "wmv": "video", "flv": "video",
    # vector graphics
    "svg": "vector",
    # archives
    "zip": "archive",
}

_TEXTLIKE = {"text", "data", "code"}


def category_of(name: str) -> str:
    ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
    return _EXT.get(ext, "other")


def _truncate(s: str) -> str:
    """Cap one file's text at what a build may read of it.

    This used to cap at PER_FILE_CHARS, the size of ONE model call, so a long document
    existed only as far as its first 16000 characters: a 143-page report (420968
    characters) was read to page 7 and no chapter of it ever reached a model. The cap
    is now the whole-file budget; pipeline splits what is over one call's worth into
    parts."""
    s = (s or "").strip()
    return s[:EXTRACT_MAX_CHARS] if len(s) > EXTRACT_MAX_CHARS else s


class NotText(Exception):
    """Raised when a file is binary, so the caller can say so instead of guessing."""


def _looks_binary(raw: bytes) -> bool:
    """A NUL byte, or a lot of bytes no text encoding produces, means this is not text.
    Without this check an unknown extension was read as UTF-8 with errors="replace" and
    the resulting mojibake was sent to the model as if it were content: tokens paid for
    noise, and a real risk the model reads meaning into it. Measured on a test file of
    raw bytes: 1192 of the first 2000 characters were replacement or control codes."""
    if not raw:
        return False
    if b"\x00" in raw:
        return True
    odd = sum(1 for b in raw if b < 9 or 13 < b < 32 or b == 127)
    return odd / len(raw) > 0.06


_B64_BLOB = re.compile(r"[A-Za-z0-9+/]{200,}={0,2}")
_WORDISH = re.compile(r"[A-Za-zΓ€-Γ–Γ˜-ΓΆΓΈ-ΓΏΔ„-ΕΏ]{3,}")


def _word_density(s: str) -> float:
    """Share of the text made of word-like runs. Prose sits above 0.6; a column of
    numbers sits near 0."""
    if not s:
        return 0.0
    return sum(len(w) for w in _WORDISH.findall(s)) / len(s)


def _read_svg(path: str) -> str:
    """An SVG is text, so the plain reader shipped the whole thing: 16000 characters
    of path data and base64 for one logo in the test corpus. Tokens were paid for
    noise, and a stray letter run inside a base64 blob even scored as a keyword hit.
    Only the parts a person would actually read are kept."""
    import xml.etree.ElementTree as ET
    with open(path, "r", encoding="utf-8", errors="replace") as f:
        raw = _B64_BLOB.sub(" ", f.read(4_000_000))
    seen, size = [], "? x ?"
    try:
        root = ET.fromstring(raw)
        for el in root.iter():
            if el.tag.rsplit("}", 1)[-1].lower() in ("title", "desc", "text", "tspan"):
                t = (el.text or "").strip()
                if t and t not in seen:
                    seen.append(t)
        size = f"{root.get('width') or '?'} x {root.get('height') or '?'}"
    except Exception:
        # Exported SVGs are not always well formed - an undeclared namespace prefix
        # is enough to stop the parser dead. Falling back to a scan keeps the words
        # rather than reporting the whole file as unreadable.
        for m in re.finditer(r"<(?:title|desc|text|tspan)\b[^>]*>(.*?)</", raw,
                             re.S | re.I):
            t = re.sub(r"<[^>]*>", " ", m.group(1)).strip()
            if t and t not in seen:
                seen.append(t)
        d = re.search(r'\bwidth\s*=\s*"([^"]+)"', raw)
        h = re.search(r'\bheight\s*=\s*"([^"]+)"', raw)
        if d or h:
            size = f"{d.group(1) if d else '?'} x {h.group(1) if h else '?'}"
    body = " ".join(seen).strip()
    return (f"SVG vector graphic, {size}. Text it contains: {body}" if body
            else f"SVG vector graphic, {size}. It contains no readable text.")


def _read_textfile(path: str) -> str:
    with open(path, "rb") as f:
        head = f.read(4096)
    if _looks_binary(head):
        raise NotText()
    with open(path, "r", encoding="utf-8", errors="replace") as f:
        return f.read(EXTRACT_MAX_CHARS + 1)


def _read_pdf(path: str) -> str:
    from pypdf import PdfReader
    reader = PdfReader(path)
    n_pages = len(reader.pages)
    out, total = [], 0
    for i, page in enumerate(reader.pages):
        if i >= PDF_MAX_PAGES:
            out.append(f"[Only the first {PDF_MAX_PAGES} of {n_pages} pages were read.]")
            break
        t = page.extract_text() or ""
        out.append(t)
        total += len(t)
        if total > EXTRACT_MAX_CHARS:
            break
    return "\n".join(out)


def _read_docx(path: str) -> str:
    import docx
    d = docx.Document(path)
    parts = [p.text for p in d.paragraphs if p.text.strip()]
    for tbl in d.tables:
        for row in tbl.rows:
            cells = [c.text.strip() for c in row.cells]
            if any(cells):
                parts.append(" | ".join(cells))
    return "\n".join(parts)


def _read_pptx(path: str) -> str:
    from pptx import Presentation
    prs = Presentation(path)
    parts = []
    for i, slide in enumerate(prs.slides, 1):
        parts.append(f"[Slide {i}]")
        for shape in slide.shapes:
            if shape.has_text_frame and shape.text_frame.text.strip():
                parts.append(shape.text_frame.text)
    return "\n".join(parts)


def _read_xlsx(path: str) -> str:
    from openpyxl import load_workbook
    wb = load_workbook(path, read_only=True, data_only=True)
    parts, total = [], 0
    for ws in wb.worksheets:
        parts.append(f"[Sheet: {ws.title}]")
        for row in ws.iter_rows(values_only=True):
            cells = [str(c) for c in row if c is not None]
            if cells:
                line = " | ".join(cells)
                parts.append(line)
                total += len(line)
            if total > EXTRACT_MAX_CHARS:
                break
        if total > EXTRACT_MAX_CHARS:
            break
    wb.close()
    return "\n".join(parts)


def _read_html(path: str) -> str:
    from bs4 import BeautifulSoup
    with open(path, "r", encoding="utf-8", errors="replace") as f:
        soup = BeautifulSoup(f.read(), "html.parser")
    for tag in soup(["script", "style", "noscript"]):
        tag.decompose()
    return soup.get_text(separator="\n")


_ZIP_BY_EXT = {
    "pdf": "_read_pdf", "docx": "_read_docx", "pptx": "_read_pptx", "xlsx": "_read_xlsx",
    "html": "_read_html", "htm": "_read_html", "odt": "_read_opendoc", "epub": "_read_opendoc",
    "svg": "_read_svg",
}


def _read_zip(path: str) -> str:
    """Members through the SAME readers a loose upload gets, within hard bounds.

    Before: every text-like or document member was read as UTF-8 regardless of what it
    was, so a .docx or .pdf inside an archive arrived as binary mojibake, a member's
    declared size was never checked (a 25 MB upload can declare gigabytes), and media
    inside were dropped without a word. Now binaries go through their extractors via a
    temporary file, sizes are capped per member and in total, and what was not read is
    listed so the Note can account for it."""
    parts, total, read_bytes, listed = [], 0, 0, []
    not_read: list[str] = []
    tmpdir = tempfile.mkdtemp(prefix="luna_zip_")
    try:
        with zipfile.ZipFile(path) as z:
            infos = [i for i in z.infolist() if not i.is_dir() and i.file_size > 0]
            if len(infos) > ZIP_MAX_MEMBERS:
                not_read.append(f"{len(infos) - ZIP_MAX_MEMBERS} further members (limit {ZIP_MAX_MEMBERS})")
                infos = infos[:ZIP_MAX_MEMBERS]
            for info in infos:
                member = info.filename
                base = os.path.basename(member)
                if base.startswith("._") or "__MACOSX/" in member:
                    continue
                cat = category_of(member)
                ext = member.rsplit(".", 1)[-1].lower() if "." in member else ""
                if info.file_size > ZIP_MEMBER_MAX_BYTES:
                    not_read.append(f"{member} (too large)")
                    continue
                if read_bytes + info.file_size > ZIP_TOTAL_MAX_BYTES:
                    not_read.append(f"{member} (archive budget used up)")
                    continue
                if cat in ("audio", "video", "image", "archive") or ext in UNSUPPORTED:
                    not_read.append(f"{member} ({cat if cat != 'other' else ext})")
                    continue
                if cat not in _TEXTLIKE and cat not in ("document", "spreadsheet", "vector", "other"):
                    not_read.append(member)
                    continue
                try:
                    reader = _ZIP_BY_EXT.get(ext)
                    if reader:
                        # Real parser: needs a path. Extracted under a bounded read.
                        out = os.path.join(tmpdir, f"m{len(listed)}.{ext}")
                        with z.open(info) as src, open(out, "wb") as dst:
                            shutil.copyfileobj(src, dst, 1 << 20)
                        read_bytes += info.file_size
                        text = globals()[reader](out)
                        try:
                            os.remove(out)
                        except OSError:
                            pass
                    else:
                        with z.open(info) as fh:
                            raw = fh.read(min(info.file_size, EXTRACT_MAX_CHARS * 4))
                        read_bytes += len(raw)
                        if _looks_binary(raw[:4096]):
                            not_read.append(f"{member} (binary)")
                            continue
                        text = raw.decode("utf-8", errors="replace")
                except Exception as ex:
                    not_read.append(f"{member} (couldn't read: {type(ex).__name__})")
                    continue
                text = (text or "").strip()
                if not text:
                    not_read.append(f"{member} (no readable text)")
                    continue
                listed.append(member)
                parts.append(f"\n=== {member} ===\n{text}")
                total += len(text)
                if total > EXTRACT_MAX_CHARS:
                    break
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True)
    if not_read:
        parts.append("\n=== Members of this archive that were NOT read ===\n"
                     + "\n".join(f"- {m}" for m in not_read[:50])
                     + (f"\n- and {len(not_read) - 50} more" if len(not_read) > 50 else ""))
    return "\n".join(parts)


def _pdf_page_images(path: str) -> list:
    """Page images out of a PDF that has no text layer, i.e. a scan.

    Scanned PDFs used to come back as "no readable text found" and were set aside, which
    for an invoice or a signed contract is the whole content of the file. Most scans
    embed each page as one image object, so pypdf can hand it over directly and no
    rasterising dependency is needed."""
    from pypdf import PdfReader
    out = []
    reader = PdfReader(path)
    for i, page in enumerate(reader.pages[:PDF_SCAN_MAX_PAGES], 1):
        try:
            for im in page.images:
                out.append((i, _pil_data_url(im.image)))
                break                      # one image per page is the scan itself
        except Exception:
            continue
    return out


def _read_opendoc(path: str) -> str:
    """.odt and .epub are ZIP containers of XML. Reading them as plain text produced
    markup and binary noise; unpacking the XML and stripping tags gives the real text."""
    import re as _re
    parts, total = [], 0
    with zipfile.ZipFile(path) as z:
        members = [m for m in z.namelist()
                   if m.endswith((".xml", ".xhtml", ".html", ".htm"))
                   and not m.startswith("META-INF/")]
        for m in members:
            try:
                raw = z.read(m).decode("utf-8", errors="replace")
            except Exception:
                continue
            txt = _re.sub(r"<[^>]+>", " ", raw)
            txt = _re.sub(r"\s+", " ", txt).strip()
            if len(txt) < 20:
                continue
            parts.append(txt)
            total += len(txt)
            if total > EXTRACT_MAX_CHARS:
                break
    return "\n".join(parts)


def _pil_data_url(img) -> str:
    """One PIL image, downscaled and encoded the way the vision model wants it."""
    from config import VLM_FRAME_MAX_PX
    img = img.convert("RGB")
    w, h = img.size
    scale = VLM_FRAME_MAX_PX / max(w, h)
    if scale < 1:
        img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))))
    buf = io.BytesIO()
    img.save(buf, format="JPEG", quality=85)
    return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()


def _image_data_url(path: str) -> str:
    from PIL import Image, ImageOps
    img = Image.open(path)
    # Phone photos carry their rotation in EXIF rather than in the pixels. Without this
    # the vision model was handed sideways images and read the text in them badly.
    img = ImageOps.exif_transpose(img)
    return _pil_data_url(img)


# Dispatch by category β†’ (kind, extractor)
_TEXT_EXTRACTORS = {
    "document": None,   # resolved below per-extension
    "spreadsheet": _read_xlsx,
    "text": _read_textfile,
    "data": _read_textfile,
    "code": _read_textfile,
    "archive": _read_zip,
}


def extract(path: str, name: str | None = None) -> dict:
    """Extract one file. Returns {name, category, kind, payload}."""
    name = name or os.path.basename(path)
    cat = category_of(name)
    ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""

    try:
        if ext in UNSUPPORTED:
            return {"name": name, "category": cat, "kind": "error",
                    "payload": f"unsupported format: {UNSUPPORTED[ext]}"}
        if cat == "image":
            return {"name": name, "category": cat, "kind": "image",
                    "payload": _image_data_url(path)}
        if cat in ("audio", "video"):
            return {"name": name, "category": cat, "kind": "media", "payload": path}

        if cat == "document":
            if ext == "pdf":
                text = _read_pdf(path)
                if not text.strip():
                    pages = _pdf_page_images(path)
                    if pages:
                        return {"name": name, "category": cat, "kind": "pages",
                                "payload": pages}
            elif ext in ("docx",):
                text = _read_docx(path)
            elif ext in ("pptx",):
                text = _read_pptx(path)
            elif ext in ("html", "htm"):
                text = _read_html(path)
            elif ext in ("odt", "epub"):
                text = _read_opendoc(path)
            else:
                text = _read_textfile(path)
        elif cat == "vector":
            text = _read_svg(path)
        elif cat in _TEXT_EXTRACTORS:
            text = _TEXT_EXTRACTORS[cat](path)
        else:  # "other" - try reading it as text; many unknowns are plain text
            text = _read_textfile(path)

        # A file with an extension we do not know is read as text on the chance that
        # it is prose. When it is not - a column of numbers, a dump, a blob - the
        # plain read used to hand the model thousands of characters of nothing. One
        # in the test corpus was 9316 characters of digits. Say what it looks like
        # and show a sample instead, so REDUCE can still list it honestly.
        if cat == "other" and text and _word_density(text) < 0.15:
            sample = " ".join(text.split())[:300]
            text = ("This is a data file, not prose: almost none of it is words. "
                    f"It starts: {sample}")
        text = _truncate(text)
        if not text.strip():
            return {"name": name, "category": cat, "kind": "error",
                    "payload": "no readable text found"}
        return {"name": name, "category": cat, "kind": "text", "payload": text}
    except NotText:
        return {"name": name, "category": cat, "kind": "error",
                "payload": "this is a binary file, not something with readable text"}
    except Exception as e:
        return {"name": name, "category": cat, "kind": "error",
                "payload": f"couldn't read ({type(e).__name__})"}