File size: 9,553 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
"""Uploaded-PDF store: extract, chunk, and answer questions against one paper.



Why this is not just "add it to the corpus": rebuilding the BM25 and FAISS

indices over 40,000 chunks to accommodate one 30-chunk document would take

minutes and be thrown away. A single paper is small enough that the

cross-encoder can score every one of its chunks directly - no index, no

approximation, and better ranking than retrieval would give.



Documents are persisted as JSON so they survive a restart, and the store is

capped so an unbounded number of uploads cannot fill the disk.

"""

from __future__ import annotations

import hashlib
import json
import time
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any

from src.data.arxiv_source import Paper
from src.data.chunking import chunk_paper

import re

MAX_UPLOAD_BYTES = 25 * 1024 * 1024  # 25 MB - larger than any sane paper
_ARXIV_STAMP_RE = re.compile(r"^\s*arxiv:\s*\d{4}\.\d{4,5}", re.IGNORECASE)
PDF_MAGIC = b"%PDF-"


class UploadError(ValueError):
    """Anything wrong with the uploaded file, phrased for the user."""


@dataclass
class UploadedDoc:
    document_id: str
    title: str
    n_chunks: int
    n_words: int
    uploaded_at: float
    filename: str

    def to_public(self) -> dict[str, Any]:
        return {
            "document_id": self.document_id,
            "title": self.title,
            "n_chunks": self.n_chunks,
            "n_words": self.n_words,
            "filename": self.filename,
        }


# Lines that appear before the title on real papers: licence grants, arXiv
# stamps, conference banners. "Attention Is All You Need" opens with Google's
# permission notice, so taking the first substantial line names the paper
# "Provided proper attribution is provided, Google hereby grants...".
_BOILERPLATE = (
    "provided proper attribution", "hereby grants", "permission to", "licence",
    "license", "copyright", "creative commons", "all rights reserved",
    "preprint", "under review", "accepted at", "to appear in", "proceedings of",
    "conference on", "workshop on", "arxiv", "doi:", "http", "www.",
    "equal contribution", "corresponding author",
)


def _title_from_layout(doc: Any) -> str | None:
    """Title = the largest text on page 1.



    Content heuristics lose to licence blocks: "Attention Is All You Need"

    opens with a multi-line Google permission notice, so both "first

    substantial line" and keyword filtering pick a fragment of the legal text.

    Font size does not have that problem - the title is set larger than

    everything around it, by typographic convention, in every paper format.

    """
    try:
        page = doc[0].get_text("dict")
    except Exception:  # noqa: BLE001
        return None

    spans: list[tuple[float, float, str]] = []
    for block in page.get("blocks", []):
        for line in block.get("lines", []):
            # arXiv stamps the id vertically down the left margin, in a large
            # font - it wins on size alone. Horizontal text only.
            direction = line.get("dir", (1.0, 0.0))
            if abs(direction[0]) < 0.9:
                continue
            for span in line.get("spans", []):
                text = " ".join(span.get("text", "").split())
                if not text or _ARXIV_STAMP_RE.match(text):
                    continue
                spans.append((round(span.get("size", 0), 1), span.get("bbox", [0, 0])[1], text))
    if not spans:
        return None

    largest = max(size for size, _, _ in spans)
    # Join every span at (near) the largest size, in reading order - titles
    # routinely wrap onto two or three lines.
    title_spans = sorted(
        [(y, t) for size, y, t in spans if size >= largest - 0.6], key=lambda st: st[0]
    )
    title = " ".join(t for _, t in title_spans).strip()
    return title if 10 <= len(title) <= 300 else None


def _guess_title(text: str, fallback: str) -> str:
    """Fallback when layout gives nothing: first plausible non-boilerplate line."""
    candidates: list[str] = []
    for raw in text.splitlines()[:60]:
        line = " ".join(raw.split())
        lowered = line.lower()
        if not (15 <= len(line) <= 250):
            continue
        if any(marker in lowered for marker in _BOILERPLATE):
            continue
        words = line.split()
        if len(words) < 3:
            continue
        if sum(c.isdigit() for c in line) > len(line) * 0.3:
            continue
        # Author lines are mostly capitalised short tokens separated by commas.
        if line.count(",") >= 3 and sum(w[:1].isupper() for w in words) > len(words) * 0.7:
            continue
        candidates.append(line)
        if len(candidates) >= 5:
            break
    return candidates[0] if candidates else fallback


class UploadStore:
    """Persisted store of uploaded documents and their chunks."""

    def __init__(self, directory: str = "cache/uploads", max_documents: int = 200) -> None:
        self.directory = Path(directory)
        self.directory.mkdir(parents=True, exist_ok=True)
        self.max_documents = max_documents

    def _path(self, document_id: str) -> Path:
        return self.directory / f"{document_id}.json"

    def _prune(self) -> None:
        files = sorted(self.directory.glob("*.json"), key=lambda f: f.stat().st_mtime)
        for path in files[: max(0, len(files) - self.max_documents)]:
            path.unlink(missing_ok=True)

    def add(self, data: bytes, filename: str) -> UploadedDoc:
        """Validate, extract, chunk and persist one PDF."""
        if not data:
            raise UploadError("The file is empty.")
        if len(data) > MAX_UPLOAD_BYTES:
            raise UploadError(
                f"That file is {len(data) / 1e6:.1f} MB; the limit is "
                f"{MAX_UPLOAD_BYTES / 1e6:.0f} MB."
            )
        if not data.startswith(PDF_MAGIC):
            raise UploadError("That does not look like a PDF.")

        # Content-addressed: re-uploading the same paper reuses the same id.
        document_id = hashlib.sha256(data).hexdigest()[:16]
        existing = self.get_meta(document_id)
        if existing is not None:
            return existing

        try:
            import pymupdf

            with pymupdf.open(stream=data, filetype="pdf") as doc:
                text = "\n".join(page.get_text() for page in doc)
                layout_title = _title_from_layout(doc)
                meta_title = " ".join((doc.metadata or {}).get("title", "").split())
        except Exception as exc:  # noqa: BLE001
            raise UploadError(f"Could not read that PDF ({type(exc).__name__}).") from exc

        words = text.split()
        if len(words) < 200:
            raise UploadError(
                "Almost no text came out of that PDF - it is probably a scan. "
                "This needs a text-based PDF, not images."
            )

        # Layout first, then embedded metadata, then the line heuristic.
        title = layout_title or (meta_title if len(meta_title) > 10 else "") or _guess_title(
            text, Path(filename).stem
        )
        paper = Paper(
            paper_id=document_id,
            version=1,
            title=title,
            abstract="",
            authors=[],
            categories=[],
            primary_category="upload",
            published="",
            updated="",
            pdf_url="",
            full_text=text,
        )
        chunks = chunk_paper(paper)
        if not chunks:
            raise UploadError("No usable passages could be extracted from that PDF.")

        meta = UploadedDoc(
            document_id=document_id,
            title=title,
            n_chunks=len(chunks),
            n_words=len(words),
            uploaded_at=time.time(),
            filename=filename,
        )
        self._path(document_id).write_text(
            json.dumps(
                {"meta": asdict(meta), "chunks": [asdict(c) for c in chunks]},
                ensure_ascii=False,
            ),
            encoding="utf-8",
        )
        self._prune()
        return meta

    def _load(self, document_id: str) -> dict[str, Any] | None:
        path = self._path(document_id)
        if not path.exists():
            return None
        try:
            return json.loads(path.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError):
            return None

    def get_meta(self, document_id: str) -> UploadedDoc | None:
        payload = self._load(document_id)
        return UploadedDoc(**payload["meta"]) if payload else None

    def passages(self, document_id: str) -> list[dict[str, Any]]:
        """Every chunk of one document, in the shape the LLM layer expects."""
        payload = self._load(document_id)
        if not payload:
            return []
        title = payload["meta"]["title"]
        return [
            {
                "chunk_id": c["chunk_id"],
                "doc_id": c["chunk_id"],
                "paper_id": document_id,
                "arxiv_url": "",
                "score": 0.0,
                "title": title,
                "section": c["section"],
                "published": "",
                "text": c["text"],
            }
            for c in payload["chunks"]
        ]