Spaces:
Running on Zero
Running on Zero
Download src/api/uploads.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 9.55 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/api/uploads.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/api/uploads.py
-
curl -L -o uploads.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/api/uploads.py
9.55 kB
| """Uploaded-PDF store: extract, chunk, and answer questions against one paper. | |
| Why this is not just "add it to the corpus": rebuilding the BM25 and FAISS | |
| indices over 40,000 chunks to accommodate one 30-chunk document would take | |
| minutes and be thrown away. A single paper is small enough that the | |
| cross-encoder can score every one of its chunks directly - no index, no | |
| approximation, and better ranking than retrieval would give. | |
| Documents are persisted as JSON so they survive a restart, and the store is | |
| capped so an unbounded number of uploads cannot fill the disk. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import time | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| from src.data.arxiv_source import Paper | |
| from src.data.chunking import chunk_paper | |
| import re | |
| MAX_UPLOAD_BYTES = 25 * 1024 * 1024 # 25 MB - larger than any sane paper | |
| _ARXIV_STAMP_RE = re.compile(r"^\s*arxiv:\s*\d{4}\.\d{4,5}", re.IGNORECASE) | |
| PDF_MAGIC = b"%PDF-" | |
| class UploadError(ValueError): | |
| """Anything wrong with the uploaded file, phrased for the user.""" | |
| class UploadedDoc: | |
| document_id: str | |
| title: str | |
| n_chunks: int | |
| n_words: int | |
| uploaded_at: float | |
| filename: str | |
| def to_public(self) -> dict[str, Any]: | |
| return { | |
| "document_id": self.document_id, | |
| "title": self.title, | |
| "n_chunks": self.n_chunks, | |
| "n_words": self.n_words, | |
| "filename": self.filename, | |
| } | |
| # Lines that appear before the title on real papers: licence grants, arXiv | |
| # stamps, conference banners. "Attention Is All You Need" opens with Google's | |
| # permission notice, so taking the first substantial line names the paper | |
| # "Provided proper attribution is provided, Google hereby grants...". | |
| _BOILERPLATE = ( | |
| "provided proper attribution", "hereby grants", "permission to", "licence", | |
| "license", "copyright", "creative commons", "all rights reserved", | |
| "preprint", "under review", "accepted at", "to appear in", "proceedings of", | |
| "conference on", "workshop on", "arxiv", "doi:", "http", "www.", | |
| "equal contribution", "corresponding author", | |
| ) | |
| def _title_from_layout(doc: Any) -> str | None: | |
| """Title = the largest text on page 1. | |
| Content heuristics lose to licence blocks: "Attention Is All You Need" | |
| opens with a multi-line Google permission notice, so both "first | |
| substantial line" and keyword filtering pick a fragment of the legal text. | |
| Font size does not have that problem - the title is set larger than | |
| everything around it, by typographic convention, in every paper format. | |
| """ | |
| try: | |
| page = doc[0].get_text("dict") | |
| except Exception: # noqa: BLE001 | |
| return None | |
| spans: list[tuple[float, float, str]] = [] | |
| for block in page.get("blocks", []): | |
| for line in block.get("lines", []): | |
| # arXiv stamps the id vertically down the left margin, in a large | |
| # font - it wins on size alone. Horizontal text only. | |
| direction = line.get("dir", (1.0, 0.0)) | |
| if abs(direction[0]) < 0.9: | |
| continue | |
| for span in line.get("spans", []): | |
| text = " ".join(span.get("text", "").split()) | |
| if not text or _ARXIV_STAMP_RE.match(text): | |
| continue | |
| spans.append((round(span.get("size", 0), 1), span.get("bbox", [0, 0])[1], text)) | |
| if not spans: | |
| return None | |
| largest = max(size for size, _, _ in spans) | |
| # Join every span at (near) the largest size, in reading order - titles | |
| # routinely wrap onto two or three lines. | |
| title_spans = sorted( | |
| [(y, t) for size, y, t in spans if size >= largest - 0.6], key=lambda st: st[0] | |
| ) | |
| title = " ".join(t for _, t in title_spans).strip() | |
| return title if 10 <= len(title) <= 300 else None | |
| def _guess_title(text: str, fallback: str) -> str: | |
| """Fallback when layout gives nothing: first plausible non-boilerplate line.""" | |
| candidates: list[str] = [] | |
| for raw in text.splitlines()[:60]: | |
| line = " ".join(raw.split()) | |
| lowered = line.lower() | |
| if not (15 <= len(line) <= 250): | |
| continue | |
| if any(marker in lowered for marker in _BOILERPLATE): | |
| continue | |
| words = line.split() | |
| if len(words) < 3: | |
| continue | |
| if sum(c.isdigit() for c in line) > len(line) * 0.3: | |
| continue | |
| # Author lines are mostly capitalised short tokens separated by commas. | |
| if line.count(",") >= 3 and sum(w[:1].isupper() for w in words) > len(words) * 0.7: | |
| continue | |
| candidates.append(line) | |
| if len(candidates) >= 5: | |
| break | |
| return candidates[0] if candidates else fallback | |
| class UploadStore: | |
| """Persisted store of uploaded documents and their chunks.""" | |
| def __init__(self, directory: str = "cache/uploads", max_documents: int = 200) -> None: | |
| self.directory = Path(directory) | |
| self.directory.mkdir(parents=True, exist_ok=True) | |
| self.max_documents = max_documents | |
| def _path(self, document_id: str) -> Path: | |
| return self.directory / f"{document_id}.json" | |
| def _prune(self) -> None: | |
| files = sorted(self.directory.glob("*.json"), key=lambda f: f.stat().st_mtime) | |
| for path in files[: max(0, len(files) - self.max_documents)]: | |
| path.unlink(missing_ok=True) | |
| def add(self, data: bytes, filename: str) -> UploadedDoc: | |
| """Validate, extract, chunk and persist one PDF.""" | |
| if not data: | |
| raise UploadError("The file is empty.") | |
| if len(data) > MAX_UPLOAD_BYTES: | |
| raise UploadError( | |
| f"That file is {len(data) / 1e6:.1f} MB; the limit is " | |
| f"{MAX_UPLOAD_BYTES / 1e6:.0f} MB." | |
| ) | |
| if not data.startswith(PDF_MAGIC): | |
| raise UploadError("That does not look like a PDF.") | |
| # Content-addressed: re-uploading the same paper reuses the same id. | |
| document_id = hashlib.sha256(data).hexdigest()[:16] | |
| existing = self.get_meta(document_id) | |
| if existing is not None: | |
| return existing | |
| try: | |
| import pymupdf | |
| with pymupdf.open(stream=data, filetype="pdf") as doc: | |
| text = "\n".join(page.get_text() for page in doc) | |
| layout_title = _title_from_layout(doc) | |
| meta_title = " ".join((doc.metadata or {}).get("title", "").split()) | |
| except Exception as exc: # noqa: BLE001 | |
| raise UploadError(f"Could not read that PDF ({type(exc).__name__}).") from exc | |
| words = text.split() | |
| if len(words) < 200: | |
| raise UploadError( | |
| "Almost no text came out of that PDF - it is probably a scan. " | |
| "This needs a text-based PDF, not images." | |
| ) | |
| # Layout first, then embedded metadata, then the line heuristic. | |
| title = layout_title or (meta_title if len(meta_title) > 10 else "") or _guess_title( | |
| text, Path(filename).stem | |
| ) | |
| paper = Paper( | |
| paper_id=document_id, | |
| version=1, | |
| title=title, | |
| abstract="", | |
| authors=[], | |
| categories=[], | |
| primary_category="upload", | |
| published="", | |
| updated="", | |
| pdf_url="", | |
| full_text=text, | |
| ) | |
| chunks = chunk_paper(paper) | |
| if not chunks: | |
| raise UploadError("No usable passages could be extracted from that PDF.") | |
| meta = UploadedDoc( | |
| document_id=document_id, | |
| title=title, | |
| n_chunks=len(chunks), | |
| n_words=len(words), | |
| uploaded_at=time.time(), | |
| filename=filename, | |
| ) | |
| self._path(document_id).write_text( | |
| json.dumps( | |
| {"meta": asdict(meta), "chunks": [asdict(c) for c in chunks]}, | |
| ensure_ascii=False, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| self._prune() | |
| return meta | |
| def _load(self, document_id: str) -> dict[str, Any] | None: | |
| path = self._path(document_id) | |
| if not path.exists(): | |
| return None | |
| try: | |
| return json.loads(path.read_text(encoding="utf-8")) | |
| except (OSError, json.JSONDecodeError): | |
| return None | |
| def get_meta(self, document_id: str) -> UploadedDoc | None: | |
| payload = self._load(document_id) | |
| return UploadedDoc(**payload["meta"]) if payload else None | |
| def passages(self, document_id: str) -> list[dict[str, Any]]: | |
| """Every chunk of one document, in the shape the LLM layer expects.""" | |
| payload = self._load(document_id) | |
| if not payload: | |
| return [] | |
| title = payload["meta"]["title"] | |
| return [ | |
| { | |
| "chunk_id": c["chunk_id"], | |
| "doc_id": c["chunk_id"], | |
| "paper_id": document_id, | |
| "arxiv_url": "", | |
| "score": 0.0, | |
| "title": title, | |
| "section": c["section"], | |
| "published": "", | |
| "text": c["text"], | |
| } | |
| for c in payload["chunks"] | |
| ] | |