Spaces:
Running on Zero
Running on Zero
Download src/data/chunking.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 7.25 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/chunking.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/data/chunking.py
-
curl -L -o chunking.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/chunking.py
7.25 kB
| """Section-aware chunking for full-text papers. | |
| FR2 in REQUIREMENTS-ML.md. Three things this has to get right: | |
| 1. Chunks must fit the reranker window (384 tokens ~= 280 words), or the tail of | |
| every chunk is silently truncated at scoring time and never contributes. | |
| 2. Every chunk carries paper id, section name and character offsets, so a | |
| citation can point at a span rather than waving at a whole paper. | |
| 3. Chunking is deterministic - re-ingesting a paper yields identical chunks, so | |
| the index and any evaluation over it stay reproducible. | |
| References sections are dropped. They are pure citation strings: high lexical | |
| overlap with every query, no answer content, and a reliable source of false | |
| positives for BM25. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| from src.data.arxiv_source import Paper | |
| # Headings we expect in an ML paper. Matched case-insensitively at line start, | |
| # optionally numbered ("3.1 Method"). | |
| SECTION_PATTERN = re.compile( | |
| r"^\s*(?:\d+(?:\.\d+)*\.?\s+)?" | |
| r"(abstract|introduction|background|related\s+work|preliminaries|method(?:s|ology)?|" | |
| r"approach|model|architecture|experiment(?:s|al\s+setup)?|results|evaluation|" | |
| r"analysis|ablation(?:\s+stud(?:y|ies))?|discussion|limitations|conclusion(?:s)?|" | |
| r"future\s+work|acknowledg(?:e)?ments|references|bibliography|appendix)" | |
| r"\s*$", | |
| re.IGNORECASE | re.MULTILINE, | |
| ) | |
| # Everything from here on is not answer content. | |
| STOP_SECTIONS = {"references", "bibliography", "acknowledgments", "acknowledgements"} | |
| # Skipped when chunking full text. The abstract is already indexed verbatim from | |
| # the arXiv metadata, so taking it again from the PDF produces a near-duplicate | |
| # chunk that competes with itself in the ranking. "frontmatter" is the title | |
| # block: author names and affiliations, no answer content. | |
| SKIP_IN_FULL_TEXT = {"abstract", "frontmatter"} | |
| class Chunk: | |
| """One retrievable unit.""" | |
| chunk_id: str | |
| paper_id: str | |
| version: int | |
| title: str | |
| section: str | |
| text: str | |
| start: int | |
| end: int | |
| published: str | |
| primary_category: str | |
| def to_corpus_entry(self) -> dict[str, str]: | |
| """The shape the retrieval stack expects: title + text, plus metadata.""" | |
| return { | |
| "title": self.title, | |
| "text": self.text, | |
| "paper_id": self.paper_id, | |
| "section": self.section, | |
| "published": self.published, | |
| "primary_category": self.primary_category, | |
| } | |
| def split_sections(text: str) -> list[tuple[str, str, int]]: | |
| """Split full text into (section_name, section_text, start_offset). | |
| Falls back to one unnamed section when no headings are found, which happens | |
| with two-column PDFs whose text extraction interleaves badly. | |
| """ | |
| matches = list(SECTION_PATTERN.finditer(text)) | |
| if not matches: | |
| return [("body", text, 0)] | |
| sections: list[tuple[str, str, int]] = [] | |
| if matches[0].start() > 0: | |
| sections.append(("frontmatter", text[: matches[0].start()], 0)) | |
| for i, match in enumerate(matches): | |
| name = re.sub(r"\s+", " ", match.group(1)).strip().lower() | |
| if name in STOP_SECTIONS: | |
| break # everything after references is discarded | |
| start = match.end() | |
| end = matches[i + 1].start() if i + 1 < len(matches) else len(text) | |
| body = text[start:end].strip() | |
| if body: | |
| sections.append((name, body, start)) | |
| return sections | |
| def _split_words(text: str, max_words: int, overlap_words: int) -> list[tuple[str, int, int]]: | |
| """Sliding window over words. Returns (text, word_start, word_end) windows.""" | |
| words = text.split() | |
| if not words: | |
| return [] | |
| if len(words) <= max_words: | |
| return [(" ".join(words), 0, len(words))] | |
| stride = max(1, max_words - overlap_words) | |
| windows: list[tuple[str, int, int]] = [] | |
| for start in range(0, len(words), stride): | |
| window = words[start : start + max_words] | |
| if len(window) < overlap_words and windows: | |
| break # trailing scrap already covered by the previous window's overlap | |
| windows.append((" ".join(window), start, start + len(window))) | |
| if start + max_words >= len(words): | |
| break | |
| return windows | |
| def chunk_paper( | |
| paper: Paper, | |
| max_words: int = 220, | |
| overlap_words: int = 40, | |
| min_words: int = 25, | |
| ) -> list[Chunk]: | |
| """Turn one paper into chunks. The abstract is always chunk 0. | |
| max_words defaults to 220 because the reranker window is 384 *tokens*, and | |
| English averages roughly 1.3 tokens per word - 220 words lands near 290 | |
| tokens, leaving room for the query and special tokens on the same input. | |
| """ | |
| chunks: list[Chunk] = [] | |
| def add(section: str, text: str, start: int, end: int) -> None: | |
| text = re.sub(r"\s+", " ", text).strip() | |
| if len(text.split()) < min_words: | |
| return | |
| chunks.append( | |
| Chunk( | |
| chunk_id=f"{paper.paper_id}#{len(chunks)}", | |
| paper_id=paper.paper_id, | |
| version=paper.version, | |
| title=paper.title, | |
| section=section, | |
| text=text, | |
| start=start, | |
| end=end, | |
| published=paper.published, | |
| primary_category=paper.primary_category, | |
| ) | |
| ) | |
| # The abstract is the highest-signal passage in any paper - always index it. | |
| add("abstract", paper.abstract, 0, len(paper.abstract)) | |
| if paper.full_text: | |
| for name, body, offset in split_sections(paper.full_text): | |
| if name in SKIP_IN_FULL_TEXT: | |
| continue | |
| for window_text, w_start, w_end in _split_words(body, max_words, overlap_words): | |
| add(name, window_text, offset + w_start, offset + w_end) | |
| return chunks | |
| def chunk_papers(papers: list[Paper], **kwargs: int) -> list[Chunk]: | |
| """Chunk a whole corpus, reporting the shape of the result.""" | |
| chunks: list[Chunk] = [] | |
| for paper in papers: | |
| chunks.extend(chunk_paper(paper, **kwargs)) | |
| with_text = sum(1 for p in papers if p.full_text) | |
| per_paper = len(chunks) / max(1, len(papers)) | |
| print( | |
| f"[chunk] {len(chunks)} chunks from {len(papers)} papers " | |
| f"({with_text} with full text, {per_paper:.1f} chunks/paper)" | |
| ) | |
| return chunks | |
| def write_chunks(chunks: list[Chunk], path: str | Path) -> None: | |
| target = Path(path) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| with open(target, "w", encoding="utf-8") as handle: | |
| for chunk in chunks: | |
| handle.write(json.dumps(asdict(chunk), ensure_ascii=False) + "\n") | |
| print(f"[chunk] wrote {len(chunks)} chunks -> {target}") | |
| def read_chunks(path: str | Path) -> list[Chunk]: | |
| with open(path, encoding="utf-8") as handle: | |
| return [Chunk(**json.loads(line)) for line in handle if line.strip()] | |
| def chunks_to_corpus(chunks: list[Chunk]) -> dict[str, dict[str, str]]: | |
| """Adapt chunks to the Corpus shape the existing retrieval stack consumes.""" | |
| return {chunk.chunk_id: chunk.to_corpus_entry() for chunk in chunks} | |