ResearchOS / src /data /chunking.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
7.25 kB
"""Section-aware chunking for full-text papers.
FR2 in REQUIREMENTS-ML.md. Three things this has to get right:
1. Chunks must fit the reranker window (384 tokens ~= 280 words), or the tail of
every chunk is silently truncated at scoring time and never contributes.
2. Every chunk carries paper id, section name and character offsets, so a
citation can point at a span rather than waving at a whole paper.
3. Chunking is deterministic - re-ingesting a paper yields identical chunks, so
the index and any evaluation over it stay reproducible.
References sections are dropped. They are pure citation strings: high lexical
overlap with every query, no answer content, and a reliable source of false
positives for BM25.
"""
from __future__ import annotations
import json
import re
from dataclasses import asdict, dataclass
from pathlib import Path
from src.data.arxiv_source import Paper
# Headings we expect in an ML paper. Matched case-insensitively at line start,
# optionally numbered ("3.1 Method").
SECTION_PATTERN = re.compile(
r"^\s*(?:\d+(?:\.\d+)*\.?\s+)?"
r"(abstract|introduction|background|related\s+work|preliminaries|method(?:s|ology)?|"
r"approach|model|architecture|experiment(?:s|al\s+setup)?|results|evaluation|"
r"analysis|ablation(?:\s+stud(?:y|ies))?|discussion|limitations|conclusion(?:s)?|"
r"future\s+work|acknowledg(?:e)?ments|references|bibliography|appendix)"
r"\s*$",
re.IGNORECASE | re.MULTILINE,
)
# Everything from here on is not answer content.
STOP_SECTIONS = {"references", "bibliography", "acknowledgments", "acknowledgements"}
# Skipped when chunking full text. The abstract is already indexed verbatim from
# the arXiv metadata, so taking it again from the PDF produces a near-duplicate
# chunk that competes with itself in the ranking. "frontmatter" is the title
# block: author names and affiliations, no answer content.
SKIP_IN_FULL_TEXT = {"abstract", "frontmatter"}
@dataclass
class Chunk:
"""One retrievable unit."""
chunk_id: str
paper_id: str
version: int
title: str
section: str
text: str
start: int
end: int
published: str
primary_category: str
def to_corpus_entry(self) -> dict[str, str]:
"""The shape the retrieval stack expects: title + text, plus metadata."""
return {
"title": self.title,
"text": self.text,
"paper_id": self.paper_id,
"section": self.section,
"published": self.published,
"primary_category": self.primary_category,
}
def split_sections(text: str) -> list[tuple[str, str, int]]:
"""Split full text into (section_name, section_text, start_offset).
Falls back to one unnamed section when no headings are found, which happens
with two-column PDFs whose text extraction interleaves badly.
"""
matches = list(SECTION_PATTERN.finditer(text))
if not matches:
return [("body", text, 0)]
sections: list[tuple[str, str, int]] = []
if matches[0].start() > 0:
sections.append(("frontmatter", text[: matches[0].start()], 0))
for i, match in enumerate(matches):
name = re.sub(r"\s+", " ", match.group(1)).strip().lower()
if name in STOP_SECTIONS:
break # everything after references is discarded
start = match.end()
end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
body = text[start:end].strip()
if body:
sections.append((name, body, start))
return sections
def _split_words(text: str, max_words: int, overlap_words: int) -> list[tuple[str, int, int]]:
"""Sliding window over words. Returns (text, word_start, word_end) windows."""
words = text.split()
if not words:
return []
if len(words) <= max_words:
return [(" ".join(words), 0, len(words))]
stride = max(1, max_words - overlap_words)
windows: list[tuple[str, int, int]] = []
for start in range(0, len(words), stride):
window = words[start : start + max_words]
if len(window) < overlap_words and windows:
break # trailing scrap already covered by the previous window's overlap
windows.append((" ".join(window), start, start + len(window)))
if start + max_words >= len(words):
break
return windows
def chunk_paper(
paper: Paper,
max_words: int = 220,
overlap_words: int = 40,
min_words: int = 25,
) -> list[Chunk]:
"""Turn one paper into chunks. The abstract is always chunk 0.
max_words defaults to 220 because the reranker window is 384 *tokens*, and
English averages roughly 1.3 tokens per word - 220 words lands near 290
tokens, leaving room for the query and special tokens on the same input.
"""
chunks: list[Chunk] = []
def add(section: str, text: str, start: int, end: int) -> None:
text = re.sub(r"\s+", " ", text).strip()
if len(text.split()) < min_words:
return
chunks.append(
Chunk(
chunk_id=f"{paper.paper_id}#{len(chunks)}",
paper_id=paper.paper_id,
version=paper.version,
title=paper.title,
section=section,
text=text,
start=start,
end=end,
published=paper.published,
primary_category=paper.primary_category,
)
)
# The abstract is the highest-signal passage in any paper - always index it.
add("abstract", paper.abstract, 0, len(paper.abstract))
if paper.full_text:
for name, body, offset in split_sections(paper.full_text):
if name in SKIP_IN_FULL_TEXT:
continue
for window_text, w_start, w_end in _split_words(body, max_words, overlap_words):
add(name, window_text, offset + w_start, offset + w_end)
return chunks
def chunk_papers(papers: list[Paper], **kwargs: int) -> list[Chunk]:
"""Chunk a whole corpus, reporting the shape of the result."""
chunks: list[Chunk] = []
for paper in papers:
chunks.extend(chunk_paper(paper, **kwargs))
with_text = sum(1 for p in papers if p.full_text)
per_paper = len(chunks) / max(1, len(papers))
print(
f"[chunk] {len(chunks)} chunks from {len(papers)} papers "
f"({with_text} with full text, {per_paper:.1f} chunks/paper)"
)
return chunks
def write_chunks(chunks: list[Chunk], path: str | Path) -> None:
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
with open(target, "w", encoding="utf-8") as handle:
for chunk in chunks:
handle.write(json.dumps(asdict(chunk), ensure_ascii=False) + "\n")
print(f"[chunk] wrote {len(chunks)} chunks -> {target}")
def read_chunks(path: str | Path) -> list[Chunk]:
with open(path, encoding="utf-8") as handle:
return [Chunk(**json.loads(line)) for line in handle if line.strip()]
def chunks_to_corpus(chunks: list[Chunk]) -> dict[str, dict[str, str]]:
"""Adapt chunks to the Corpus shape the existing retrieval stack consumes."""
return {chunk.chunk_id: chunk.to_corpus_entry() for chunk in chunks}