File size: 7,246 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
"""Section-aware chunking for full-text papers.

FR2 in REQUIREMENTS-ML.md. Three things this has to get right:

1. Chunks must fit the reranker window (384 tokens ~= 280 words), or the tail of
   every chunk is silently truncated at scoring time and never contributes.
2. Every chunk carries paper id, section name and character offsets, so a
   citation can point at a span rather than waving at a whole paper.
3. Chunking is deterministic - re-ingesting a paper yields identical chunks, so
   the index and any evaluation over it stay reproducible.

References sections are dropped. They are pure citation strings: high lexical
overlap with every query, no answer content, and a reliable source of false
positives for BM25.
"""

from __future__ import annotations

import json
import re
from dataclasses import asdict, dataclass
from pathlib import Path

from src.data.arxiv_source import Paper

# Headings we expect in an ML paper. Matched case-insensitively at line start,
# optionally numbered ("3.1 Method").
SECTION_PATTERN = re.compile(
    r"^\s*(?:\d+(?:\.\d+)*\.?\s+)?"
    r"(abstract|introduction|background|related\s+work|preliminaries|method(?:s|ology)?|"
    r"approach|model|architecture|experiment(?:s|al\s+setup)?|results|evaluation|"
    r"analysis|ablation(?:\s+stud(?:y|ies))?|discussion|limitations|conclusion(?:s)?|"
    r"future\s+work|acknowledg(?:e)?ments|references|bibliography|appendix)"
    r"\s*$",
    re.IGNORECASE | re.MULTILINE,
)

# Everything from here on is not answer content.
STOP_SECTIONS = {"references", "bibliography", "acknowledgments", "acknowledgements"}

# Skipped when chunking full text. The abstract is already indexed verbatim from
# the arXiv metadata, so taking it again from the PDF produces a near-duplicate
# chunk that competes with itself in the ranking. "frontmatter" is the title
# block: author names and affiliations, no answer content.
SKIP_IN_FULL_TEXT = {"abstract", "frontmatter"}


@dataclass
class Chunk:
    """One retrievable unit."""

    chunk_id: str
    paper_id: str
    version: int
    title: str
    section: str
    text: str
    start: int
    end: int
    published: str
    primary_category: str

    def to_corpus_entry(self) -> dict[str, str]:
        """The shape the retrieval stack expects: title + text, plus metadata."""
        return {
            "title": self.title,
            "text": self.text,
            "paper_id": self.paper_id,
            "section": self.section,
            "published": self.published,
            "primary_category": self.primary_category,
        }


def split_sections(text: str) -> list[tuple[str, str, int]]:
    """Split full text into (section_name, section_text, start_offset).

    Falls back to one unnamed section when no headings are found, which happens
    with two-column PDFs whose text extraction interleaves badly.
    """
    matches = list(SECTION_PATTERN.finditer(text))
    if not matches:
        return [("body", text, 0)]

    sections: list[tuple[str, str, int]] = []
    if matches[0].start() > 0:
        sections.append(("frontmatter", text[: matches[0].start()], 0))

    for i, match in enumerate(matches):
        name = re.sub(r"\s+", " ", match.group(1)).strip().lower()
        if name in STOP_SECTIONS:
            break  # everything after references is discarded
        start = match.end()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
        body = text[start:end].strip()
        if body:
            sections.append((name, body, start))
    return sections


def _split_words(text: str, max_words: int, overlap_words: int) -> list[tuple[str, int, int]]:
    """Sliding window over words. Returns (text, word_start, word_end) windows."""
    words = text.split()
    if not words:
        return []
    if len(words) <= max_words:
        return [(" ".join(words), 0, len(words))]

    stride = max(1, max_words - overlap_words)
    windows: list[tuple[str, int, int]] = []
    for start in range(0, len(words), stride):
        window = words[start : start + max_words]
        if len(window) < overlap_words and windows:
            break  # trailing scrap already covered by the previous window's overlap
        windows.append((" ".join(window), start, start + len(window)))
        if start + max_words >= len(words):
            break
    return windows


def chunk_paper(
    paper: Paper,
    max_words: int = 220,
    overlap_words: int = 40,
    min_words: int = 25,
) -> list[Chunk]:
    """Turn one paper into chunks. The abstract is always chunk 0.

    max_words defaults to 220 because the reranker window is 384 *tokens*, and
    English averages roughly 1.3 tokens per word - 220 words lands near 290
    tokens, leaving room for the query and special tokens on the same input.
    """
    chunks: list[Chunk] = []

    def add(section: str, text: str, start: int, end: int) -> None:
        text = re.sub(r"\s+", " ", text).strip()
        if len(text.split()) < min_words:
            return
        chunks.append(
            Chunk(
                chunk_id=f"{paper.paper_id}#{len(chunks)}",
                paper_id=paper.paper_id,
                version=paper.version,
                title=paper.title,
                section=section,
                text=text,
                start=start,
                end=end,
                published=paper.published,
                primary_category=paper.primary_category,
            )
        )

    # The abstract is the highest-signal passage in any paper - always index it.
    add("abstract", paper.abstract, 0, len(paper.abstract))

    if paper.full_text:
        for name, body, offset in split_sections(paper.full_text):
            if name in SKIP_IN_FULL_TEXT:
                continue
            for window_text, w_start, w_end in _split_words(body, max_words, overlap_words):
                add(name, window_text, offset + w_start, offset + w_end)

    return chunks


def chunk_papers(papers: list[Paper], **kwargs: int) -> list[Chunk]:
    """Chunk a whole corpus, reporting the shape of the result."""
    chunks: list[Chunk] = []
    for paper in papers:
        chunks.extend(chunk_paper(paper, **kwargs))
    with_text = sum(1 for p in papers if p.full_text)
    per_paper = len(chunks) / max(1, len(papers))
    print(
        f"[chunk] {len(chunks)} chunks from {len(papers)} papers "
        f"({with_text} with full text, {per_paper:.1f} chunks/paper)"
    )
    return chunks


def write_chunks(chunks: list[Chunk], path: str | Path) -> None:
    target = Path(path)
    target.parent.mkdir(parents=True, exist_ok=True)
    with open(target, "w", encoding="utf-8") as handle:
        for chunk in chunks:
            handle.write(json.dumps(asdict(chunk), ensure_ascii=False) + "\n")
    print(f"[chunk] wrote {len(chunks)} chunks -> {target}")


def read_chunks(path: str | Path) -> list[Chunk]:
    with open(path, encoding="utf-8") as handle:
        return [Chunk(**json.loads(line)) for line in handle if line.strip()]


def chunks_to_corpus(chunks: list[Chunk]) -> dict[str, dict[str, str]]:
    """Adapt chunks to the Corpus shape the existing retrieval stack consumes."""
    return {chunk.chunk_id: chunk.to_corpus_entry() for chunk in chunks}