File size: 8,586 Bytes
387a4cf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
"""Prepare immutable, paper-disjoint inputs from the published HyperAgents CSV.

Global context: only the trusted host reads labeled files. The model receives a
formatted paper, an abstract, or selected training feedback through the broker.
"""

from __future__ import annotations

import ast
import csv
import hashlib
import io
import json
import re
import unicodedata
import urllib.request
from collections import Counter
from pathlib import Path

REVISION = "59a68f672dfb92c74aeb7e61535d776fb36e172d"
DATA_URL = f"https://raw.githubusercontent.com/facebookresearch/HyperAgents/{REVISION}/domains/paper_review/dataset.csv"
DATA_HASH = "efa8e77721d6d41c5d9ea69aaab8f0c16664618876e2c9dfe897a3852bb242c9"
# Mastheads recur inside converted PDFs. Match just the status phrase, including
# its venue, year and nearby spacing, to preserve adjacent scientific text.
STATUS = re.compile(
    r"(?:published\s+as|under\s+review\s+as|accepted\s+as)\s+(?:an?\s+)?(?:conference|workshop|spotlight|oral|poster)\s+paper(?:\s+(?:at|for|in|to)\s+(?:ICLR|NeurIPS|ICML|[A-Z]{2,12})(?:\s+20\d{2})?)?",
    re.I,
)
METALINE = re.compile(
    r"(?im)^[ \t]*(?:final decision|acceptance decision|decision|meta[- ]review|openreview (?:id|forum)|venue status)\s*:[^\n]*$"
)
BLIND_HEADER = re.compile(
    r"(?:anonymous\s+authors?\s*)?paper\s+under\s+double[\s-]+blind\s+review", re.I
)
LINK = re.compile(
    r"https?\s*:\s*//\s*(?:www\.)?openreview\.net/(?:forum|pdf)\?id=[\w-]+", re.I
)
MANUSCRIPT = re.compile(
    r"\ATitle:\s*(.+?)\s*\n+\s*Abstract:\s*(.+?)(?=\n+\s*Section:|\Z)", re.S | re.I
)
SIZES = {"train": 100, "validation": 100, "test": 100, "pilot": 24}


def digest(value: str | bytes) -> str:
    """Return a content identity independent of platform newlines and Python hash seeds."""
    return hashlib.sha256(
        value.encode("utf-8") if isinstance(value, str) else value
    ).hexdigest()


def canonical(value: object) -> str:
    """Use one unambiguous representation for published hashes and cache keys."""
    return json.dumps(
        value,
        ensure_ascii=False,
        sort_keys=True,
        separators=(",", ":"),
        allow_nan=False,
    )


def dump(path: Path, value: object) -> None:
    """Atomically publish readable JSON so observers see whole snapshots."""
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_suffix(path.suffix + ".part")
    tmp.write_text(
        json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n",
        encoding="utf-8",
    )
    tmp.replace(path)


def load(path: Path):
    """Read a trusted JSON artifact."""
    return json.loads(path.read_text(encoding="utf-8"))


def read_split(project: Path, split: str):
    """Verify manuscript contents, extracted writer inputs, labels and registered order."""
    papers = load(project / ".private" / "data" / (split + ".json"))
    manifest = load(project / "results" / "split-manifest.json")["splits"][split]
    identity = [(p["id"], digest(p["text"]), p["label"]) for p in papers]
    if digest(canonical(identity)) != manifest["sha256"]:
        raise ValueError(f"{split} split fingerprint mismatch")
    for paper in papers:
        parsed = MANUSCRIPT.search(paper["text"])
        if not parsed or tuple(s.strip() for s in parsed.groups()) != (
            paper["title"],
            paper["abstract"],
        ):
            raise ValueError("Writer inputs differ from the fingerprinted manuscript")
    return papers


def clean(raw: str) -> str:
    """Decode the released Python string representation before removing outcome cues."""
    if raw.startswith(("'", '"')):
        try:
            raw = ast.literal_eval(raw)
        except (ValueError, SyntaxError):
            raw = raw.strip("'\"").replace("\\n", "\n")
    raw = unicodedata.normalize("NFKC", raw).replace("\r\n", "\n")
    return LINK.sub(
        "", METALINE.sub("", BLIND_HEADER.sub("", STATUS.sub("", raw)))
    ).strip()


def words(text: str) -> set[str]:
    """Normalize title and abstract words for duplicate detection."""
    return set(re.findall(r"\w+", text.casefold()))


def similar(left: set[str], right: set[str], threshold: float) -> bool:
    """Jaccard overlap rejects unrelated papers sharing a short generic title."""
    return bool(left and right) and len(left & right) >= threshold * len(left | right)


def unique_papers(rows):
    """Group versions transitively by ID, title, body, or near-matching title and abstract."""
    items, groups, keys, signatures = [], [], {}, []
    missing = 0

    def root(index):
        while groups[index] != index:
            groups[index] = groups[groups[index]]
            index = groups[index]
        return index

    for row in rows:
        body = clean(row["paper_text"])
        match = MANUSCRIPT.search(body)
        if not match or row["outcome"] not in {"accept", "reject"}:
            missing += 1
            continue
        title, abstract = (s.strip() for s in match.groups())
        index = len(items)
        groups.append(index)
        title_set, abstract_set = words(title), words(abstract)
        for key in (
            "id:" + row["question_id"],
            "title:" + " ".join(re.findall(r"\w+", title.casefold())),
            "body:" + digest(body),
        ):
            if key in keys:
                groups[root(index)] = root(keys[key])
            keys[key] = index
        for other, (t, a) in enumerate(signatures):
            if similar(t, title_set, 0.75) and similar(a, abstract_set, 0.82):
                groups[root(index)] = root(other)
        signatures.append((title_set, abstract_set))
        items.append(
            dict(
                id=row["question_id"],
                title=title,
                abstract=abstract,
                text=body,
                label=row["outcome"],
                text_sha256=digest(body),
            )
        )
    components = {}
    for i, paper in enumerate(items):
        components.setdefault(root(i), []).append(paper)
    # Conflicting historical outcomes cannot define a unique group target.
    stable = [
        min(group, key=lambda x: x["id"])
        for group in components.values()
        if len({x["label"] for x in group}) == 1
    ]
    return stable, dict(
        parsed=len(items),
        excluded_parse_or_label=missing,
        groups=len(components),
        retained_groups=len(stable),
    )


def prepare(project: Path) -> dict:
    """Download, verify, split and fingerprint the corpus before any model calls."""
    private = project / ".private" / "data"
    private.mkdir(parents=True, exist_ok=True)
    source = private / "source.csv"
    if not source.exists():
        with urllib.request.urlopen(DATA_URL, timeout=120) as response:
            payload = response.read()
        if digest(payload) != DATA_HASH:
            raise ValueError("Released CSV checksum mismatch")
        temporary = source.with_suffix(".part")
        temporary.write_bytes(payload)
        temporary.replace(source)
    raw = source.read_bytes()
    if digest(raw) != DATA_HASH:
        raise ValueError("Local CSV checksum mismatch")
    csv.field_size_limit(16 * 1024 * 1024)
    papers, audit = unique_papers(
        csv.DictReader(io.StringIO(raw.decode("utf-8-sig"), newline=""))
    )
    by_label = {
        label: sorted(
            (p for p in papers if p["label"] == label),
            key=lambda p: digest("rqgm-split-v1/42/" + p["id"]),
        )
        for label in ("accept", "reject")
    }
    manifest = dict(source=DATA_URL, source_sha256=DATA_HASH, audit=audit, splits={})
    offset = 0
    for name, count in SIZES.items():
        subset = sorted(
            [
                p
                for pool in by_label.values()
                for p in pool[offset : offset + count // 2]
            ],
            key=lambda p: digest("order/" + p["id"]),
        )
        if Counter(p["label"] for p in subset) != {
            "accept": count // 2,
            "reject": count // 2,
        }:
            raise ValueError(f"Insufficient eligible data for {name}")
        dump(private / f"{name}.json", subset)
        ident = [(p["id"], p["text_sha256"], p["label"]) for p in subset]
        manifest["splits"][name] = dict(
            count=count, sha256=digest(canonical(ident)), ids=[p["id"] for p in subset]
        )
        offset += count // 2
    dump(project / "results" / "split-manifest.json", manifest)
    return manifest


if __name__ == "__main__":
    print({k: v["sha256"] for k, v in prepare(Path.cwd())["splits"].items()})