File size: 5,599 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
"""Pooled LLM relevance judging, to recover false negatives.

The problem this solves
-----------------------
Generated-query labels mark exactly one paper relevant - the one the query was
written from. But a question about paper A is often also answered by B and C.
Those count as misses, so absolute recall reads lower than reality.

The fix is pooling, the same technique TREC uses: take the union of the top-k
results from every configuration under test, judge only that pool, and treat
anything unjudged as non-relevant. You never judge the whole corpus, only the
few hundred documents any system actually surfaced.

Honesty boundary
----------------
The judgements here come from a model, so metrics computed against them are a
SYNTHETIC signal. They are reported as a secondary number, never as the
headline. The generated-query labels stay primary because they are structural -
the query was written from that paper - whereas these are an opinion.

Pooling is also biased toward what current systems retrieve: a paper no
configuration surfaced is never judged, so it can never be counted relevant.
That inflates all systems equally and understates how much is being missed.
"""

from __future__ import annotations

import json
import threading
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any

from src.api.llm import _complete

JUDGE_SYSTEM_PROMPT = """You judge whether a research paper is relevant to a search \
query, the way a careful researcher would.

RELEVANT means: someone who typed this query would consider this paper a useful \
result. It addresses the same problem, method, or application area.

NOT_RELEVANT means: the paper is about something else. Sharing generic vocabulary \
("neural network", "training", "benchmark") is not relevance.

Be strict. Most papers in a search result are not relevant.

Answer with exactly one word: RELEVANT or NOT_RELEVANT."""


def _judge_one(query: str, title: str, abstract: str, model: str, max_tokens: int) -> int:
    user = (
        f"Query: {query}\n\n"
        f"Paper title: {title}\n"
        f"Paper abstract: {abstract[:1400]}\n\n"
        "RELEVANT or NOT_RELEVANT?"
    )
    try:
        out = _complete(JUDGE_SYSTEM_PROMPT, user, model, max_tokens, 0.0)
    except Exception:  # noqa: BLE001 - an unjudged pair is simply non-relevant
        return 0
    return 1 if "NOT_RELEVANT" not in out.upper() and "RELEVANT" in out.upper() else 0


def build_pool(
    runs: dict[str, dict[str, dict[str, float]]], depth: int = 10
) -> dict[str, set[str]]:
    """Union of the top-`depth` papers each configuration returned, per query."""
    pool: dict[str, set[str]] = {}
    for run in runs.values():
        for qid, hits in run.items():
            top = sorted(hits.items(), key=lambda kv: kv[1], reverse=True)[:depth]
            pool.setdefault(qid, set()).update(pid for pid, _ in top)
    return pool


def judge_pool(
    pool: dict[str, set[str]],
    queries: dict[str, str],
    papers_by_id: dict[str, dict[str, Any]],
    base_qrels: dict[str, dict[str, int]],
    model: str,
    out_path: str | Path,
    max_tokens: int = 700,
    workers: int = 8,
) -> dict[str, dict[str, int]]:
    """Judge every pooled (query, paper) pair. Resumable; cached to disk."""
    target = Path(out_path)
    target.parent.mkdir(parents=True, exist_ok=True)

    cache: dict[str, int] = {}
    if target.exists():
        for line in target.read_text(encoding="utf-8").splitlines():
            if line.strip():
                rec = json.loads(line)
                cache[f"{rec['query_id']}|{rec['paper_id']}"] = rec["relevant"]
        print(f"[judge] resuming: {len(cache)} cached judgements")

    tasks: list[tuple[str, str]] = []
    for qid, paper_ids in pool.items():
        for pid in paper_ids:
            # The structural gold is already known - never spend a call on it.
            if pid in base_qrels.get(qid, {}):
                continue
            if f"{qid}|{pid}" in cache:
                continue
            if pid in papers_by_id:
                tasks.append((qid, pid))

    print(f"[judge] {len(tasks)} pairs to judge across {len(pool)} queries")
    lock = threading.Lock()
    handle = open(target, "a", encoding="utf-8")
    done = 0

    def work(item: tuple[str, str]) -> None:
        nonlocal done
        qid, pid = item
        meta = papers_by_id[pid]
        verdict = _judge_one(
            queries[qid], meta.get("title", ""), meta.get("abstract", ""), model, max_tokens
        )
        with lock:
            cache[f"{qid}|{pid}"] = verdict
            handle.write(
                json.dumps({"query_id": qid, "paper_id": pid, "relevant": verdict}) + "\n"
            )
            handle.flush()
            done += 1
            if done % 100 == 0:
                print(f"[judge] {done}/{len(tasks)}")

    try:
        if tasks:
            with ThreadPoolExecutor(max_workers=workers) as pool_exec:
                list(pool_exec.map(work, tasks))
    finally:
        handle.close()

    # Structural gold stays relevant; pooled positives are added on top.
    merged: dict[str, dict[str, int]] = {
        qid: dict(rels) for qid, rels in base_qrels.items()
    }
    added = 0
    for key, relevant in cache.items():
        qid, pid = key.split("|", 1)
        if relevant and qid in merged and pid not in merged[qid]:
            merged[qid][pid] = 1
            added += 1

    per_query = added / max(1, len(merged))
    print(f"[judge] added {added} relevant papers ({per_query:.2f} extra per query)")
    return merged