Spaces:
Running on Zero
Running on Zero
File size: 5,599 Bytes
e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | """Pooled LLM relevance judging, to recover false negatives.
The problem this solves
-----------------------
Generated-query labels mark exactly one paper relevant - the one the query was
written from. But a question about paper A is often also answered by B and C.
Those count as misses, so absolute recall reads lower than reality.
The fix is pooling, the same technique TREC uses: take the union of the top-k
results from every configuration under test, judge only that pool, and treat
anything unjudged as non-relevant. You never judge the whole corpus, only the
few hundred documents any system actually surfaced.
Honesty boundary
----------------
The judgements here come from a model, so metrics computed against them are a
SYNTHETIC signal. They are reported as a secondary number, never as the
headline. The generated-query labels stay primary because they are structural -
the query was written from that paper - whereas these are an opinion.
Pooling is also biased toward what current systems retrieve: a paper no
configuration surfaced is never judged, so it can never be counted relevant.
That inflates all systems equally and understates how much is being missed.
"""
from __future__ import annotations
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any
from src.api.llm import _complete
JUDGE_SYSTEM_PROMPT = """You judge whether a research paper is relevant to a search \
query, the way a careful researcher would.
RELEVANT means: someone who typed this query would consider this paper a useful \
result. It addresses the same problem, method, or application area.
NOT_RELEVANT means: the paper is about something else. Sharing generic vocabulary \
("neural network", "training", "benchmark") is not relevance.
Be strict. Most papers in a search result are not relevant.
Answer with exactly one word: RELEVANT or NOT_RELEVANT."""
def _judge_one(query: str, title: str, abstract: str, model: str, max_tokens: int) -> int:
user = (
f"Query: {query}\n\n"
f"Paper title: {title}\n"
f"Paper abstract: {abstract[:1400]}\n\n"
"RELEVANT or NOT_RELEVANT?"
)
try:
out = _complete(JUDGE_SYSTEM_PROMPT, user, model, max_tokens, 0.0)
except Exception: # noqa: BLE001 - an unjudged pair is simply non-relevant
return 0
return 1 if "NOT_RELEVANT" not in out.upper() and "RELEVANT" in out.upper() else 0
def build_pool(
runs: dict[str, dict[str, dict[str, float]]], depth: int = 10
) -> dict[str, set[str]]:
"""Union of the top-`depth` papers each configuration returned, per query."""
pool: dict[str, set[str]] = {}
for run in runs.values():
for qid, hits in run.items():
top = sorted(hits.items(), key=lambda kv: kv[1], reverse=True)[:depth]
pool.setdefault(qid, set()).update(pid for pid, _ in top)
return pool
def judge_pool(
pool: dict[str, set[str]],
queries: dict[str, str],
papers_by_id: dict[str, dict[str, Any]],
base_qrels: dict[str, dict[str, int]],
model: str,
out_path: str | Path,
max_tokens: int = 700,
workers: int = 8,
) -> dict[str, dict[str, int]]:
"""Judge every pooled (query, paper) pair. Resumable; cached to disk."""
target = Path(out_path)
target.parent.mkdir(parents=True, exist_ok=True)
cache: dict[str, int] = {}
if target.exists():
for line in target.read_text(encoding="utf-8").splitlines():
if line.strip():
rec = json.loads(line)
cache[f"{rec['query_id']}|{rec['paper_id']}"] = rec["relevant"]
print(f"[judge] resuming: {len(cache)} cached judgements")
tasks: list[tuple[str, str]] = []
for qid, paper_ids in pool.items():
for pid in paper_ids:
# The structural gold is already known - never spend a call on it.
if pid in base_qrels.get(qid, {}):
continue
if f"{qid}|{pid}" in cache:
continue
if pid in papers_by_id:
tasks.append((qid, pid))
print(f"[judge] {len(tasks)} pairs to judge across {len(pool)} queries")
lock = threading.Lock()
handle = open(target, "a", encoding="utf-8")
done = 0
def work(item: tuple[str, str]) -> None:
nonlocal done
qid, pid = item
meta = papers_by_id[pid]
verdict = _judge_one(
queries[qid], meta.get("title", ""), meta.get("abstract", ""), model, max_tokens
)
with lock:
cache[f"{qid}|{pid}"] = verdict
handle.write(
json.dumps({"query_id": qid, "paper_id": pid, "relevant": verdict}) + "\n"
)
handle.flush()
done += 1
if done % 100 == 0:
print(f"[judge] {done}/{len(tasks)}")
try:
if tasks:
with ThreadPoolExecutor(max_workers=workers) as pool_exec:
list(pool_exec.map(work, tasks))
finally:
handle.close()
# Structural gold stays relevant; pooled positives are added on top.
merged: dict[str, dict[str, int]] = {
qid: dict(rels) for qid, rels in base_qrels.items()
}
added = 0
for key, relevant in cache.items():
qid, pid = key.split("|", 1)
if relevant and qid in merged and pid not in merged[qid]:
merged[qid][pid] = 1
added += 1
per_query = added / max(1, len(merged))
print(f"[judge] added {added} relevant papers ({per_query:.2f} extra per query)")
return merged
|