Spaces:
Running on Zero
Running on Zero
File size: 2,087 Bytes
e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """Per-paper result diversification.
FR4.2 in REQUIREMENTS-ML.md, and a requirement that did not exist in the parent
project. On SciFact one document was one abstract, so a top-5 was automatically
five distinct sources. Here a paper yields dozens of chunks, and the five
best-scoring chunks for "explain JEPA" are frequently five consecutive
paragraphs of the same paper - which is a straight failure for any question that
needs more than one source.
Capping chunks per paper costs a little precision on single-source questions and
buys the multi-source coverage that "explain" and "compare" require.
"""
from __future__ import annotations
from typing import Any
Run = dict[str, dict[str, float]]
def cap_per_paper(
scored: dict[str, float],
corpus: dict[str, dict[str, Any]],
top_k: int,
max_per_paper: int = 2,
) -> list[tuple[str, float]]:
"""Take the best `top_k` chunks, allowing at most `max_per_paper` from any paper.
Greedy over the score-sorted list: keep a chunk unless its paper already has
`max_per_paper` entries. Order within the result is still by score.
"""
ordered = sorted(scored.items(), key=lambda kv: kv[1], reverse=True)
seen: dict[str, int] = {}
kept: list[tuple[str, float]] = []
overflow: list[tuple[str, float]] = []
for chunk_id, score in ordered:
paper_id = corpus.get(chunk_id, {}).get("paper_id", chunk_id)
if seen.get(paper_id, 0) < max_per_paper:
seen[paper_id] = seen.get(paper_id, 0) + 1
kept.append((chunk_id, score))
if len(kept) == top_k:
return kept
else:
overflow.append((chunk_id, score))
# Corpus too small or too concentrated to fill top_k under the cap - top up
# rather than return a short list.
kept.extend(overflow[: top_k - len(kept)])
return kept[:top_k]
def distinct_papers(results: list[dict[str, Any]]) -> int:
"""How many distinct papers a result set spans. Used by the US2/US3 checks."""
return len({r.get("paper_id") for r in results if r.get("paper_id")})
|