File size: 4,036 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
"""Negative mining.

Two datasets are produced from exactly the same positives, in the same order,
with the same number of negatives per group. The ONLY difference is where the
negatives come from:

  hard    - passages the retriever already ranked highly for this query
            (ranks `hard_min_rank`..`top_k`), gold excluded. These look right
            and are not, so the model has to learn to discriminate.
  random  - passages drawn uniformly from the corpus, gold excluded. Almost
            always trivially unrelated.

Rows 4 and 5 of the results table differ only in which of these files is used.
"""

from __future__ import annotations

import json
import random
from pathlib import Path
from typing import Any

from src.data.loader import Corpus, Qrels, passage_text

Run = dict[str, dict[str, float]]


def _ranked_ids(candidates: dict[str, float]) -> list[str]:
    return [d for d, _ in sorted(candidates.items(), key=lambda kv: kv[1], reverse=True)]


def build_groups(
    corpus: Corpus,
    queries: dict[str, str],
    qrels: Qrels,
    run: Run,
    num_negatives: int,
    hard_min_rank: int,
    rng: random.Random,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    """Return (hard_groups, random_groups) - parallel lists sharing positives."""
    all_doc_ids = list(corpus.keys())
    hard_groups: list[dict[str, Any]] = []
    random_groups: list[dict[str, Any]] = []
    short_pools = 0

    for qid, query_text in queries.items():
        gold_ids = [d for d, rel in qrels.get(qid, {}).items() if rel > 0 and d in corpus]
        if not gold_ids:
            continue

        candidates = _ranked_ids(run.get(qid, {}))
        # Everything the retriever surfaced at or below hard_min_rank that is not gold.
        hard_pool = [d for d in candidates[hard_min_rank - 1 :] if d not in gold_ids]

        for positive_id in gold_ids:
            if len(hard_pool) >= num_negatives:
                hard_ids = rng.sample(hard_pool, num_negatives)
            else:
                short_pools += 1
                hard_ids = list(hard_pool)
                while len(hard_ids) < num_negatives:
                    pick = rng.choice(all_doc_ids)
                    if pick not in gold_ids and pick not in hard_ids:
                        hard_ids.append(pick)

            random_ids: list[str] = []
            while len(random_ids) < num_negatives:
                pick = rng.choice(all_doc_ids)
                if pick not in gold_ids and pick not in random_ids:
                    random_ids.append(pick)

            base = {
                "query_id": qid,
                "query": query_text,
                "positive_id": positive_id,
                "positive": passage_text(corpus, positive_id),
            }
            hard_groups.append(
                {
                    **base,
                    "negative_ids": hard_ids,
                    "negatives": [passage_text(corpus, d) for d in hard_ids],
                }
            )
            random_groups.append(
                {
                    **base,
                    "negative_ids": random_ids,
                    "negatives": [passage_text(corpus, d) for d in random_ids],
                }
            )

    if short_pools:
        print(f"[mine] {short_pools} groups had a short hard pool and were topped up randomly")
    return hard_groups, random_groups


def write_jsonl(groups: list[dict[str, Any]], path: str | Path) -> None:
    """Write one JSON object per line."""
    target = Path(path)
    target.parent.mkdir(parents=True, exist_ok=True)
    with open(target, "w", encoding="utf-8") as handle:
        for group in groups:
            handle.write(json.dumps(group, ensure_ascii=False) + "\n")
    print(f"[mine] wrote {len(groups)} groups -> {target}")


def read_jsonl(path: str | Path) -> list[dict[str, Any]]:
    """Read a JSONL file back into a list of dicts."""
    with open(path, encoding="utf-8") as handle:
        return [json.loads(line) for line in handle if line.strip()]