ResearchOS / scripts /mine_negatives.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
3.34 kB
"""Stage 3 - build hard_negatives.jsonl and random_negatives.jsonl.
python scripts/mine_negatives.py --config configs/mine_negatives.yaml
"""
from __future__ import annotations
import argparse
import json
import random
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.config import device, load_config, report_vram, seed_everything
from src.data.loader import flatten_corpus, load_split
from src.data.negatives import build_groups, write_jsonl
from src.retrieval.bm25 import BM25Retriever
from src.retrieval.dense import DenseRetriever
from src.retrieval.fusion import reciprocal_rank_fusion
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", required=True)
args = parser.parse_args()
cfg = load_config(args.config)
seed = seed_everything(cfg["seed"])
rng = random.Random(seed)
print(f"[seed] {seed}")
corpus, queries, qrels = load_split(
cfg["dataset"]["url"], cfg["dataset"]["split"], cfg["dataset"]["download_dir"]
)
print(f"[data] train split: {len(queries)} queries | {len(corpus)} docs")
doc_ids, doc_texts = flatten_corpus(corpus)
top_k = cfg["retrieval"]["top_k"]
cache_dir = cfg["paths"]["cache_dir"]
run_path = Path(cfg["paths"]["train_run"])
if run_path.exists():
fused = json.loads(run_path.read_text(encoding="utf-8"))
print(f"[cache] reusing {run_path}")
else:
bm25_cfg = cfg["retrieval"]["bm25"]
bm25 = BM25Retriever(k1=bm25_cfg["k1"], b=bm25_cfg["b"], stemmer_lang=bm25_cfg["stemmer"])
bm25.index(doc_ids, doc_texts, cache_dir=cache_dir)
bm25_run = bm25.search(queries, top_k=top_k)
dense_cfg = cfg["retrieval"]["dense"]
dense = DenseRetriever(
model_name=dense_cfg["model"],
batch_size=dense_cfg["batch_size"],
max_length=dense_cfg["max_length"],
query_prefix=dense_cfg["query_prefix"],
device=device(),
)
dense.index_corpus(doc_ids, doc_texts, cache_dir=cache_dir)
dense_run = dense.search(queries, top_k=top_k)
fused = reciprocal_rank_fusion(
[bm25_run, dense_run], k=cfg["retrieval"]["fusion"]["rrf_k"], top_k=top_k
)
run_path.parent.mkdir(parents=True, exist_ok=True)
run_path.write_text(json.dumps(fused), encoding="utf-8")
print(f"[cache] train run -> {run_path}")
hard, rand = build_groups(
corpus=corpus,
queries=queries,
qrels=qrels,
run=fused,
num_negatives=cfg["mining"]["num_negatives"],
hard_min_rank=cfg["mining"]["hard_min_rank"],
rng=rng,
)
# The ablation is only meaningful if these two datasets are identical
# everywhere except the negatives.
assert len(hard) == len(rand), "hard and random datasets must be the same size"
for h, r in zip(hard, rand):
assert h["positive_id"] == r["positive_id"], "positives must match across datasets"
assert len(h["negatives"]) == len(r["negatives"]), "negative counts must match"
write_jsonl(hard, cfg["paths"]["hard_negatives"])
write_jsonl(rand, cfg["paths"]["random_negatives"])
report_vram("mine_negatives")
if __name__ == "__main__":
main()