Spaces:
Running on Zero
Running on Zero
Download scripts/mine_negatives.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 3.34 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/mine_negatives.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/mine_negatives.py
-
curl -L -o mine_negatives.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/mine_negatives.py
3.34 kB
| """Stage 3 - build hard_negatives.jsonl and random_negatives.jsonl. | |
| python scripts/mine_negatives.py --config configs/mine_negatives.yaml | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import random | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from src.config import device, load_config, report_vram, seed_everything | |
| from src.data.loader import flatten_corpus, load_split | |
| from src.data.negatives import build_groups, write_jsonl | |
| from src.retrieval.bm25 import BM25Retriever | |
| from src.retrieval.dense import DenseRetriever | |
| from src.retrieval.fusion import reciprocal_rank_fusion | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", required=True) | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| seed = seed_everything(cfg["seed"]) | |
| rng = random.Random(seed) | |
| print(f"[seed] {seed}") | |
| corpus, queries, qrels = load_split( | |
| cfg["dataset"]["url"], cfg["dataset"]["split"], cfg["dataset"]["download_dir"] | |
| ) | |
| print(f"[data] train split: {len(queries)} queries | {len(corpus)} docs") | |
| doc_ids, doc_texts = flatten_corpus(corpus) | |
| top_k = cfg["retrieval"]["top_k"] | |
| cache_dir = cfg["paths"]["cache_dir"] | |
| run_path = Path(cfg["paths"]["train_run"]) | |
| if run_path.exists(): | |
| fused = json.loads(run_path.read_text(encoding="utf-8")) | |
| print(f"[cache] reusing {run_path}") | |
| else: | |
| bm25_cfg = cfg["retrieval"]["bm25"] | |
| bm25 = BM25Retriever(k1=bm25_cfg["k1"], b=bm25_cfg["b"], stemmer_lang=bm25_cfg["stemmer"]) | |
| bm25.index(doc_ids, doc_texts, cache_dir=cache_dir) | |
| bm25_run = bm25.search(queries, top_k=top_k) | |
| dense_cfg = cfg["retrieval"]["dense"] | |
| dense = DenseRetriever( | |
| model_name=dense_cfg["model"], | |
| batch_size=dense_cfg["batch_size"], | |
| max_length=dense_cfg["max_length"], | |
| query_prefix=dense_cfg["query_prefix"], | |
| device=device(), | |
| ) | |
| dense.index_corpus(doc_ids, doc_texts, cache_dir=cache_dir) | |
| dense_run = dense.search(queries, top_k=top_k) | |
| fused = reciprocal_rank_fusion( | |
| [bm25_run, dense_run], k=cfg["retrieval"]["fusion"]["rrf_k"], top_k=top_k | |
| ) | |
| run_path.parent.mkdir(parents=True, exist_ok=True) | |
| run_path.write_text(json.dumps(fused), encoding="utf-8") | |
| print(f"[cache] train run -> {run_path}") | |
| hard, rand = build_groups( | |
| corpus=corpus, | |
| queries=queries, | |
| qrels=qrels, | |
| run=fused, | |
| num_negatives=cfg["mining"]["num_negatives"], | |
| hard_min_rank=cfg["mining"]["hard_min_rank"], | |
| rng=rng, | |
| ) | |
| # The ablation is only meaningful if these two datasets are identical | |
| # everywhere except the negatives. | |
| assert len(hard) == len(rand), "hard and random datasets must be the same size" | |
| for h, r in zip(hard, rand): | |
| assert h["positive_id"] == r["positive_id"], "positives must match across datasets" | |
| assert len(h["negatives"]) == len(r["negatives"]), "negative counts must match" | |
| write_jsonl(hard, cfg["paths"]["hard_negatives"]) | |
| write_jsonl(rand, cfg["paths"]["random_negatives"]) | |
| report_vram("mine_negatives") | |
| if __name__ == "__main__": | |
| main() | |