Spaces:
Running on Zero
Running on Zero
File size: 4,744 Bytes
e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | """Stage 1 - baseline retrieval.
BM25, dense, and their RRF fusion, evaluated on the SciFact test split.
Writes rows 1 and 2 of the results table and caches the fused run for later
stages.
python scripts/run_baseline.py --config configs/baseline.yaml
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.config import device, load_config, report_vram, seed_everything
from src.data.loader import flatten_corpus, load_split
from src.rerank.evaluate import evaluate_run, latency_stats, summarise, write_row
from src.retrieval.bm25 import BM25Retriever
from src.retrieval.dense import DenseRetriever
from src.protocols import SearchRetriever
from src.retrieval.fusion import Run, reciprocal_rank_fusion
def timed_search(
retriever: SearchRetriever, queries: dict[str, str], top_k: int
) -> tuple[Run, list[float]]:
"""Search one query at a time so the timings reflect the serving path."""
run: Run = {}
timings_ms: list[float] = []
for qid, text in queries.items():
start = time.perf_counter()
single = retriever.search({qid: text}, top_k=top_k)
timings_ms.append((time.perf_counter() - start) * 1000.0)
run[qid] = single[qid]
return run, timings_ms
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", required=True)
args = parser.parse_args()
cfg = load_config(args.config)
seed = seed_everything(cfg["seed"])
print(f"[seed] {seed}")
corpus, queries, qrels = load_split(
cfg["dataset"]["url"], cfg["dataset"]["split"], cfg["dataset"]["download_dir"]
)
print(f"[data] {len(corpus)} docs | {len(queries)} queries | {len(qrels)} qrels")
doc_ids, doc_texts = flatten_corpus(corpus)
top_k = cfg["retrieval"]["top_k"]
k_values = cfg["eval"]["k_values"]
cache_dir = cfg["paths"]["cache_dir"]
results_path = cfg["paths"]["results"]
# ---- Row 1: BM25 only -------------------------------------------------
bm25_cfg = cfg["retrieval"]["bm25"]
bm25 = BM25Retriever(k1=bm25_cfg["k1"], b=bm25_cfg["b"], stemmer_lang=bm25_cfg["stemmer"])
bm25.index(doc_ids, doc_texts, cache_dir=cache_dir)
bm25_run, bm25_ms = timed_search(bm25, queries, top_k)
bm25_metrics = evaluate_run(qrels, bm25_run, k_values)
print(f"[bm25] {summarise(bm25_metrics)}")
write_row(
"bm25",
{"metrics": bm25_metrics, "latency": latency_stats(bm25_ms), "params": None, "seed": seed},
results_path,
)
# ---- Row 2: BM25 + dense, fused with RRF ------------------------------
dense_cfg = cfg["retrieval"]["dense"]
dense = DenseRetriever(
model_name=dense_cfg["model"],
batch_size=dense_cfg["batch_size"],
max_length=dense_cfg["max_length"],
query_prefix=dense_cfg["query_prefix"],
device=device(),
)
dense.index_corpus(doc_ids, doc_texts, cache_dir=cache_dir)
dense_run, dense_ms = timed_search(dense, queries, top_k)
dense_metrics = evaluate_run(qrels, dense_run, k_values)
print(f"[dense] {summarise(dense_metrics)}")
rrf_k = cfg["retrieval"]["fusion"]["rrf_k"]
fuse_start = time.perf_counter()
fused = reciprocal_rank_fusion([bm25_run, dense_run], k=rrf_k, top_k=top_k)
fuse_ms_per_query = (time.perf_counter() - fuse_start) * 1000.0 / max(1, len(queries))
fused_metrics = evaluate_run(qrels, fused, k_values)
fused_ms = [b + d + fuse_ms_per_query for b, d in zip(bm25_ms, dense_ms)]
print(f"[rrf ] {summarise(fused_metrics)}")
write_row(
"bm25_dense_rrf",
{
"metrics": fused_metrics,
"latency": latency_stats(fused_ms),
"params": dense_cfg["params"],
"seed": seed,
"dense_only_metrics": dense_metrics,
},
results_path,
)
fused_path = Path(cfg["paths"]["fused_run"])
fused_path.parent.mkdir(parents=True, exist_ok=True)
fused_path.write_text(json.dumps(fused), encoding="utf-8")
print(f"[cache] fused top-{top_k} run -> {fused_path}")
# Per-query retrieval timings, so later stages can report end-to-end latency.
latency_path = fused_path.with_name(fused_path.stem + "_latency.json")
latency_path.write_text(
json.dumps({qid: ms for qid, ms in zip(queries.keys(), fused_ms)}), encoding="utf-8"
)
print(f"[cache] per-query retrieval latency -> {latency_path}")
report_vram("baseline")
if __name__ == "__main__":
main()
|