File size: 2,310 Bytes
e5c059f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""Build the BM25 and dense index caches without starting the whole pipeline.

    python scripts/build_index.py --config configs/ml.yaml

MLPipeline builds these on first start, but it also loads the reranker and
every paper record, which at 270k papers costs several GB of RAM that
indexing never uses. This builds exactly the same caches (same ids, same
text, same fingerprints) in a lean process - the one to run on a laptop -
and the app then finds them ready and starts in seconds.
"""

from __future__ import annotations

import argparse
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

from src.config import device, load_config  # noqa: E402
from src.data.chunking import chunks_to_corpus, read_chunks  # noqa: E402
from src.retrieval.bm25 import BM25Retriever  # noqa: E402
from src.retrieval.dense import DenseRetriever  # noqa: E402


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--config", default="configs/ml.yaml")
    args = ap.parse_args()

    cfg = load_config(args.config)
    paths, rc = cfg["paths"], cfg["retrieval"]
    corpus = chunks_to_corpus(read_chunks(paths["chunks"]))
    doc_ids = list(corpus.keys())
    # Must match MLPipeline.__init__ exactly, or the caches would never be hit.
    doc_texts = [f"{corpus[d]['title']} {corpus[d]['text']}".strip() for d in doc_ids]
    del corpus
    print(f"[index] {len(doc_ids):,} passages -> {paths['index_cache']}", flush=True)

    t = time.time()
    BM25Retriever(k1=rc["bm25"]["k1"], b=rc["bm25"]["b"],
                  stemmer_lang=rc["bm25"]["stemmer"]).index(
        doc_ids, doc_texts, cache_dir=paths["index_cache"])
    print(f"[index] bm25 ready in {time.time() - t:.0f}s", flush=True)

    t = time.time()
    DenseRetriever(model_name=rc["dense"]["model"], batch_size=rc["dense"]["batch_size"],
                   max_length=rc["dense"]["max_length"], query_prefix=rc["dense"]["query_prefix"],
                   device=device()).index_corpus(doc_ids, doc_texts, cache_dir=paths["index_cache"])
    print(f"[index] dense ready in {time.time() - t:.0f}s", flush=True)


if __name__ == "__main__":
    main()