File size: 3,228 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""ONNX Runtime reranker.

Interface-compatible with src.rerank.model.Reranker so the same evaluation and
benchmark scripts can drive either backend and the comparison stays honest.
"""

from __future__ import annotations

import time
from pathlib import Path

import numpy as np
import onnxruntime as ort
from transformers import AutoTokenizer

from src.data.loader import Corpus, passage_text

Run = dict[str, dict[str, float]]


class OnnxReranker:
    """Cross-encoder inference through ONNX Runtime on CPU."""

    def __init__(
        self,
        onnx_path: str,
        tokenizer_path: str,
        max_length: int = 384,
        batch_size: int = 32,
        intra_op_threads: int = 0,
    ) -> None:
        if not Path(onnx_path).exists():
            raise FileNotFoundError(f"ONNX model not found at {onnx_path}")
        options = ort.SessionOptions()
        options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
        if intra_op_threads:
            options.intra_op_num_threads = intra_op_threads

        self.session = ort.InferenceSession(
            onnx_path, sess_options=options, providers=["CPUExecutionProvider"]
        )
        self.input_names = [i.name for i in self.session.get_inputs()]
        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
        self.max_length = max_length
        self.batch_size = batch_size

    def score(self, pairs: list[tuple[str, str]]) -> np.ndarray:
        """Relevance logits for a batch of (query, passage) pairs."""
        scores: list[np.ndarray] = []
        for start in range(0, len(pairs), self.batch_size):
            chunk = pairs[start : start + self.batch_size]
            encoded = self.tokenizer(
                [q for q, _ in chunk],
                [p for _, p in chunk],
                padding=True,
                truncation=True,
                max_length=self.max_length,
                return_tensors="np",
            )
            feed = {name: encoded[name].astype(np.int64) for name in self.input_names}
            logits = self.session.run(["logits"], feed)[0]
            scores.append(np.asarray(logits, dtype=np.float32).reshape(-1))
        return np.concatenate(scores) if scores else np.zeros(0, dtype=np.float32)

    def rerank(
        self,
        queries: dict[str, str],
        run: Run,
        corpus: Corpus,
        top_k: int = 50,
    ) -> tuple[Run, list[float]]:
        """Rerank each query's candidates. Returns the new run and per-query ms."""
        reranked: Run = {}
        timings_ms: list[float] = []
        for qid, candidates in run.items():
            doc_ids = [d for d, _ in sorted(candidates.items(), key=lambda kv: kv[1], reverse=True)]
            doc_ids = doc_ids[:top_k]
            if not doc_ids:
                reranked[qid] = {}
                timings_ms.append(0.0)
                continue
            pairs = [(queries[qid], passage_text(corpus, d)) for d in doc_ids]
            start = time.perf_counter()
            scores = self.score(pairs)
            timings_ms.append((time.perf_counter() - start) * 1000.0)
            reranked[qid] = {d: float(s) for d, s in zip(doc_ids, scores)}
        return reranked, timings_ms