aikacl commited on
Commit
7d2b7f7
·
verified ·
1 Parent(s): cbcaa96

Create kalm_reranker

Browse files
Files changed (1) hide show
  1. kalm_reranker +349 -0
kalm_reranker ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
4
+
5
+ import numpy as np
6
+ import torch
7
+ import torch.nn.functional as F
8
+ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
9
+ from transformers.modeling_outputs import BaseModelOutput
10
+
11
+
12
+ DEFAULT_INSTRUCTION = "Given a query, retrieve documents that answer the query."
13
+ DEFAULT_SYSTEM_INSTRUCTION = (
14
+ "Judge whether the Document meets the requirements based on the Query and "
15
+ 'the Instruct provided. Note that the answer can only be "yes" or "no".'
16
+ )
17
+
18
+
19
+ class KaLMReranker:
20
+ """Score query-document relevance with a KaLM encoder-decoder reranker.
21
+
22
+ The returned score is ``P(yes)`` after applying a two-class softmax to the
23
+ model's ``yes`` and ``no`` logits.
24
+ """
25
+
26
+ def __init__(
27
+ self,
28
+ model_name_or_path: str,
29
+ *,
30
+ device: Optional[Union[str, torch.device]] = None,
31
+ dtype: Optional[Union[str, torch.dtype]] = None,
32
+ batch_size: int = 32,
33
+ query_max_length: int = 512,
34
+ max_length: int = 1024,
35
+ chunk_size: Optional[int] = 4,
36
+ instruction: str = DEFAULT_INSTRUCTION,
37
+ system_instruction: str = DEFAULT_SYSTEM_INSTRUCTION,
38
+ **model_kwargs: Any,
39
+ ) -> None:
40
+ if not isinstance(model_name_or_path, str) or not model_name_or_path:
41
+ raise ValueError("model_name_or_path must be a non-empty string.")
42
+ if batch_size <= 0:
43
+ raise ValueError("batch_size must be positive.")
44
+ if query_max_length <= 0 or max_length <= 0:
45
+ raise ValueError("query_max_length and max_length must be positive.")
46
+ if chunk_size is not None and chunk_size <= 0:
47
+ raise ValueError("chunk_size must be positive or None.")
48
+ if not isinstance(instruction, str) or not isinstance(system_instruction, str):
49
+ raise TypeError("instruction and system_instruction must be strings.")
50
+
51
+ self.device = self._resolve_device(device)
52
+ self.dtype = self._resolve_dtype(dtype, self.device)
53
+ self.batch_size = batch_size
54
+ self.query_max_length = query_max_length
55
+ self.max_length = max_length
56
+ self.chunk_size = chunk_size
57
+ self.instruction = instruction
58
+ self.system_instruction = system_instruction
59
+
60
+ self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
61
+ if self.tokenizer.pad_token_id is None:
62
+ if self.tokenizer.eos_token_id is None:
63
+ raise ValueError("The tokenizer must define a pad token or an EOS token.")
64
+ self.tokenizer.pad_token = self.tokenizer.eos_token
65
+
66
+ self.tokenizer.padding_side = "right"
67
+
68
+ self.model = AutoModelForSeq2SeqLM.from_pretrained(
69
+ model_name_or_path,
70
+ dtype=self.dtype,
71
+ **model_kwargs,
72
+ )
73
+
74
+ for parameter in self.model.parameters():
75
+ if parameter.is_floating_point() and parameter.dtype != self.dtype:
76
+ parameter.data = parameter.data.to(dtype=self.dtype)
77
+ self.model.to(device=self.device)
78
+ self.model.eval()
79
+
80
+ self.yes_token_id = self._answer_token_id("yes")
81
+ self.no_token_id = self._answer_token_id("no")
82
+
83
+ @staticmethod
84
+ def _resolve_device(device: Optional[Union[str, torch.device]]) -> torch.device:
85
+ if device is None:
86
+ device = "cuda" if torch.cuda.is_available() else "cpu"
87
+ resolved = torch.device(device)
88
+ if resolved.type == "cuda" and not torch.cuda.is_available():
89
+ raise RuntimeError("CUDA was requested, but no CUDA device is available.")
90
+ return resolved
91
+
92
+ @staticmethod
93
+ def _resolve_dtype(
94
+ dtype: Optional[Union[str, torch.dtype]], device: torch.device
95
+ ) -> torch.dtype:
96
+ if dtype is None:
97
+ return torch.bfloat16 if device.type == "cuda" else torch.float32
98
+ if isinstance(dtype, torch.dtype):
99
+ return dtype
100
+ if not isinstance(dtype, str):
101
+ raise TypeError("dtype must be a torch.dtype or a string such as 'bfloat16'.")
102
+ normalized = dtype.lower().removeprefix("torch.")
103
+ supported = {
104
+ "bfloat16": torch.bfloat16,
105
+ "bf16": torch.bfloat16,
106
+ "float16": torch.float16,
107
+ "fp16": torch.float16,
108
+ "float32": torch.float32,
109
+ "fp32": torch.float32,
110
+ }
111
+ if normalized not in supported:
112
+ raise ValueError(f"Unsupported dtype: {dtype!r}.")
113
+ return supported[normalized]
114
+
115
+ def _answer_token_id(self, answer: str) -> int:
116
+ token_ids = self.tokenizer(answer, add_special_tokens=False)["input_ids"]
117
+ if not token_ids:
118
+ raise ValueError(f"Failed to tokenize the answer {answer!r}.")
119
+ return token_ids[-1]
120
+
121
+ def _get_encoder(self):
122
+ if hasattr(self.model, "get_encoder"):
123
+ return self.model.get_encoder()
124
+ if hasattr(self.model, "encoder"):
125
+ return self.model.encoder
126
+ raise AttributeError(f"Cannot find the encoder on {type(self.model).__name__}.")
127
+
128
+ @staticmethod
129
+ def _pool_encoder_chunks(
130
+ hidden_states: torch.Tensor,
131
+ attention_mask: torch.Tensor,
132
+ chunk_size: int,
133
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
134
+ batch_size, sequence_length, hidden_size = hidden_states.shape
135
+ num_chunks = (sequence_length + chunk_size - 1) // chunk_size
136
+ padded_length = num_chunks * chunk_size
137
+ pad_length = padded_length - sequence_length
138
+
139
+ if pad_length:
140
+ hidden_states = F.pad(hidden_states, (0, 0, 0, pad_length))
141
+ attention_mask = F.pad(attention_mask, (0, pad_length))
142
+
143
+ hidden_states = hidden_states.view(
144
+ batch_size, num_chunks, chunk_size, hidden_size
145
+ )
146
+ chunk_mask = attention_mask.view(batch_size, num_chunks, chunk_size)
147
+ expanded_mask = chunk_mask.unsqueeze(-1).to(hidden_states.dtype)
148
+ pooled_hidden = (hidden_states * expanded_mask).sum(dim=2)
149
+ pooled_hidden = pooled_hidden / chunk_mask.sum(dim=2).clamp(min=1).unsqueeze(-1)
150
+ pooled_mask = (chunk_mask.sum(dim=2) > 0).to(attention_mask.dtype)
151
+ return pooled_hidden, pooled_mask
152
+
153
+ def _decoder_text(self, query: str, instruction: str) -> str:
154
+ query_ids = self.tokenizer(
155
+ query,
156
+ add_special_tokens=False,
157
+ truncation=True,
158
+ max_length=self.query_max_length,
159
+ )["input_ids"]
160
+ truncated_query = self.tokenizer.decode(
161
+ query_ids,
162
+ skip_special_tokens=False,
163
+ clean_up_tokenization_spaces=False,
164
+ )
165
+ return (
166
+ "<bos><start_of_turn>user\n"
167
+ f"{self.system_instruction}\n\n"
168
+ f"<Instruct>: {instruction}\n"
169
+ f"<Query>: {truncated_query}<end_of_turn>\n"
170
+ "<start_of_turn>model\n\n\n\n"
171
+ )
172
+
173
+ @staticmethod
174
+ def _validate_pairs(
175
+ pairs: Sequence[Tuple[str, str]],
176
+ ) -> List[Tuple[str, str]]:
177
+ if isinstance(pairs, (str, bytes)) or not isinstance(pairs, Sequence):
178
+ raise TypeError("pairs must be a sequence of (query, document) pairs.")
179
+ validated: List[Tuple[str, str]] = []
180
+ for index, pair in enumerate(pairs):
181
+ if (
182
+ isinstance(pair, (str, bytes))
183
+ or not isinstance(pair, Sequence)
184
+ or len(pair) != 2
185
+ ):
186
+ raise ValueError(f"pairs[{index}] must contain exactly two strings.")
187
+ query, document = pair
188
+ if not isinstance(query, str) or not isinstance(document, str):
189
+ raise TypeError(f"pairs[{index}] must contain exactly two strings.")
190
+ validated.append((query, document))
191
+ return validated
192
+
193
+ @torch.inference_mode()
194
+ def _predict_batch(
195
+ self, pairs: Sequence[Tuple[str, str]], instruction: str
196
+ ) -> List[float]:
197
+ encoder_texts = [f"<Document>: {document}" for _, document in pairs]
198
+ decoder_texts = [self._decoder_text(query, instruction) for query, _ in pairs]
199
+
200
+ encoder_batch = self.tokenizer(
201
+ encoder_texts,
202
+ padding=True,
203
+ truncation=True,
204
+ max_length=self.max_length,
205
+ add_special_tokens=False,
206
+ return_tensors="pt",
207
+ ).to(self.device)
208
+ decoder_batch = self.tokenizer(
209
+ decoder_texts,
210
+ padding=True,
211
+ pad_to_multiple_of=8,
212
+ add_special_tokens=False,
213
+ return_tensors="pt",
214
+ ).to(self.device)
215
+
216
+ if self.chunk_size is None:
217
+ outputs = self.model(
218
+ input_ids=encoder_batch["input_ids"],
219
+ attention_mask=encoder_batch["attention_mask"],
220
+ decoder_input_ids=decoder_batch["input_ids"],
221
+ decoder_attention_mask=decoder_batch["attention_mask"],
222
+ return_dict=True,
223
+ )
224
+ else:
225
+ encoder_outputs = self._get_encoder()(
226
+ input_ids=encoder_batch["input_ids"],
227
+ attention_mask=encoder_batch["attention_mask"],
228
+ return_dict=True,
229
+ )
230
+ pooled_hidden, pooled_mask = self._pool_encoder_chunks(
231
+ encoder_outputs.last_hidden_state,
232
+ encoder_batch["attention_mask"],
233
+ self.chunk_size,
234
+ )
235
+ outputs = self.model(
236
+ encoder_outputs=BaseModelOutput(last_hidden_state=pooled_hidden),
237
+ attention_mask=pooled_mask,
238
+ decoder_input_ids=decoder_batch["input_ids"],
239
+ decoder_attention_mask=decoder_batch["attention_mask"],
240
+ return_dict=True,
241
+ )
242
+
243
+ sequence_lengths = decoder_batch["attention_mask"].sum(dim=1) - 1
244
+ batch_indices = torch.arange(outputs.logits.shape[0], device=self.device)
245
+ last_logits = outputs.logits[batch_indices, sequence_lengths]
246
+ yes_no_logits = torch.stack(
247
+ (
248
+ last_logits[:, self.yes_token_id],
249
+ last_logits[:, self.no_token_id],
250
+ ),
251
+ dim=-1,
252
+ ).float()
253
+ if not torch.isfinite(yes_no_logits).all():
254
+ bad_count = (~torch.isfinite(yes_no_logits).all(dim=-1)).sum().item()
255
+ raise RuntimeError(
256
+ f"The model produced non-finite yes/no logits for {bad_count} input(s). "
257
+ "Use bfloat16 or float32 instead of float16."
258
+ )
259
+ return torch.softmax(yes_no_logits, dim=-1)[:, 0].cpu().tolist()
260
+
261
+ def predict(
262
+ self,
263
+ pairs: Sequence[Tuple[str, str]],
264
+ *,
265
+ instruction: Optional[str] = None,
266
+ batch_size: Optional[int] = None,
267
+ ) -> List[float]:
268
+ """Return ``P(yes)`` scores in the same order as ``pairs``."""
269
+ validated_pairs = self._validate_pairs(pairs)
270
+ if not validated_pairs:
271
+ return []
272
+ effective_instruction = self.instruction if instruction is None else instruction
273
+ if not isinstance(effective_instruction, str):
274
+ raise TypeError("instruction must be a string or None.")
275
+ effective_batch_size = self.batch_size if batch_size is None else batch_size
276
+ if not isinstance(effective_batch_size, int) or effective_batch_size <= 0:
277
+ raise ValueError("batch_size must be a positive integer.")
278
+
279
+
280
+ length_sorted_indices = np.argsort(
281
+ [-(len(query) + len(document)) for query, document in validated_pairs]
282
+ )
283
+ sorted_pairs = [validated_pairs[index] for index in length_sorted_indices]
284
+
285
+ tested_batch_size = effective_batch_size
286
+ while tested_batch_size > 1:
287
+ try:
288
+ self._predict_batch(
289
+ sorted_pairs[: min(len(sorted_pairs), tested_batch_size)],
290
+ effective_instruction,
291
+ )
292
+ break
293
+ except torch.cuda.OutOfMemoryError:
294
+ if torch.cuda.is_available():
295
+ torch.cuda.empty_cache()
296
+ tested_batch_size = max(1, tested_batch_size * 3 // 4)
297
+
298
+ sorted_scores: List[float] = []
299
+ try:
300
+ for start in range(0, len(sorted_pairs), tested_batch_size):
301
+ sorted_scores.extend(
302
+ self._predict_batch(
303
+ sorted_pairs[start : start + tested_batch_size],
304
+ effective_instruction,
305
+ )
306
+ )
307
+ except torch.cuda.OutOfMemoryError as error:
308
+ if torch.cuda.is_available():
309
+ torch.cuda.empty_cache()
310
+ raise RuntimeError(
311
+ "CUDA ran out of memory during reranking. Retry with a smaller batch_size "
312
+ "or shorter max_length."
313
+ ) from error
314
+ inverse_indices = np.argsort(length_sorted_indices)
315
+ return [sorted_scores[index] for index in inverse_indices]
316
+
317
+ def rank(
318
+ self,
319
+ query: str,
320
+ documents: Sequence[str],
321
+ *,
322
+ instruction: Optional[str] = None,
323
+ top_k: Optional[int] = None,
324
+ batch_size: Optional[int] = None,
325
+ ) -> List[Dict[str, Union[int, float]]]:
326
+ """Rank documents and return ``corpus_id``/``score`` dictionaries."""
327
+ if not isinstance(query, str):
328
+ raise TypeError("query must be a string.")
329
+ if isinstance(documents, (str, bytes)) or not isinstance(documents, Sequence):
330
+ raise TypeError("documents must be a sequence of strings.")
331
+ if any(not isinstance(document, str) for document in documents):
332
+ raise TypeError("every document must be a string.")
333
+ if top_k is not None and (not isinstance(top_k, int) or top_k < 0):
334
+ raise ValueError("top_k must be a non-negative integer or None.")
335
+
336
+ scores = self.predict(
337
+ [(query, document) for document in documents],
338
+ instruction=instruction,
339
+ batch_size=batch_size,
340
+ )
341
+ rankings: List[Dict[str, Union[int, float]]] = [
342
+ {"corpus_id": corpus_id, "score": score}
343
+ for corpus_id, score in enumerate(scores)
344
+ ]
345
+ rankings.sort(key=lambda item: item["score"], reverse=True)
346
+ return rankings if top_k is None else rankings[:top_k]
347
+
348
+
349
+ __all__ = ["KaLMReranker"]