from contextlib import contextmanager import torch import torch.nn.functional as F from .tokenizer import Sub3MTokenizer from pathlib import Path TOKENIZER = Path(__file__).resolve().parents[1] / "tokenizer" def cross_entropy_sum(logits, targets): return F.cross_entropy(logits.float(), targets, reduction="sum") @contextmanager def fp32_math(device): """Full FP32 inference, including attention, without leaking backend flags.""" from torch.nn.attention import SDPBackend, sdpa_kernel previous_precision = torch.get_float32_matmul_precision() previous_tf32 = torch.backends.cuda.matmul.allow_tf32 previous_cudnn_tf32 = torch.backends.cudnn.allow_tf32 try: torch.set_float32_matmul_precision("highest") torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False with torch.autocast(device.type, enabled=False), sdpa_kernel(SDPBackend.MATH): yield finally: torch.set_float32_matmul_precision(previous_precision) torch.backends.cuda.matmul.allow_tf32 = previous_tf32 torch.backends.cudnn.allow_tf32 = previous_cudnn_tf32 class Scorer: def __init__(self, model, tokenizer=None, batch_size=16): if batch_size < 1: raise ValueError("Batch size must be positive") self.model = model.eval() self.tokenizer = tokenizer or Sub3MTokenizer(TOKENIZER) self.batch_size = batch_size self.device = next(model.parameters()).device if any(p.is_floating_point() and p.dtype != torch.float32 for p in model.parameters()): raise ValueError("This scorer requires FP32 inference weights") def inference_context(self): return fp32_math(self.device) @torch.no_grad() def score(self, requests, max_length=None): limit = max_length or self.model.config.max_seq_len if not 1 <= limit <= self.model.config.max_seq_len: raise ValueError("Scoring context is outside the model's context limit") results = [] for begin in range(0, len(requests), self.batch_size): inputs, targets, lengths, counts = [], [], [], [] for context, continuation in requests[begin:begin + self.batch_size]: if not continuation or len(continuation) > limit: raise ValueError("Continuation must contain between one and max_length tokens") context = context or [self.tokenizer.boundary_id] sequence = (list(context) + list(continuation))[-(limit + 1):] inputs.extend(sequence[:-1]) targets.append(continuation) lengths.append(len(sequence) - 1) counts.append(len(continuation)) ids = torch.tensor(inputs, dtype=torch.long, device=self.device) cu = F.pad(torch.tensor(lengths, device=self.device).cumsum(0), (1, 0)).to(torch.int32) positions = torch.cat([torch.arange(length, device=self.device) for length in lengths]) with self.inference_context(): logits = self.model(ids, cu, positions) offset = 0 for length, count, target in zip(lengths, counts, targets): selected = logits[offset + length - count:offset + length].float() target = torch.tensor(target, device=self.device) value = -cross_entropy_sum(selected, target) greedy = selected.argmax(-1).eq(target).all() results.append((float(value), bool(greedy))) offset += length return results def rolling(self, text): tokens = self.tokenizer.encode(text) limit = self.model.config.max_seq_len requests = [] for start in range(0, len(tokens), limit): continuation = tokens[start:start + limit] if not start: context = [self.tokenizer.boundary_id] elif len(continuation) < limit: # LM-Eval's final partial window keeps enough preceding tokens # to use the full model context; full-sized windows keep one. context_size = limit - len(continuation) + 1 context = tokens[start - context_size:start] else: context = [tokens[start - 1]] requests.append((context, continuation)) return sum(score for score, _ in self.score(requests)) @torch.no_grad() def generate(self, prompt, max_new_tokens=128, temperature=0.0, until=(), seed=17): if max_new_tokens < 0 or temperature < 0: raise ValueError("Generation length and temperature must be nonnegative") if isinstance(until, str): until = [until] if any(not stop for stop in until): raise ValueError("Stop strings must be nonempty") prefix = self.tokenizer.encode(prompt) or [self.tokenizer.boundary_id] generated = [] rng = torch.Generator(device=self.device).manual_seed(seed) for _ in range(max_new_tokens): ids = torch.tensor((prefix + generated)[-self.model.config.max_seq_len:], device=self.device) with self.inference_context(): logits = self.model(ids)[-1].float() token = int(logits.argmax()) if temperature == 0 else int(torch.multinomial( (logits / temperature).softmax(-1), 1, generator=rng)) if token == self.tokenizer.boundary_id: break generated.append(token) text = self.tokenizer.decode(generated) endings = [text.index(stop) for stop in until if stop in text] if endings: return text[:min(endings)] return self.tokenizer.decode(generated)