Jonnester's picture
Purrence-3M
921bf58
Raw History Blame Contribute Delete
5.79 kB
from contextlib import contextmanager
import torch
import torch.nn.functional as F
from .tokenizer import Sub3MTokenizer
from pathlib import Path
TOKENIZER = Path(__file__).resolve().parents[1] / "tokenizer"
def cross_entropy_sum(logits, targets):
return F.cross_entropy(logits.float(), targets, reduction="sum")
@contextmanager
def fp32_math(device):
"""Full FP32 inference, including attention, without leaking backend flags."""
from torch.nn.attention import SDPBackend, sdpa_kernel
previous_precision = torch.get_float32_matmul_precision()
previous_tf32 = torch.backends.cuda.matmul.allow_tf32
previous_cudnn_tf32 = torch.backends.cudnn.allow_tf32
try:
torch.set_float32_matmul_precision("highest")
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
with torch.autocast(device.type, enabled=False), sdpa_kernel(SDPBackend.MATH):
yield
finally:
torch.set_float32_matmul_precision(previous_precision)
torch.backends.cuda.matmul.allow_tf32 = previous_tf32
torch.backends.cudnn.allow_tf32 = previous_cudnn_tf32
class Scorer:
def __init__(self, model, tokenizer=None, batch_size=16):
if batch_size < 1:
raise ValueError("Batch size must be positive")
self.model = model.eval()
self.tokenizer = tokenizer or Sub3MTokenizer(TOKENIZER)
self.batch_size = batch_size
self.device = next(model.parameters()).device
if any(p.is_floating_point() and p.dtype != torch.float32
for p in model.parameters()):
raise ValueError("This scorer requires FP32 inference weights")
def inference_context(self):
return fp32_math(self.device)
@torch.no_grad()
def score(self, requests, max_length=None):
limit = max_length or self.model.config.max_seq_len
if not 1 <= limit <= self.model.config.max_seq_len:
raise ValueError("Scoring context is outside the model's context limit")
results = []
for begin in range(0, len(requests), self.batch_size):
inputs, targets, lengths, counts = [], [], [], []
for context, continuation in requests[begin:begin + self.batch_size]:
if not continuation or len(continuation) > limit:
raise ValueError("Continuation must contain between one and max_length tokens")
context = context or [self.tokenizer.boundary_id]
sequence = (list(context) + list(continuation))[-(limit + 1):]
inputs.extend(sequence[:-1])
targets.append(continuation)
lengths.append(len(sequence) - 1)
counts.append(len(continuation))
ids = torch.tensor(inputs, dtype=torch.long, device=self.device)
cu = F.pad(torch.tensor(lengths, device=self.device).cumsum(0), (1, 0)).to(torch.int32)
positions = torch.cat([torch.arange(length, device=self.device) for length in lengths])
with self.inference_context():
logits = self.model(ids, cu, positions)
offset = 0
for length, count, target in zip(lengths, counts, targets):
selected = logits[offset + length - count:offset + length].float()
target = torch.tensor(target, device=self.device)
value = -cross_entropy_sum(selected, target)
greedy = selected.argmax(-1).eq(target).all()
results.append((float(value), bool(greedy)))
offset += length
return results
def rolling(self, text):
tokens = self.tokenizer.encode(text)
limit = self.model.config.max_seq_len
requests = []
for start in range(0, len(tokens), limit):
continuation = tokens[start:start + limit]
if not start:
context = [self.tokenizer.boundary_id]
elif len(continuation) < limit:
# LM-Eval's final partial window keeps enough preceding tokens
# to use the full model context; full-sized windows keep one.
context_size = limit - len(continuation) + 1
context = tokens[start - context_size:start]
else:
context = [tokens[start - 1]]
requests.append((context, continuation))
return sum(score for score, _ in self.score(requests))
@torch.no_grad()
def generate(self, prompt, max_new_tokens=128, temperature=0.0, until=(), seed=17):
if max_new_tokens < 0 or temperature < 0:
raise ValueError("Generation length and temperature must be nonnegative")
if isinstance(until, str):
until = [until]
if any(not stop for stop in until):
raise ValueError("Stop strings must be nonempty")
prefix = self.tokenizer.encode(prompt) or [self.tokenizer.boundary_id]
generated = []
rng = torch.Generator(device=self.device).manual_seed(seed)
for _ in range(max_new_tokens):
ids = torch.tensor((prefix + generated)[-self.model.config.max_seq_len:], device=self.device)
with self.inference_context():
logits = self.model(ids)[-1].float()
token = int(logits.argmax()) if temperature == 0 else int(torch.multinomial(
(logits / temperature).softmax(-1), 1, generator=rng))
if token == self.tokenizer.boundary_id:
break
generated.append(token)
text = self.tokenizer.decode(generated)
endings = [text.index(stop) for stop in until if stop in text]
if endings:
return text[:min(endings)]
return self.tokenizer.decode(generated)