"""Token-weighted language-model loss and held-out continuation BLEU.""" import json import math from collections import defaultdict from contextlib import nullcontext import torch from sacrebleu.metrics import BLEU from torch.nn import functional as F from .data import make_batch def precision_context(device, mixed_precision): if device.type == "cuda" and mixed_precision: return torch.autocast(device_type="cuda", dtype=torch.float16) return nullcontext() @torch.inference_mode() def score_loss(model, tokens, tokenizer, device, args): model.eval() loss_sum = 0.0 position = 0 total = len(tokens) - 1 while position < total: inputs, labels, padding_mask, count = make_batch( tokens, position, total - position, args.eval_batch_size, args.context_size, tokenizer.pad_token_id, ) with precision_context(device, args.mixed_precision): logits = model(inputs.to(device), padding_mask.to(device)) loss = F.cross_entropy(logits.float().transpose(1, 2), labels.to(device), ignore_index=-100, reduction="sum") loss_sum += loss.item() position += count mean_loss = loss_sum / total return {"loss": mean_loss, "perplexity": math.exp(min(mean_loss, 700)), "target_tokens": total} @torch.inference_mode() def score_bleu(model, documents, tokenizer, device, args, output_file): model.eval() buckets = defaultdict(list) for document in documents: ids = document["ids"] # Keep at least one text token (not only EOS) for the continuation. prompt_length = min(args.prompt_tokens, len(ids) - 2) prompt = ids[:prompt_length] reference = ids[prompt_length:prompt_length + args.max_new_tokens] buckets[len(prompt)].append((document["doc_id"], prompt, reference)) predictions = [] references = [] generated_tokens = 0 with output_file.open("w", encoding="utf-8") as stream: for group in buckets.values(): for start in range(0, len(group), args.eval_batch_size): batch = group[start:start + args.eval_batch_size] inputs = torch.tensor([row[1] for row in batch], device=device) with precision_context(device, args.mixed_precision): generated = model.generate(inputs, max_new_tokens=args.max_new_tokens, eos_token_id=tokenizer.eos_token_id).cpu() for output, (document_id, prompt, reference) in zip(generated, batch): continuation = output[len(prompt):].tolist() if tokenizer.eos_token_id in continuation: continuation = continuation[:continuation.index(tokenizer.eos_token_id)] prediction_text = tokenizer.decode(continuation, skip_special_tokens=True) reference_text = tokenizer.decode(reference, skip_special_tokens=True) predictions.append(prediction_text) references.append(reference_text) generated_tokens += len(continuation) stream.write(json.dumps({ "doc_id": document_id, "prompt": tokenizer.decode(prompt, skip_special_tokens=True), "reference": reference_text, "prediction": prediction_text, }, ensure_ascii=False) + "\n") bleu = BLEU(tokenize="13a") score = bleu.corpus_score(predictions, [references]) return {"bleu": score.score, "bleu_signature": str(bleu.get_signature()), "samples": len(predictions), "generated_tokens": generated_tokens}