walke007's picture
Upload Part 2 checkpoints and recorded results
a3baf17 verified
Raw History Blame Contribute Delete
3.78 kB
"""Token-weighted language-model loss and held-out continuation BLEU."""
import json
import math
from collections import defaultdict
from contextlib import nullcontext
import torch
from sacrebleu.metrics import BLEU
from torch.nn import functional as F
from .data import make_batch
def precision_context(device, mixed_precision):
if device.type == "cuda" and mixed_precision:
return torch.autocast(device_type="cuda", dtype=torch.float16)
return nullcontext()
@torch.inference_mode()
def score_loss(model, tokens, tokenizer, device, args):
model.eval()
loss_sum = 0.0
position = 0
total = len(tokens) - 1
while position < total:
inputs, labels, padding_mask, count = make_batch(
tokens, position, total - position, args.eval_batch_size,
args.context_size, tokenizer.pad_token_id,
)
with precision_context(device, args.mixed_precision):
logits = model(inputs.to(device), padding_mask.to(device))
loss = F.cross_entropy(logits.float().transpose(1, 2), labels.to(device),
ignore_index=-100, reduction="sum")
loss_sum += loss.item()
position += count
mean_loss = loss_sum / total
return {"loss": mean_loss, "perplexity": math.exp(min(mean_loss, 700)),
"target_tokens": total}
@torch.inference_mode()
def score_bleu(model, documents, tokenizer, device, args, output_file):
model.eval()
buckets = defaultdict(list)
for document in documents:
ids = document["ids"]
# Keep at least one text token (not only EOS) for the continuation.
prompt_length = min(args.prompt_tokens, len(ids) - 2)
prompt = ids[:prompt_length]
reference = ids[prompt_length:prompt_length + args.max_new_tokens]
buckets[len(prompt)].append((document["doc_id"], prompt, reference))
predictions = []
references = []
generated_tokens = 0
with output_file.open("w", encoding="utf-8") as stream:
for group in buckets.values():
for start in range(0, len(group), args.eval_batch_size):
batch = group[start:start + args.eval_batch_size]
inputs = torch.tensor([row[1] for row in batch], device=device)
with precision_context(device, args.mixed_precision):
generated = model.generate(inputs, max_new_tokens=args.max_new_tokens,
eos_token_id=tokenizer.eos_token_id).cpu()
for output, (document_id, prompt, reference) in zip(generated, batch):
continuation = output[len(prompt):].tolist()
if tokenizer.eos_token_id in continuation:
continuation = continuation[:continuation.index(tokenizer.eos_token_id)]
prediction_text = tokenizer.decode(continuation, skip_special_tokens=True)
reference_text = tokenizer.decode(reference, skip_special_tokens=True)
predictions.append(prediction_text)
references.append(reference_text)
generated_tokens += len(continuation)
stream.write(json.dumps({
"doc_id": document_id,
"prompt": tokenizer.decode(prompt, skip_special_tokens=True),
"reference": reference_text,
"prediction": prediction_text,
}, ensure_ascii=False) + "\n")
bleu = BLEU(tokenize="13a")
score = bleu.corpus_score(predictions, [references])
return {"bleu": score.score, "bleu_signature": str(bleu.get_signature()),
"samples": len(predictions), "generated_tokens": generated_tokens}