File size: 3,776 Bytes
a3baf17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
"""Token-weighted language-model loss and held-out continuation BLEU."""

import json
import math
from collections import defaultdict
from contextlib import nullcontext

import torch
from sacrebleu.metrics import BLEU
from torch.nn import functional as F

from .data import make_batch


def precision_context(device, mixed_precision):
    if device.type == "cuda" and mixed_precision:
        return torch.autocast(device_type="cuda", dtype=torch.float16)
    return nullcontext()


@torch.inference_mode()
def score_loss(model, tokens, tokenizer, device, args):
    model.eval()
    loss_sum = 0.0
    position = 0
    total = len(tokens) - 1
    while position < total:
        inputs, labels, padding_mask, count = make_batch(
            tokens, position, total - position, args.eval_batch_size,
            args.context_size, tokenizer.pad_token_id,
        )
        with precision_context(device, args.mixed_precision):
            logits = model(inputs.to(device), padding_mask.to(device))
            loss = F.cross_entropy(logits.float().transpose(1, 2), labels.to(device),
                                   ignore_index=-100, reduction="sum")
        loss_sum += loss.item()
        position += count
    mean_loss = loss_sum / total
    return {"loss": mean_loss, "perplexity": math.exp(min(mean_loss, 700)),
            "target_tokens": total}


@torch.inference_mode()
def score_bleu(model, documents, tokenizer, device, args, output_file):
    model.eval()
    buckets = defaultdict(list)
    for document in documents:
        ids = document["ids"]
        # Keep at least one text token (not only EOS) for the continuation.
        prompt_length = min(args.prompt_tokens, len(ids) - 2)
        prompt = ids[:prompt_length]
        reference = ids[prompt_length:prompt_length + args.max_new_tokens]
        buckets[len(prompt)].append((document["doc_id"], prompt, reference))

    predictions = []
    references = []
    generated_tokens = 0
    with output_file.open("w", encoding="utf-8") as stream:
        for group in buckets.values():
            for start in range(0, len(group), args.eval_batch_size):
                batch = group[start:start + args.eval_batch_size]
                inputs = torch.tensor([row[1] for row in batch], device=device)
                with precision_context(device, args.mixed_precision):
                    generated = model.generate(inputs, max_new_tokens=args.max_new_tokens,
                                               eos_token_id=tokenizer.eos_token_id).cpu()
                for output, (document_id, prompt, reference) in zip(generated, batch):
                    continuation = output[len(prompt):].tolist()
                    if tokenizer.eos_token_id in continuation:
                        continuation = continuation[:continuation.index(tokenizer.eos_token_id)]
                    prediction_text = tokenizer.decode(continuation, skip_special_tokens=True)
                    reference_text = tokenizer.decode(reference, skip_special_tokens=True)
                    predictions.append(prediction_text)
                    references.append(reference_text)
                    generated_tokens += len(continuation)
                    stream.write(json.dumps({
                        "doc_id": document_id,
                        "prompt": tokenizer.decode(prompt, skip_special_tokens=True),
                        "reference": reference_text,
                        "prediction": prediction_text,
                    }, ensure_ascii=False) + "\n")

    bleu = BLEU(tokenize="13a")
    score = bleu.corpus_score(predictions, [references])
    return {"bleu": score.score, "bleu_signature": str(bleu.get_signature()),
            "samples": len(predictions), "generated_tokens": generated_tokens}