Download source/part2/evaluate.py from walke007/anlp-assignment2-part2: direct link, hf CLI and curl.
- Browser
- Download file 3.78 kB
-
https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/evaluate.py
- Command line
-
hf download hf://walke007/anlp-assignment2-part2/source/part2/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/evaluate.py
3.78 kB
| """Token-weighted language-model loss and held-out continuation BLEU.""" | |
| import json | |
| import math | |
| from collections import defaultdict | |
| from contextlib import nullcontext | |
| import torch | |
| from sacrebleu.metrics import BLEU | |
| from torch.nn import functional as F | |
| from .data import make_batch | |
| def precision_context(device, mixed_precision): | |
| if device.type == "cuda" and mixed_precision: | |
| return torch.autocast(device_type="cuda", dtype=torch.float16) | |
| return nullcontext() | |
| def score_loss(model, tokens, tokenizer, device, args): | |
| model.eval() | |
| loss_sum = 0.0 | |
| position = 0 | |
| total = len(tokens) - 1 | |
| while position < total: | |
| inputs, labels, padding_mask, count = make_batch( | |
| tokens, position, total - position, args.eval_batch_size, | |
| args.context_size, tokenizer.pad_token_id, | |
| ) | |
| with precision_context(device, args.mixed_precision): | |
| logits = model(inputs.to(device), padding_mask.to(device)) | |
| loss = F.cross_entropy(logits.float().transpose(1, 2), labels.to(device), | |
| ignore_index=-100, reduction="sum") | |
| loss_sum += loss.item() | |
| position += count | |
| mean_loss = loss_sum / total | |
| return {"loss": mean_loss, "perplexity": math.exp(min(mean_loss, 700)), | |
| "target_tokens": total} | |
| def score_bleu(model, documents, tokenizer, device, args, output_file): | |
| model.eval() | |
| buckets = defaultdict(list) | |
| for document in documents: | |
| ids = document["ids"] | |
| # Keep at least one text token (not only EOS) for the continuation. | |
| prompt_length = min(args.prompt_tokens, len(ids) - 2) | |
| prompt = ids[:prompt_length] | |
| reference = ids[prompt_length:prompt_length + args.max_new_tokens] | |
| buckets[len(prompt)].append((document["doc_id"], prompt, reference)) | |
| predictions = [] | |
| references = [] | |
| generated_tokens = 0 | |
| with output_file.open("w", encoding="utf-8") as stream: | |
| for group in buckets.values(): | |
| for start in range(0, len(group), args.eval_batch_size): | |
| batch = group[start:start + args.eval_batch_size] | |
| inputs = torch.tensor([row[1] for row in batch], device=device) | |
| with precision_context(device, args.mixed_precision): | |
| generated = model.generate(inputs, max_new_tokens=args.max_new_tokens, | |
| eos_token_id=tokenizer.eos_token_id).cpu() | |
| for output, (document_id, prompt, reference) in zip(generated, batch): | |
| continuation = output[len(prompt):].tolist() | |
| if tokenizer.eos_token_id in continuation: | |
| continuation = continuation[:continuation.index(tokenizer.eos_token_id)] | |
| prediction_text = tokenizer.decode(continuation, skip_special_tokens=True) | |
| reference_text = tokenizer.decode(reference, skip_special_tokens=True) | |
| predictions.append(prediction_text) | |
| references.append(reference_text) | |
| generated_tokens += len(continuation) | |
| stream.write(json.dumps({ | |
| "doc_id": document_id, | |
| "prompt": tokenizer.decode(prompt, skip_special_tokens=True), | |
| "reference": reference_text, | |
| "prediction": prediction_text, | |
| }, ensure_ascii=False) + "\n") | |
| bleu = BLEU(tokenize="13a") | |
| score = bleu.corpus_score(predictions, [references]) | |
| return {"bleu": score.score, "bleu_signature": str(bleu.get_signature()), | |
| "samples": len(predictions), "generated_tokens": generated_tokens} | |