Spaces:
Sleeping
Sleeping
File size: 2,583 Bytes
143710c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """BLEU / ROUGE / METEOR computation.
All metrics compare ONE generated caption against the MULTIPLE (5) human
reference captions for that image -- never against just one reference,
since there's no single "correct" caption.
"""
from __future__ import annotations
from nltk.translate.bleu_score import SmoothingFunction, corpus_bleu
from nltk.translate.meteor_score import meteor_score
from rouge_score import rouge_scorer
def compute_bleu(hypotheses: list[list[str]], references: list[list[list[str]]]) -> dict[str, float]:
"""Corpus-level BLEU-1..4.
hypotheses: list of tokenized generated captions, one per image
references: list of lists of tokenized reference captions, one outer list per image
"""
smoothing = SmoothingFunction().method1
scores = {}
for n in (1, 2, 3, 4):
weights = tuple(1.0 / n for _ in range(n)) + tuple(0.0 for _ in range(4 - n))
scores[f"bleu_{n}"] = corpus_bleu(references, hypotheses, weights=weights, smoothing_function=smoothing)
return scores
def compute_rouge_l(hypotheses: list[str], references: list[list[str]]) -> float:
"""Average ROUGE-L (F-measure), taken against the best-matching reference per image.
hypotheses: list of generated caption strings (not tokenized)
references: list of lists of reference caption strings, one outer list per image
"""
scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=True)
total = 0.0
for hyp, refs in zip(hypotheses, references):
best = max(scorer.score(ref, hyp)["rougeL"].fmeasure for ref in refs)
total += best
return total / max(len(hypotheses), 1)
def compute_meteor(hypotheses: list[list[str]], references: list[list[list[str]]]) -> float:
"""Average METEOR score across all images.
hypotheses: list of tokenized generated captions
references: list of lists of tokenized reference captions
"""
total = 0.0
for hyp, refs in zip(hypotheses, references):
total += meteor_score(refs, hyp)
return total / max(len(hypotheses), 1)
def compute_all_metrics(
hypotheses_str: list[str],
references_str: list[list[str]],
) -> dict[str, float]:
"""Compute BLEU-1..4, ROUGE-L, METEOR from plain (untokenized) strings."""
hyp_tokens = [h.split() for h in hypotheses_str]
ref_tokens = [[r.split() for r in refs] for refs in references_str]
results = compute_bleu(hyp_tokens, ref_tokens)
results["rouge_l"] = compute_rouge_l(hypotheses_str, references_str)
results["meteor"] = compute_meteor(hyp_tokens, ref_tokens)
return results |