Spaces:
Sleeping
Sleeping
| """BLEU / ROUGE / METEOR computation. | |
| All metrics compare ONE generated caption against the MULTIPLE (5) human | |
| reference captions for that image -- never against just one reference, | |
| since there's no single "correct" caption. | |
| """ | |
| from __future__ import annotations | |
| from nltk.translate.bleu_score import SmoothingFunction, corpus_bleu | |
| from nltk.translate.meteor_score import meteor_score | |
| from rouge_score import rouge_scorer | |
| def compute_bleu(hypotheses: list[list[str]], references: list[list[list[str]]]) -> dict[str, float]: | |
| """Corpus-level BLEU-1..4. | |
| hypotheses: list of tokenized generated captions, one per image | |
| references: list of lists of tokenized reference captions, one outer list per image | |
| """ | |
| smoothing = SmoothingFunction().method1 | |
| scores = {} | |
| for n in (1, 2, 3, 4): | |
| weights = tuple(1.0 / n for _ in range(n)) + tuple(0.0 for _ in range(4 - n)) | |
| scores[f"bleu_{n}"] = corpus_bleu(references, hypotheses, weights=weights, smoothing_function=smoothing) | |
| return scores | |
| def compute_rouge_l(hypotheses: list[str], references: list[list[str]]) -> float: | |
| """Average ROUGE-L (F-measure), taken against the best-matching reference per image. | |
| hypotheses: list of generated caption strings (not tokenized) | |
| references: list of lists of reference caption strings, one outer list per image | |
| """ | |
| scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=True) | |
| total = 0.0 | |
| for hyp, refs in zip(hypotheses, references): | |
| best = max(scorer.score(ref, hyp)["rougeL"].fmeasure for ref in refs) | |
| total += best | |
| return total / max(len(hypotheses), 1) | |
| def compute_meteor(hypotheses: list[list[str]], references: list[list[list[str]]]) -> float: | |
| """Average METEOR score across all images. | |
| hypotheses: list of tokenized generated captions | |
| references: list of lists of tokenized reference captions | |
| """ | |
| total = 0.0 | |
| for hyp, refs in zip(hypotheses, references): | |
| total += meteor_score(refs, hyp) | |
| return total / max(len(hypotheses), 1) | |
| def compute_all_metrics( | |
| hypotheses_str: list[str], | |
| references_str: list[list[str]], | |
| ) -> dict[str, float]: | |
| """Compute BLEU-1..4, ROUGE-L, METEOR from plain (untokenized) strings.""" | |
| hyp_tokens = [h.split() for h in hypotheses_str] | |
| ref_tokens = [[r.split() for r in refs] for refs in references_str] | |
| results = compute_bleu(hyp_tokens, ref_tokens) | |
| results["rouge_l"] = compute_rouge_l(hypotheses_str, references_str) | |
| results["meteor"] = compute_meteor(hyp_tokens, ref_tokens) | |
| return results |