| ''' |
| Automatic generation evaluation metrics wrapper |
| |
| The most useful function here is |
| |
| get_all_metrics(refs, cands) |
| ''' |
| from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer |
| from pycocoevalcap.spice.spice import Spice |
| from pycocoevalcap.meteor.meteor import Meteor |
| from pycocoevalcap.bleu.bleu import Bleu |
| from pycocoevalcap.cider.cider import Cider |
| from pycocoevalcap.rouge.rouge import Rouge |
| from pycocoevalcap.spice.spice import Spice |
|
|
|
|
| def get_all_metrics(refs, cands, return_per_cap=False): |
| metrics = [] |
| names = [] |
|
|
| pycoco_eval_cap_scorers = [(Bleu(4), 'bleu'), |
| (Meteor(), 'meteor'), |
| (Rouge(), 'rouge'), |
| (Cider(), 'cider'), |
| (Spice(), 'spice')] |
|
|
| for scorer, name in pycoco_eval_cap_scorers: |
| overall, per_cap = pycoco_eval(scorer, refs, cands) |
| if return_per_cap: |
| metrics.append(per_cap) |
| else: |
| metrics.append(overall) |
| names.append(name) |
|
|
| metrics = dict(zip(names, metrics)) |
| return metrics |
|
|
|
|
| def tokenize(refs, cands, no_op=False): |
| |
| |
| tokenizer = PTBTokenizer() |
|
|
| if no_op: |
| refs = {idx: [r for r in c_refs] for idx, c_refs in enumerate(refs)} |
| cands = {idx: [c] for idx, c in enumerate(cands)} |
|
|
| else: |
| refs = {idx: [{'caption':r} for r in c_refs] for idx, c_refs in enumerate(refs)} |
| cands = {idx: [{'caption':c}] for idx, c in enumerate(cands)} |
|
|
| refs = tokenizer.tokenize(refs) |
| cands = tokenizer.tokenize(cands) |
|
|
| return refs, cands |
|
|
|
|
| def pycoco_eval(scorer, refs, cands): |
| ''' |
| scorer is assumed to have a compute_score function. |
| refs is a list of lists of strings |
| cands is a list of predictions |
| ''' |
| refs, cands = tokenize(refs, cands) |
| average_score, scores = scorer.compute_score(refs, cands) |
| return average_score, scores |
|
|
