AhmedSSabir's picture
Add BERT-CNN reward caption reranker with evaluation
4ae724e verified
Raw History Blame Contribute Delete
10.1 kB
{
"results": {
"paper_vilbert/first-beam": {
"metrics": {
"BLEU_1": 0.7521088640033475,
"BLEU_4": 0.35091666039484937,
"METEOR": 0.2740649213423215,
"ROUGE_L": 0.55730837007525,
"CIDEr": 1.1149750812231705,
"SPICE": 0.2051467123007049,
"BERTScore_F1": 0.9405835866928101
}
},
"paper_vilbert/A2-cider": {
"metrics": {
"BLEU_1": 0.7543677420694943,
"BLEU_4": 0.35038564573334036,
"METEOR": 0.2794252054355059,
"ROUGE_L": 0.5624724309601203,
"CIDEr": 1.1474607323724648,
"SPICE": 0.2119032114095339,
"BERTScore_F1": 0.9412955045700073
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.03248565114929445,
"ci95": [
0.021273590435114172,
0.04381817372846462
]
},
"SPICE": {
"difference": 0.006756499108828964,
"ci95": [
0.005064563632588412,
0.008515563454492548
]
},
"BERTScore": {
"difference": 0.0007119835257530212,
"ci95": [
0.0003816543799638748,
0.0010461517238616943
]
}
}
},
"paper_vilbert/A2-cider-seed43": {
"metrics": {
"BLEU_1": 0.7541702331990072,
"BLEU_4": 0.34819763235786194,
"METEOR": 0.2787522418418623,
"ROUGE_L": 0.5608437344904296,
"CIDEr": 1.1418308401942123,
"SPICE": 0.21100101218968775,
"BERTScore_F1": 0.9412525296211243
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.026855758971042053,
"ci95": [
0.01573842950042596,
0.03774575797807877
]
},
"SPICE": {
"difference": 0.005854299888982864,
"ci95": [
0.004230277004097649,
0.007545450207437246
]
},
"BERTScore": {
"difference": 0.0006690580606460571,
"ci95": [
0.0003416048058867455,
0.0010037949207425117
]
}
}
},
"paper_vilbert/A2-cider-seed44": {
"metrics": {
"BLEU_1": 0.7536831546611341,
"BLEU_4": 0.3487910562683307,
"METEOR": 0.27876933022763234,
"ROUGE_L": 0.5611273923252967,
"CIDEr": 1.1415035816492287,
"SPICE": 0.21069880850916087,
"BERTScore_F1": 0.941270112991333
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.026528500426058355,
"ci95": [
0.015461564008056681,
0.03719260043088689
]
},
"SPICE": {
"difference": 0.005552096208455978,
"ci95": [
0.003921174774884093,
0.007217001777398386
]
},
"BERTScore": {
"difference": 0.0006866474270820617,
"ci95": [
0.00036229795545339585,
0.0010176115068793294
]
}
}
},
"paper_vilbert/A5-cider-bce": {
"metrics": {
"BLEU_1": 0.7574909938546143,
"BLEU_4": 0.3500147832234189,
"METEOR": 0.2774411435162455,
"ROUGE_L": 0.5613011089891018,
"CIDEr": 1.138433173459532,
"SPICE": 0.2094046320227589,
"BERTScore_F1": 0.9411762952804565
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.02345809223636181,
"ci95": [
0.012649300630465888,
0.03436401651784764
]
},
"SPICE": {
"difference": 0.004257919722053996,
"ci95": [
0.002680062928676328,
0.005822506461482289
]
},
"BERTScore": {
"difference": 0.0005927220940589905,
"ci95": [
0.0002773158437013626,
0.0009150636693835257
]
}
}
},
"paper_vilbert/A3-similarity": {
"metrics": {
"BLEU_1": 0.7567390153520221,
"BLEU_4": 0.34532203526619026,
"METEOR": 0.2746760806523903,
"ROUGE_L": 0.5581204003716966,
"CIDEr": 1.1236317100690412,
"SPICE": 0.20939972263298126,
"BERTScore_F1": 0.9409727454185486
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.008656628845870942,
"ci95": [
-0.005240936708872434,
0.022294550429155506
]
},
"SPICE": {
"difference": 0.004253010332276365,
"ci95": [
0.0021455036363641974,
0.006455195285110698
]
},
"BERTScore": {
"difference": 0.00038920217752456667,
"ci95": [
-5.482336878776545e-06,
0.0007888154581189152
]
}
}
},
"paper_vilbert/A4-mix": {
"metrics": {
"BLEU_1": 0.7548203516520693,
"BLEU_4": 0.34916432806093023,
"METEOR": 0.27727195666860444,
"ROUGE_L": 0.5599557599251835,
"CIDEr": 1.1340847222949046,
"SPICE": 0.20926844383712132,
"BERTScore_F1": 0.9410042762756348
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.019109641071734386,
"ci95": [
0.00842330897878974,
0.029478617264110704
]
},
"SPICE": {
"difference": 0.004121731536416366,
"ci95": [
0.0025829483501072035,
0.005695482341309813
]
},
"BERTScore": {
"difference": 0.0004207610845565796,
"ci95": [
0.00011979014217853545,
0.0007201787564158439
]
}
}
},
"paper_vilbert/A2b-cider-no-position": {
"metrics": {
"BLEU_1": 0.7349882653742538,
"BLEU_4": 0.3279217603767499,
"METEOR": 0.28095047529732425,
"ROUGE_L": 0.556251651422652,
"CIDEr": 1.1201967590087958,
"SPICE": 0.21534489968484521,
"BERTScore_F1": 0.9401296973228455
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.005221677785625393,
"ci95": [
-0.010827136039780402,
0.021138612213461378
]
},
"SPICE": {
"difference": 0.010198187384140286,
"ci95": [
0.007624108874537086,
0.012760126964106425
]
},
"BERTScore": {
"difference": -0.0004538540959358215,
"ci95": [
-0.0009028402432799339,
-9.768431782722619e-06
]
}
}
},
"paper_vilbert/unchanged-bert-cnn": {
"metrics": {
"BLEU_1": 0.7402983239765452,
"BLEU_4": 0.3258168163635417,
"METEOR": 0.27156185747101835,
"ROUGE_L": 0.5485864278264938,
"CIDEr": 1.085457625454499,
"SPICE": 0.20556140379403381,
"BERTScore_F1": 0.9393018484115601
},
"vs_first_beam": {
"CIDEr": {
"difference": -0.029517455768671607,
"ci95": [
-0.045417418545184024,
-0.013603843796331073
]
},
"SPICE": {
"difference": 0.00041469149332888405,
"ci95": [
-0.002018283043677736,
0.0029015339359160126
]
},
"BERTScore": {
"difference": -0.00128170405626297,
"ci95": [
-0.001723966481089592,
-0.0008351772513985635
]
}
}
},
"blip_in_domain/first-beam": {
"metrics": {
"BLEU_1": 0.6972557211045558,
"BLEU_4": 0.338047272629492,
"METEOR": 0.25417657356781453,
"ROUGE_L": 0.5347548020857551,
"CIDEr": 1.0631640320236824,
"SPICE": 0.19848560541377797,
"BERTScore_F1": 0.9363407492637634
}
},
"blip_in_domain/unchanged-bert-cnn": {
"metrics": {
"BLEU_1": 0.6927075614451419,
"BLEU_4": 0.31308187825193373,
"METEOR": 0.24749698021813377,
"ROUGE_L": 0.5309808503782891,
"CIDEr": 1.0020373032616425,
"SPICE": 0.1886035723550942,
"BERTScore_F1": 0.9362419843673706
},
"vs_first_beam": {
"CIDEr": {
"difference": -0.06112672876203984,
"ci95": [
-0.07703248993243476,
-0.04466810364650549
]
},
"SPICE": {
"difference": -0.009882033058683756,
"ci95": [
-0.012468651209439591,
-0.007370175182655841
]
},
"BERTScore": {
"difference": -9.878027776721865e-05,
"ci95": [
-0.0006621658074436709,
0.00047508578209090043
]
}
}
},
"blip_in_domain/A2-cider": {
"metrics": {
"BLEU_1": 0.7451779342194806,
"BLEU_4": 0.36119635247762377,
"METEOR": 0.26878435097515646,
"ROUGE_L": 0.564238028759961,
"CIDEr": 1.1469812911562853,
"SPICE": 0.2090844645720823,
"BERTScore_F1": 0.9410110116004944
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.08381725913260307,
"ci95": [
0.07264195855986173,
0.0950220839363521
]
},
"SPICE": {
"difference": 0.010598859158304288,
"ci95": [
0.008764631329274094,
0.012371659436999622
]
},
"BERTScore": {
"difference": 0.004670334048569202,
"ci95": [
0.0042247722623869775,
0.005140163039322942
]
}
}
},
"blip_in_domain/A3-similarity": {
"metrics": {
"BLEU_1": 0.7270299442143029,
"BLEU_4": 0.3501662789530852,
"METEOR": 0.2629617379523437,
"ROUGE_L": 0.5606769902901408,
"CIDEr": 1.112112593182917,
"SPICE": 0.2037673968122235,
"BERTScore_F1": 0.9406426548957825
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.04894856115923462,
"ci95": [
0.03589291862594552,
0.06221725785689125
]
},
"SPICE": {
"difference": 0.005281791398445559,
"ci95": [
0.0031942825409635583,
0.007364935030469625
]
},
"BERTScore": {
"difference": 0.004301912151277065,
"ci95": [
0.003804747149115428,
0.0048193004098720845
]
}
}
},
"blip_in_domain/A5-cider-bce": {
"metrics": {
"BLEU_1": 0.7327806336397135,
"BLEU_4": 0.35607684672554313,
"METEOR": 0.26600139071984985,
"ROUGE_L": 0.5605545245545978,
"CIDEr": 1.1298572297403193,
"SPICE": 0.2070218030565185,
"BERTScore_F1": 0.940541684627533
},
"vs_first_beam": {
"CIDEr": {
"difference": 0.06669319771663691,
"ci95": [
0.056269795137920055,
0.07739607247334784
]
},
"SPICE": {
"difference": 0.008536197642740549,
"ci95": [
0.006805229287105627,
0.010291442021231376
]
},
"BERTScore": {
"difference": 0.004200989380478859,
"ci95": [
0.0037530666100792586,
0.004658673680387437
]
}
}
}
},
"settings": {
"spice": "SPICE 1.0 with Stanford CoreNLP 3.6.0 jars from Maven Central (SHA1 verified)",
"bertscore": "bert-score 0.3.12 default lang=en (roberta-large, layer 17), no baseline rescaling",
"bootstrap": "Paired image bootstrap, 5,000 resamples, seed 42, vs BLIP/ViLBERT first beam",
"note": "BERTScore setup reads ~.0043 higher than the paper; compare only within measured rows"
}
}