{ "results": { "paper_vilbert/first-beam": { "metrics": { "BLEU_1": 0.7521088640033475, "BLEU_4": 0.35091666039484937, "METEOR": 0.2740649213423215, "ROUGE_L": 0.55730837007525, "CIDEr": 1.1149750812231705, "SPICE": 0.2051467123007049, "BERTScore_F1": 0.9405835866928101 } }, "paper_vilbert/A2-cider": { "metrics": { "BLEU_1": 0.7543677420694943, "BLEU_4": 0.35038564573334036, "METEOR": 0.2794252054355059, "ROUGE_L": 0.5624724309601203, "CIDEr": 1.1474607323724648, "SPICE": 0.2119032114095339, "BERTScore_F1": 0.9412955045700073 }, "vs_first_beam": { "CIDEr": { "difference": 0.03248565114929445, "ci95": [ 0.021273590435114172, 0.04381817372846462 ] }, "SPICE": { "difference": 0.006756499108828964, "ci95": [ 0.005064563632588412, 0.008515563454492548 ] }, "BERTScore": { "difference": 0.0007119835257530212, "ci95": [ 0.0003816543799638748, 0.0010461517238616943 ] } } }, "paper_vilbert/A2-cider-seed43": { "metrics": { "BLEU_1": 0.7541702331990072, "BLEU_4": 0.34819763235786194, "METEOR": 0.2787522418418623, "ROUGE_L": 0.5608437344904296, "CIDEr": 1.1418308401942123, "SPICE": 0.21100101218968775, "BERTScore_F1": 0.9412525296211243 }, "vs_first_beam": { "CIDEr": { "difference": 0.026855758971042053, "ci95": [ 0.01573842950042596, 0.03774575797807877 ] }, "SPICE": { "difference": 0.005854299888982864, "ci95": [ 0.004230277004097649, 0.007545450207437246 ] }, "BERTScore": { "difference": 0.0006690580606460571, "ci95": [ 0.0003416048058867455, 0.0010037949207425117 ] } } }, "paper_vilbert/A2-cider-seed44": { "metrics": { "BLEU_1": 0.7536831546611341, "BLEU_4": 0.3487910562683307, "METEOR": 0.27876933022763234, "ROUGE_L": 0.5611273923252967, "CIDEr": 1.1415035816492287, "SPICE": 0.21069880850916087, "BERTScore_F1": 0.941270112991333 }, "vs_first_beam": { "CIDEr": { "difference": 0.026528500426058355, "ci95": [ 0.015461564008056681, 0.03719260043088689 ] }, "SPICE": { "difference": 0.005552096208455978, "ci95": [ 0.003921174774884093, 0.007217001777398386 ] }, "BERTScore": { "difference": 0.0006866474270820617, "ci95": [ 0.00036229795545339585, 0.0010176115068793294 ] } } }, "paper_vilbert/A5-cider-bce": { "metrics": { "BLEU_1": 0.7574909938546143, "BLEU_4": 0.3500147832234189, "METEOR": 0.2774411435162455, "ROUGE_L": 0.5613011089891018, "CIDEr": 1.138433173459532, "SPICE": 0.2094046320227589, "BERTScore_F1": 0.9411762952804565 }, "vs_first_beam": { "CIDEr": { "difference": 0.02345809223636181, "ci95": [ 0.012649300630465888, 0.03436401651784764 ] }, "SPICE": { "difference": 0.004257919722053996, "ci95": [ 0.002680062928676328, 0.005822506461482289 ] }, "BERTScore": { "difference": 0.0005927220940589905, "ci95": [ 0.0002773158437013626, 0.0009150636693835257 ] } } }, "paper_vilbert/A3-similarity": { "metrics": { "BLEU_1": 0.7567390153520221, "BLEU_4": 0.34532203526619026, "METEOR": 0.2746760806523903, "ROUGE_L": 0.5581204003716966, "CIDEr": 1.1236317100690412, "SPICE": 0.20939972263298126, "BERTScore_F1": 0.9409727454185486 }, "vs_first_beam": { "CIDEr": { "difference": 0.008656628845870942, "ci95": [ -0.005240936708872434, 0.022294550429155506 ] }, "SPICE": { "difference": 0.004253010332276365, "ci95": [ 0.0021455036363641974, 0.006455195285110698 ] }, "BERTScore": { "difference": 0.00038920217752456667, "ci95": [ -5.482336878776545e-06, 0.0007888154581189152 ] } } }, "paper_vilbert/A4-mix": { "metrics": { "BLEU_1": 0.7548203516520693, "BLEU_4": 0.34916432806093023, "METEOR": 0.27727195666860444, "ROUGE_L": 0.5599557599251835, "CIDEr": 1.1340847222949046, "SPICE": 0.20926844383712132, "BERTScore_F1": 0.9410042762756348 }, "vs_first_beam": { "CIDEr": { "difference": 0.019109641071734386, "ci95": [ 0.00842330897878974, 0.029478617264110704 ] }, "SPICE": { "difference": 0.004121731536416366, "ci95": [ 0.0025829483501072035, 0.005695482341309813 ] }, "BERTScore": { "difference": 0.0004207610845565796, "ci95": [ 0.00011979014217853545, 0.0007201787564158439 ] } } }, "paper_vilbert/A2b-cider-no-position": { "metrics": { "BLEU_1": 0.7349882653742538, "BLEU_4": 0.3279217603767499, "METEOR": 0.28095047529732425, "ROUGE_L": 0.556251651422652, "CIDEr": 1.1201967590087958, "SPICE": 0.21534489968484521, "BERTScore_F1": 0.9401296973228455 }, "vs_first_beam": { "CIDEr": { "difference": 0.005221677785625393, "ci95": [ -0.010827136039780402, 0.021138612213461378 ] }, "SPICE": { "difference": 0.010198187384140286, "ci95": [ 0.007624108874537086, 0.012760126964106425 ] }, "BERTScore": { "difference": -0.0004538540959358215, "ci95": [ -0.0009028402432799339, -9.768431782722619e-06 ] } } }, "paper_vilbert/unchanged-bert-cnn": { "metrics": { "BLEU_1": 0.7402983239765452, "BLEU_4": 0.3258168163635417, "METEOR": 0.27156185747101835, "ROUGE_L": 0.5485864278264938, "CIDEr": 1.085457625454499, "SPICE": 0.20556140379403381, "BERTScore_F1": 0.9393018484115601 }, "vs_first_beam": { "CIDEr": { "difference": -0.029517455768671607, "ci95": [ -0.045417418545184024, -0.013603843796331073 ] }, "SPICE": { "difference": 0.00041469149332888405, "ci95": [ -0.002018283043677736, 0.0029015339359160126 ] }, "BERTScore": { "difference": -0.00128170405626297, "ci95": [ -0.001723966481089592, -0.0008351772513985635 ] } } }, "blip_in_domain/first-beam": { "metrics": { "BLEU_1": 0.6972557211045558, "BLEU_4": 0.338047272629492, "METEOR": 0.25417657356781453, "ROUGE_L": 0.5347548020857551, "CIDEr": 1.0631640320236824, "SPICE": 0.19848560541377797, "BERTScore_F1": 0.9363407492637634 } }, "blip_in_domain/unchanged-bert-cnn": { "metrics": { "BLEU_1": 0.6927075614451419, "BLEU_4": 0.31308187825193373, "METEOR": 0.24749698021813377, "ROUGE_L": 0.5309808503782891, "CIDEr": 1.0020373032616425, "SPICE": 0.1886035723550942, "BERTScore_F1": 0.9362419843673706 }, "vs_first_beam": { "CIDEr": { "difference": -0.06112672876203984, "ci95": [ -0.07703248993243476, -0.04466810364650549 ] }, "SPICE": { "difference": -0.009882033058683756, "ci95": [ -0.012468651209439591, -0.007370175182655841 ] }, "BERTScore": { "difference": -9.878027776721865e-05, "ci95": [ -0.0006621658074436709, 0.00047508578209090043 ] } } }, "blip_in_domain/A2-cider": { "metrics": { "BLEU_1": 0.7451779342194806, "BLEU_4": 0.36119635247762377, "METEOR": 0.26878435097515646, "ROUGE_L": 0.564238028759961, "CIDEr": 1.1469812911562853, "SPICE": 0.2090844645720823, "BERTScore_F1": 0.9410110116004944 }, "vs_first_beam": { "CIDEr": { "difference": 0.08381725913260307, "ci95": [ 0.07264195855986173, 0.0950220839363521 ] }, "SPICE": { "difference": 0.010598859158304288, "ci95": [ 0.008764631329274094, 0.012371659436999622 ] }, "BERTScore": { "difference": 0.004670334048569202, "ci95": [ 0.0042247722623869775, 0.005140163039322942 ] } } }, "blip_in_domain/A3-similarity": { "metrics": { "BLEU_1": 0.7270299442143029, "BLEU_4": 0.3501662789530852, "METEOR": 0.2629617379523437, "ROUGE_L": 0.5606769902901408, "CIDEr": 1.112112593182917, "SPICE": 0.2037673968122235, "BERTScore_F1": 0.9406426548957825 }, "vs_first_beam": { "CIDEr": { "difference": 0.04894856115923462, "ci95": [ 0.03589291862594552, 0.06221725785689125 ] }, "SPICE": { "difference": 0.005281791398445559, "ci95": [ 0.0031942825409635583, 0.007364935030469625 ] }, "BERTScore": { "difference": 0.004301912151277065, "ci95": [ 0.003804747149115428, 0.0048193004098720845 ] } } }, "blip_in_domain/A5-cider-bce": { "metrics": { "BLEU_1": 0.7327806336397135, "BLEU_4": 0.35607684672554313, "METEOR": 0.26600139071984985, "ROUGE_L": 0.5605545245545978, "CIDEr": 1.1298572297403193, "SPICE": 0.2070218030565185, "BERTScore_F1": 0.940541684627533 }, "vs_first_beam": { "CIDEr": { "difference": 0.06669319771663691, "ci95": [ 0.056269795137920055, 0.07739607247334784 ] }, "SPICE": { "difference": 0.008536197642740549, "ci95": [ 0.006805229287105627, 0.010291442021231376 ] }, "BERTScore": { "difference": 0.004200989380478859, "ci95": [ 0.0037530666100792586, 0.004658673680387437 ] } } } }, "settings": { "spice": "SPICE 1.0 with Stanford CoreNLP 3.6.0 jars from Maven Central (SHA1 verified)", "bertscore": "bert-score 0.3.12 default lang=en (roberta-large, layer 17), no baseline rescaling", "bootstrap": "Paired image bootstrap, 5,000 resamples, seed 42, vs BLIP/ViLBERT first beam", "note": "BERTScore setup reads ~.0043 higher than the paper; compare only within measured rows" } }