AhmedSSabir's picture
Add BERT-CNN reward caption reranker with evaluation
4ae724e verified
Raw History Blame Contribute Delete
12.9 kB
{
"primary": "A2-cider",
"tests": {
"paper_vilbert": {
"images": 5000,
"oracle_cider": 1.6182621834020914,
"metrics": {
"first-beam": {
"BLEU_1": 0.7521088640033475,
"BLEU_2": 0.591623450790047,
"BLEU_3": 0.4557287535721338,
"BLEU_4": 0.35091666039484937,
"CIDEr": 1.1149750812231705,
"first_position_fraction": 1.0
},
"unchanged-bert-cnn": {
"BLEU_1": 0.7402983239765452,
"BLEU_2": 0.5757457487950892,
"BLEU_3": 0.4342284148485376,
"BLEU_4": 0.3258168163635417,
"CIDEr": 1.0854576254544988,
"first_position_fraction": 0.1234
},
"A1-position-only": {
"BLEU_1": 0.7521088640033475,
"BLEU_2": 0.591623450790047,
"BLEU_3": 0.4557287535721338,
"BLEU_4": 0.35091666039484937,
"CIDEr": 1.1149750812231705,
"first_position_fraction": 1.0
},
"A2-cider": {
"BLEU_1": 0.7543677420694943,
"BLEU_2": 0.5953936138426826,
"BLEU_3": 0.4569996763225077,
"BLEU_4": 0.35038564573334036,
"CIDEr": 1.1474607323724648,
"first_position_fraction": 0.6104
},
"A2b-cider-no-position": {
"BLEU_1": 0.7349882653742538,
"BLEU_2": 0.574064632808988,
"BLEU_3": 0.4349227793617935,
"BLEU_4": 0.3279217603767499,
"CIDEr": 1.1201967590087958,
"first_position_fraction": 0.0792
},
"A3-similarity": {
"BLEU_1": 0.7567390153520221,
"BLEU_2": 0.5967878186289933,
"BLEU_3": 0.4553340280166201,
"BLEU_4": 0.34532203526619026,
"CIDEr": 1.1236317100690412,
"first_position_fraction": 0.4184
},
"A4-mix": {
"BLEU_1": 0.7548203516520693,
"BLEU_2": 0.5943072626591133,
"BLEU_3": 0.4557471546473046,
"BLEU_4": 0.34916432806093023,
"CIDEr": 1.1340847222949046,
"first_position_fraction": 0.6976
},
"A5-cider-bce": {
"BLEU_1": 0.7574909938546143,
"BLEU_2": 0.5972567267723976,
"BLEU_3": 0.45769277436880385,
"BLEU_4": 0.3500147832234189,
"CIDEr": 1.138433173459532,
"first_position_fraction": 0.6734
},
"A2-cider-seed43": {
"BLEU_1": 0.7541702331990072,
"BLEU_2": 0.594308423725063,
"BLEU_3": 0.4552943668707535,
"BLEU_4": 0.34819763235786194,
"CIDEr": 1.141830840194212,
"first_position_fraction": 0.6404
},
"A2-cider-seed44": {
"BLEU_1": 0.7536831546611341,
"BLEU_2": 0.5939024570607181,
"BLEU_3": 0.4554360547642447,
"BLEU_4": 0.3487910562683307,
"CIDEr": 1.1415035816492287,
"first_position_fraction": 0.649
}
},
"paired_cider": {
"A1-position-only": {
"first-beam": {
"difference": 0.0,
"ci95": [
0.0,
0.0
]
},
"unchanged-bert-cnn": {
"difference": 0.029517455768671607,
"ci95": [
0.013393474718649963,
0.045857509258577636
]
}
},
"A2-cider": {
"first-beam": {
"difference": 0.03248565114929444,
"ci95": [
0.021257191765305255,
0.043816346383125496
]
},
"unchanged-bert-cnn": {
"difference": 0.06200310691796606,
"ci95": [
0.04885120969712054,
0.07502684445185444
]
}
},
"A2b-cider-no-position": {
"first-beam": {
"difference": 0.005221677785625393,
"ci95": [
-0.010528698602534392,
0.021006286338626806
]
},
"unchanged-bert-cnn": {
"difference": 0.03473913355429701,
"ci95": [
0.022716135169115907,
0.04680816402302316
]
}
},
"A3-similarity": {
"first-beam": {
"difference": 0.008656628845870939,
"ci95": [
-0.005162178870035752,
0.022361781396491313
]
},
"unchanged-bert-cnn": {
"difference": 0.03817408461454255,
"ci95": [
0.026284460869596557,
0.050244014037753415
]
}
},
"A4-mix": {
"first-beam": {
"difference": 0.01910964107173439,
"ci95": [
0.008700685679583426,
0.02960173203844372
]
},
"unchanged-bert-cnn": {
"difference": 0.048627096840406,
"ci95": [
0.03582370057655935,
0.0611321669986398
]
}
},
"A5-cider-bce": {
"first-beam": {
"difference": 0.02345809223636181,
"ci95": [
0.012693902384894774,
0.03436222518224583
]
},
"unchanged-bert-cnn": {
"difference": 0.05297554800503342,
"ci95": [
0.04007371819842809,
0.06584487452205912
]
}
},
"A2-cider-seed43": {
"first-beam": {
"difference": 0.026855758971042046,
"ci95": [
0.015854291701270503,
0.037843832407025886
]
},
"unchanged-bert-cnn": {
"difference": 0.05637321473971365,
"ci95": [
0.04363054530483112,
0.06932505081453445
]
}
},
"A2-cider-seed44": {
"first-beam": {
"difference": 0.02652850042605835,
"ci95": [
0.01569128166699668,
0.03741393660479623
]
},
"unchanged-bert-cnn": {
"difference": 0.056045956194729955,
"ci95": [
0.04295636498490685,
0.0689257872719245
]
}
}
}
},
"blip_in_domain": {
"images": 5000,
"oracle_cider": 1.5192155611503675,
"metrics": {
"first-beam": {
"BLEU_1": 0.6972557211045558,
"BLEU_2": 0.5589953953306551,
"BLEU_3": 0.4375854037004591,
"BLEU_4": 0.338047272629492,
"CIDEr": 1.0631640320236821,
"first_position_fraction": 1.0
},
"unchanged-bert-cnn": {
"BLEU_1": 0.6927075614451419,
"BLEU_2": 0.5467138200318495,
"BLEU_3": 0.41722106465653414,
"BLEU_4": 0.31308187825193373,
"CIDEr": 1.0020373032616423,
"first_position_fraction": 0.1024
},
"A1-position-only": {
"BLEU_1": 0.6972557211045558,
"BLEU_2": 0.5589953953306551,
"BLEU_3": 0.4375854037004591,
"BLEU_4": 0.338047272629492,
"CIDEr": 1.0631640320236821,
"first_position_fraction": 1.0
},
"A2-cider": {
"BLEU_1": 0.7451779342194806,
"BLEU_2": 0.602038711524827,
"BLEU_3": 0.47015388585036233,
"BLEU_4": 0.36119635247762377,
"CIDEr": 1.1469812911562853,
"first_position_fraction": 0.597
},
"A2b-cider-no-position": {
"BLEU_1": 0.7492076721661279,
"BLEU_2": 0.6001318679611358,
"BLEU_3": 0.4626851747509669,
"BLEU_4": 0.3497497264938604,
"CIDEr": 1.1210157484672216,
"first_position_fraction": 0.1508
},
"A3-similarity": {
"BLEU_1": 0.7270299442143029,
"BLEU_2": 0.5884846320744176,
"BLEU_3": 0.4578148682664639,
"BLEU_4": 0.3501662789530852,
"CIDEr": 1.1121125931829168,
"first_position_fraction": 0.419
},
"A4-mix": {
"BLEU_1": 0.7241440818235776,
"BLEU_2": 0.5832507254587247,
"BLEU_3": 0.4542950873251976,
"BLEU_4": 0.3483276836523094,
"CIDEr": 1.1075998944881613,
"first_position_fraction": 0.6734
},
"A5-cider-bce": {
"BLEU_1": 0.7327806336397135,
"BLEU_2": 0.5927275276644527,
"BLEU_3": 0.46321467684746426,
"BLEU_4": 0.35607684672554313,
"CIDEr": 1.1298572297403193,
"first_position_fraction": 0.649
},
"A2-cider-seed43": {
"BLEU_1": 0.7432503976837088,
"BLEU_2": 0.6010193575130967,
"BLEU_3": 0.46965775473868077,
"BLEU_4": 0.3607841801407428,
"CIDEr": 1.1461092136840518,
"first_position_fraction": 0.6182
},
"A2-cider-seed44": {
"BLEU_1": 0.7419365512853906,
"BLEU_2": 0.5997652510833471,
"BLEU_3": 0.46847684409748,
"BLEU_4": 0.35962784991833674,
"CIDEr": 1.1420322976633694,
"first_position_fraction": 0.62
}
},
"paired_cider": {
"A1-position-only": {
"first-beam": {
"difference": 0.0,
"ci95": [
0.0,
0.0
]
},
"unchanged-bert-cnn": {
"difference": 0.06112672876203984,
"ci95": [
0.04461033866764987,
0.07732889442325831
]
}
},
"A2-cider": {
"first-beam": {
"difference": 0.08381725913260307,
"ci95": [
0.0727875544504028,
0.0952077601843989
]
},
"unchanged-bert-cnn": {
"difference": 0.1449439878946429,
"ci95": [
0.13123145918065157,
0.1584672668531495
]
}
},
"A2b-cider-no-position": {
"first-beam": {
"difference": 0.057851716443539314,
"ci95": [
0.04271872527034004,
0.07325111089796867
]
},
"unchanged-bert-cnn": {
"difference": 0.11897844520557917,
"ci95": [
0.1065857808370289,
0.13147903804945957
]
}
},
"A3-similarity": {
"first-beam": {
"difference": 0.04894856115923461,
"ci95": [
0.035741020466093225,
0.06260445814702015
]
},
"unchanged-bert-cnn": {
"difference": 0.11007528992127445,
"ci95": [
0.09785163686033818,
0.1229914533939251
]
}
},
"A4-mix": {
"first-beam": {
"difference": 0.04443586246447917,
"ci95": [
0.03405757119093585,
0.05511339311451561
]
},
"unchanged-bert-cnn": {
"difference": 0.105562591226519,
"ci95": [
0.09213401867542881,
0.11889410360514222
]
}
},
"A5-cider-bce": {
"first-beam": {
"difference": 0.06669319771663693,
"ci95": [
0.05628127853231627,
0.07764260246364937
]
},
"unchanged-bert-cnn": {
"difference": 0.12781992647867677,
"ci95": [
0.11451074580965637,
0.14125596300371077
]
}
},
"A2-cider-seed43": {
"first-beam": {
"difference": 0.0829451816603695,
"ci95": [
0.07194278777091113,
0.09434764973335309
]
},
"unchanged-bert-cnn": {
"difference": 0.14407191042240938,
"ci95": [
0.1303761099103887,
0.15753553193319397
]
}
},
"A2-cider-seed44": {
"first-beam": {
"difference": 0.07886826563968741,
"ci95": [
0.0679104338592083,
0.09013364703943051
]
},
"unchanged-bert-cnn": {
"difference": 0.13999499440172725,
"ci95": [
0.12623425125883314,
0.15353143250912157
]
}
}
}
}
},
"uncertainty": "Paired image bootstrap, 10,000 resamples, fixed reference IDF; not training-seed variation",
"not_evaluated_here": [
"METEOR",
"ROUGE_L",
"SPICE",
"BERTScore"
],
"published": false
}