{ "primary": "A2-cider", "tests": { "paper_vilbert": { "images": 5000, "oracle_cider": 1.6182621834020914, "metrics": { "first-beam": { "BLEU_1": 0.7521088640033475, "BLEU_2": 0.591623450790047, "BLEU_3": 0.4557287535721338, "BLEU_4": 0.35091666039484937, "CIDEr": 1.1149750812231705, "first_position_fraction": 1.0 }, "unchanged-bert-cnn": { "BLEU_1": 0.7402983239765452, "BLEU_2": 0.5757457487950892, "BLEU_3": 0.4342284148485376, "BLEU_4": 0.3258168163635417, "CIDEr": 1.0854576254544988, "first_position_fraction": 0.1234 }, "A1-position-only": { "BLEU_1": 0.7521088640033475, "BLEU_2": 0.591623450790047, "BLEU_3": 0.4557287535721338, "BLEU_4": 0.35091666039484937, "CIDEr": 1.1149750812231705, "first_position_fraction": 1.0 }, "A2-cider": { "BLEU_1": 0.7543677420694943, "BLEU_2": 0.5953936138426826, "BLEU_3": 0.4569996763225077, "BLEU_4": 0.35038564573334036, "CIDEr": 1.1474607323724648, "first_position_fraction": 0.6104 }, "A2b-cider-no-position": { "BLEU_1": 0.7349882653742538, "BLEU_2": 0.574064632808988, "BLEU_3": 0.4349227793617935, "BLEU_4": 0.3279217603767499, "CIDEr": 1.1201967590087958, "first_position_fraction": 0.0792 }, "A3-similarity": { "BLEU_1": 0.7567390153520221, "BLEU_2": 0.5967878186289933, "BLEU_3": 0.4553340280166201, "BLEU_4": 0.34532203526619026, "CIDEr": 1.1236317100690412, "first_position_fraction": 0.4184 }, "A4-mix": { "BLEU_1": 0.7548203516520693, "BLEU_2": 0.5943072626591133, "BLEU_3": 0.4557471546473046, "BLEU_4": 0.34916432806093023, "CIDEr": 1.1340847222949046, "first_position_fraction": 0.6976 }, "A5-cider-bce": { "BLEU_1": 0.7574909938546143, "BLEU_2": 0.5972567267723976, "BLEU_3": 0.45769277436880385, "BLEU_4": 0.3500147832234189, "CIDEr": 1.138433173459532, "first_position_fraction": 0.6734 }, "A2-cider-seed43": { "BLEU_1": 0.7541702331990072, "BLEU_2": 0.594308423725063, "BLEU_3": 0.4552943668707535, "BLEU_4": 0.34819763235786194, "CIDEr": 1.141830840194212, "first_position_fraction": 0.6404 }, "A2-cider-seed44": { "BLEU_1": 0.7536831546611341, "BLEU_2": 0.5939024570607181, "BLEU_3": 0.4554360547642447, "BLEU_4": 0.3487910562683307, "CIDEr": 1.1415035816492287, "first_position_fraction": 0.649 } }, "paired_cider": { "A1-position-only": { "first-beam": { "difference": 0.0, "ci95": [ 0.0, 0.0 ] }, "unchanged-bert-cnn": { "difference": 0.029517455768671607, "ci95": [ 0.013393474718649963, 0.045857509258577636 ] } }, "A2-cider": { "first-beam": { "difference": 0.03248565114929444, "ci95": [ 0.021257191765305255, 0.043816346383125496 ] }, "unchanged-bert-cnn": { "difference": 0.06200310691796606, "ci95": [ 0.04885120969712054, 0.07502684445185444 ] } }, "A2b-cider-no-position": { "first-beam": { "difference": 0.005221677785625393, "ci95": [ -0.010528698602534392, 0.021006286338626806 ] }, "unchanged-bert-cnn": { "difference": 0.03473913355429701, "ci95": [ 0.022716135169115907, 0.04680816402302316 ] } }, "A3-similarity": { "first-beam": { "difference": 0.008656628845870939, "ci95": [ -0.005162178870035752, 0.022361781396491313 ] }, "unchanged-bert-cnn": { "difference": 0.03817408461454255, "ci95": [ 0.026284460869596557, 0.050244014037753415 ] } }, "A4-mix": { "first-beam": { "difference": 0.01910964107173439, "ci95": [ 0.008700685679583426, 0.02960173203844372 ] }, "unchanged-bert-cnn": { "difference": 0.048627096840406, "ci95": [ 0.03582370057655935, 0.0611321669986398 ] } }, "A5-cider-bce": { "first-beam": { "difference": 0.02345809223636181, "ci95": [ 0.012693902384894774, 0.03436222518224583 ] }, "unchanged-bert-cnn": { "difference": 0.05297554800503342, "ci95": [ 0.04007371819842809, 0.06584487452205912 ] } }, "A2-cider-seed43": { "first-beam": { "difference": 0.026855758971042046, "ci95": [ 0.015854291701270503, 0.037843832407025886 ] }, "unchanged-bert-cnn": { "difference": 0.05637321473971365, "ci95": [ 0.04363054530483112, 0.06932505081453445 ] } }, "A2-cider-seed44": { "first-beam": { "difference": 0.02652850042605835, "ci95": [ 0.01569128166699668, 0.03741393660479623 ] }, "unchanged-bert-cnn": { "difference": 0.056045956194729955, "ci95": [ 0.04295636498490685, 0.0689257872719245 ] } } } }, "blip_in_domain": { "images": 5000, "oracle_cider": 1.5192155611503675, "metrics": { "first-beam": { "BLEU_1": 0.6972557211045558, "BLEU_2": 0.5589953953306551, "BLEU_3": 0.4375854037004591, "BLEU_4": 0.338047272629492, "CIDEr": 1.0631640320236821, "first_position_fraction": 1.0 }, "unchanged-bert-cnn": { "BLEU_1": 0.6927075614451419, "BLEU_2": 0.5467138200318495, "BLEU_3": 0.41722106465653414, "BLEU_4": 0.31308187825193373, "CIDEr": 1.0020373032616423, "first_position_fraction": 0.1024 }, "A1-position-only": { "BLEU_1": 0.6972557211045558, "BLEU_2": 0.5589953953306551, "BLEU_3": 0.4375854037004591, "BLEU_4": 0.338047272629492, "CIDEr": 1.0631640320236821, "first_position_fraction": 1.0 }, "A2-cider": { "BLEU_1": 0.7451779342194806, "BLEU_2": 0.602038711524827, "BLEU_3": 0.47015388585036233, "BLEU_4": 0.36119635247762377, "CIDEr": 1.1469812911562853, "first_position_fraction": 0.597 }, "A2b-cider-no-position": { "BLEU_1": 0.7492076721661279, "BLEU_2": 0.6001318679611358, "BLEU_3": 0.4626851747509669, "BLEU_4": 0.3497497264938604, "CIDEr": 1.1210157484672216, "first_position_fraction": 0.1508 }, "A3-similarity": { "BLEU_1": 0.7270299442143029, "BLEU_2": 0.5884846320744176, "BLEU_3": 0.4578148682664639, "BLEU_4": 0.3501662789530852, "CIDEr": 1.1121125931829168, "first_position_fraction": 0.419 }, "A4-mix": { "BLEU_1": 0.7241440818235776, "BLEU_2": 0.5832507254587247, "BLEU_3": 0.4542950873251976, "BLEU_4": 0.3483276836523094, "CIDEr": 1.1075998944881613, "first_position_fraction": 0.6734 }, "A5-cider-bce": { "BLEU_1": 0.7327806336397135, "BLEU_2": 0.5927275276644527, "BLEU_3": 0.46321467684746426, "BLEU_4": 0.35607684672554313, "CIDEr": 1.1298572297403193, "first_position_fraction": 0.649 }, "A2-cider-seed43": { "BLEU_1": 0.7432503976837088, "BLEU_2": 0.6010193575130967, "BLEU_3": 0.46965775473868077, "BLEU_4": 0.3607841801407428, "CIDEr": 1.1461092136840518, "first_position_fraction": 0.6182 }, "A2-cider-seed44": { "BLEU_1": 0.7419365512853906, "BLEU_2": 0.5997652510833471, "BLEU_3": 0.46847684409748, "BLEU_4": 0.35962784991833674, "CIDEr": 1.1420322976633694, "first_position_fraction": 0.62 } }, "paired_cider": { "A1-position-only": { "first-beam": { "difference": 0.0, "ci95": [ 0.0, 0.0 ] }, "unchanged-bert-cnn": { "difference": 0.06112672876203984, "ci95": [ 0.04461033866764987, 0.07732889442325831 ] } }, "A2-cider": { "first-beam": { "difference": 0.08381725913260307, "ci95": [ 0.0727875544504028, 0.0952077601843989 ] }, "unchanged-bert-cnn": { "difference": 0.1449439878946429, "ci95": [ 0.13123145918065157, 0.1584672668531495 ] } }, "A2b-cider-no-position": { "first-beam": { "difference": 0.057851716443539314, "ci95": [ 0.04271872527034004, 0.07325111089796867 ] }, "unchanged-bert-cnn": { "difference": 0.11897844520557917, "ci95": [ 0.1065857808370289, 0.13147903804945957 ] } }, "A3-similarity": { "first-beam": { "difference": 0.04894856115923461, "ci95": [ 0.035741020466093225, 0.06260445814702015 ] }, "unchanged-bert-cnn": { "difference": 0.11007528992127445, "ci95": [ 0.09785163686033818, 0.1229914533939251 ] } }, "A4-mix": { "first-beam": { "difference": 0.04443586246447917, "ci95": [ 0.03405757119093585, 0.05511339311451561 ] }, "unchanged-bert-cnn": { "difference": 0.105562591226519, "ci95": [ 0.09213401867542881, 0.11889410360514222 ] } }, "A5-cider-bce": { "first-beam": { "difference": 0.06669319771663693, "ci95": [ 0.05628127853231627, 0.07764260246364937 ] }, "unchanged-bert-cnn": { "difference": 0.12781992647867677, "ci95": [ 0.11451074580965637, 0.14125596300371077 ] } }, "A2-cider-seed43": { "first-beam": { "difference": 0.0829451816603695, "ci95": [ 0.07194278777091113, 0.09434764973335309 ] }, "unchanged-bert-cnn": { "difference": 0.14407191042240938, "ci95": [ 0.1303761099103887, 0.15753553193319397 ] } }, "A2-cider-seed44": { "first-beam": { "difference": 0.07886826563968741, "ci95": [ 0.0679104338592083, 0.09013364703943051 ] }, "unchanged-bert-cnn": { "difference": 0.13999499440172725, "ci95": [ 0.12623425125883314, 0.15353143250912157 ] } } } } }, "uncertainty": "Paired image bootstrap, 10,000 resamples, fixed reference IDF; not training-seed variation", "not_evaluated_here": [ "METEOR", "ROUGE_L", "SPICE", "BERTScore" ], "published": false }