AhmedSSabir's picture
Add BERT-CNN reward caption reranker with evaluation
4ae724e verified
Raw History Blame Contribute Delete
1.76 kB
{
"dataset": "Original Flickr8K ExpertAnnotations.txt",
"pairs": 5822,
"images": 1000,
"checkpoint_sha256": "125a083e3258b7ee038b6fb3e9d40a2d6078e2a325fbc1c4ba9cd062e674daaf",
"metrics": {
"clipscore_vit_b32": {
"examples": 5822,
"spearman": 0.7273458253579359,
"kendall_tau_b": 0.5760866201430647,
"kendall_tau_c": 0.5631096474308253,
"pearson": 0.7484619266030074
},
"bert_cnn_resnet152": {
"examples": 5822,
"spearman": 0.23897661511730053,
"kendall_tau_b": 0.17511563484276757,
"kendall_tau_c": 0.17116331913460656,
"pearson": 0.20429245894996126,
"normalized_human_mae": 0.47293288321293026,
"empty_image_contexts": 218
},
"bert_cnn_clip_imagenet": {
"examples": 5822,
"spearman": 0.23477193485123116,
"kendall_tau_b": 0.1718365123138481,
"kendall_tau_c": 0.1679471771227406,
"pearson": 0.19951677072314783,
"normalized_human_mae": 0.5044658286184057,
"empty_image_contexts": 386
},
"bert_cnn_combined": {
"examples": 5822,
"spearman": 0.3646498307040494,
"kendall_tau_b": 0.2680415373382081,
"kendall_tau_c": 0.2620033341773608,
"pearson": 0.3111999615665536,
"normalized_human_mae": 0.4910941707480373,
"empty_image_contexts": 0
}
},
"protocol": "Original mean human ratings; three fixed image-only context policies; no tuning on human ratings",
"limitations": [
"Visual extractors differ from training-data extraction; not original-paper replication",
"Normalized-human MAE is descriptive; binary-trained outputs are not calibrated human ratings",
"Image-context ablations share test data and are not independent validation sets"
]
}