WVY-Cartoon-14m / benchmark.json
StarpowerTechnology's picture
WVY-Cartoon-14m: weights, model card, video, benchmark
b419c76 verified
Raw History Blame Contribute Delete
2.95 kB
{
"pretrained": {
"mse": 0.0003099249831090371,
"psnr": 35.758202923291414,
"trait_accuracy": {
"background": 1.0,
"skin": 1.0,
"pose": 1.0,
"outfit": 1.0,
"hair": 1.0,
"eyes": 1.0,
"brows": 1.0,
"nose": 1.0,
"mouth": 1.0,
"prop": 0.9952380952380953
},
"trait_accuracy_overall": 0.9994607710973308,
"all_traits_correct": 0.9952380952380953,
"by_split": {
"unseen_combo": {
"n": 300,
"all_traits_correct": 1.0,
"mse": 0.00025095561771498374
},
"unseen_pair:hoodie pink+red-orange": {
"n": 40,
"all_traits_correct": 1.0,
"mse": 0.00040739864598435813
},
"unseen_pair:mohawk+hearts": {
"n": 40,
"all_traits_correct": 1.0,
"mse": 0.0002757114172709407
},
"unseen_pair:waving+purple messy": {
"n": 40,
"all_traits_correct": 0.95,
"mse": 0.0006889351265272125
}
}
},
"fine-tuned": {
"mse": 0.000322742107528029,
"psnr": 35.53760317924316,
"trait_accuracy": {
"background": 1.0,
"skin": 1.0,
"pose": 1.0,
"outfit": 1.0,
"hair": 1.0,
"eyes": 1.0,
"brows": 0.9975845410628019,
"nose": 1.0,
"mouth": 1.0,
"prop": 0.9952380952380953
},
"trait_accuracy_overall": 0.9991911566459962,
"all_traits_correct": 0.9928571428571429,
"by_split": {
"unseen_combo": {
"n": 300,
"all_traits_correct": 0.9966666666666667,
"mse": 0.0002636634569595723
},
"unseen_pair:hoodie pink+red-orange": {
"n": 40,
"all_traits_correct": 1.0,
"mse": 0.00039764878019923346
},
"unseen_pair:mohawk+hearts": {
"n": 40,
"all_traits_correct": 1.0,
"mse": 0.00033613829873502254
},
"unseen_pair:waving+purple messy": {
"n": 40,
"all_traits_correct": 0.95,
"mse": 0.0006775291229132562
}
}
},
"lookup baseline": {
"mse": 0.04635754014653642,
"psnr": 15.501059515327885,
"trait_accuracy": {
"background": 0.6811594202898551,
"skin": 0.714975845410628,
"pose": 0.6687116564417178,
"outfit": 0.3684210526315789,
"hair": 0.5314009661835749,
"eyes": 0.4492753623188406,
"brows": 0.533816425120773,
"nose": 0.5555555555555556,
"mouth": 0.48792270531400966,
"prop": 0.6428571428571429
},
"trait_accuracy_overall": 0.5661903478026422,
"all_traits_correct": 0.0,
"by_split": {
"unseen_combo": {
"n": 300,
"all_traits_correct": 0.0,
"mse": 0.04259609623967359
},
"unseen_pair:hoodie pink+red-orange": {
"n": 40,
"all_traits_correct": 0.0,
"mse": 0.046313752955757084
},
"unseen_pair:mohawk+hearts": {
"n": 40,
"all_traits_correct": 0.0,
"mse": 0.05363947679288685
},
"unseen_pair:waving+purple messy": {
"n": 40,
"all_traits_correct": 0.0,
"mse": 0.06733021999243646
}
}
},
"_info": {
"rule_breaking_test_prompts": 6,
"test_prompts": 420,
"train_images": 10000,
"params": 14662400,
"ms_per_image_cpu": 0.4,
"resolution": "96x96"
}
}