FORGE3DBench: final eval protocol + eval_final.py, batched Ours inference, held-out view tars, missing-object lists, README
cca6827 verified Invalid JSON:Unexpected token 'I', ..." "psnr": Infinity,
"... is not valid JSON
| { | |
| "equiv_heldout_view": [ | |
| "EQUIV (cams < 10 units: 19 views, bit-identical renders 11, max metric diff 2.05e-08, max pix diff 5.96e-08): PASS", | |
| "far cams (>=10 units, far-clip fix may change render): 9 views, changed 4" | |
| ], | |
| "gtself_fb150_image": { | |
| "means": { | |
| "n_objects": 5, | |
| "iv": { | |
| "lpips": 0.155406928434968, | |
| "ssim": 0.8513417273759842, | |
| "psnr": 17.656574010849, | |
| "clip": 0.9003176599740982, | |
| "coverage": 0.9979206158323255, | |
| "n_views_total": 20, | |
| "n_objects": 5 | |
| }, | |
| "nv": { | |
| "lpips": 0.15581146900983234, | |
| "ssim": 0.8652680912232938, | |
| "psnr": 17.692288932800295, | |
| "clip": 0.9001341432019284, | |
| "coverage": 0.9976489886148329, | |
| "n_views_total": 87, | |
| "n_objects": 5 | |
| }, | |
| "geometry": { | |
| "cd_l1": 0.0, | |
| "f01": 1.0, | |
| "f02": 1.0, | |
| "f05": 1.0, | |
| "normal_consistency": 1.0, | |
| "vol_iou": 1.0, | |
| "cd_l1_rel": 0.0, | |
| "f01_rel": 1.0 | |
| } | |
| }, | |
| "excluded": [], | |
| "failed": {} | |
| }, | |
| "gtself_fb150_gtrender": { | |
| "means": { | |
| "n_objects": 5, | |
| "iv": { | |
| "lpips": 1.342538415443493e-18, | |
| "ssim": 1.0, | |
| "psnr": Infinity, | |
| "clip": 0.9999999850988388, | |
| "coverage": 0.9979206158323255, | |
| "n_views_total": 20, | |
| "n_objects": 5 | |
| }, | |
| "nv": { | |
| "lpips": 1.3391858052656775e-11, | |
| "ssim": 1.0, | |
| "psnr": Infinity, | |
| "clip": 0.9999999983059732, | |
| "coverage": 0.9976489886148329, | |
| "n_views_total": 87, | |
| "n_objects": 5 | |
| }, | |
| "geometry": { | |
| "cd_l1": 0.0, | |
| "f01": 1.0, | |
| "f02": 1.0, | |
| "f05": 1.0, | |
| "normal_consistency": 1.0, | |
| "vol_iou": 1.0, | |
| "cd_l1_rel": 0.0, | |
| "f01_rel": 1.0 | |
| } | |
| }, | |
| "excluded": [], | |
| "failed": {} | |
| }, | |
| "gtself_omni300_image": { | |
| "means": { | |
| "n_objects": 4, | |
| "iv": { | |
| "lpips": 0.06666350021259859, | |
| "ssim": 0.927033793181181, | |
| "psnr": 22.581916213035583, | |
| "clip": 0.959827896207571, | |
| "coverage": 0.9807583123661334, | |
| "n_views_total": 16, | |
| "n_objects": 4 | |
| }, | |
| "nv": { | |
| "lpips": 0.07639438135083765, | |
| "ssim": 0.9303558357059956, | |
| "psnr": 22.14875966310501, | |
| "clip": 0.9491641670465469, | |
| "coverage": 0.9805030219477395, | |
| "n_views_total": 16, | |
| "n_objects": 4 | |
| }, | |
| "geometry": { | |
| "cd_l1": 1.3470572326923502e-05, | |
| "f01": 1.0, | |
| "f02": 1.0, | |
| "f05": 1.0, | |
| "normal_consistency": 0.9999342858791351, | |
| "vol_iou": 1.0, | |
| "cd_l1_rel": 1.3470572341895741e-05, | |
| "f01_rel": 1.0 | |
| } | |
| }, | |
| "excluded": [ | |
| "insole_001" | |
| ], | |
| "failed": {} | |
| }, | |
| "gtself_omni300_gtrender": { | |
| "means": { | |
| "n_objects": 4, | |
| "iv": { | |
| "lpips": 6.209717131904555e-12, | |
| "ssim": 1.0, | |
| "psnr": 186.3747682571411, | |
| "clip": 0.9999999776482582, | |
| "coverage": 0.9807583123661334, | |
| "n_views_total": 16, | |
| "n_objects": 4 | |
| }, | |
| "nv": { | |
| "lpips": 1.548370281258005e-11, | |
| "ssim": 1.0, | |
| "psnr": 186.5544662475586, | |
| "clip": 0.9999999850988388, | |
| "coverage": 0.9805030219477395, | |
| "n_views_total": 16, | |
| "n_objects": 4 | |
| }, | |
| "geometry": { | |
| "cd_l1": 1.0609970786390477e-05, | |
| "f01": 1.0, | |
| "f02": 1.0, | |
| "f05": 1.0, | |
| "normal_consistency": 0.9999463707208633, | |
| "vol_iou": 1.0, | |
| "cd_l1_rel": 1.0609970796787391e-05, | |
| "f01_rel": 1.0 | |
| } | |
| }, | |
| "excluded": [ | |
| "insole_001" | |
| ], | |
| "failed": {} | |
| }, | |
| "gtself_toys300": { | |
| "means": { | |
| "n_objects": 5, | |
| "iv": { | |
| "lpips": 0.04836386516690254, | |
| "ssim": 0.8744846999645233, | |
| "psnr": 26.259253120422365, | |
| "clip": 0.9426980346441269, | |
| "coverage": NaN, | |
| "n_views_total": 20, | |
| "n_objects": 5 | |
| }, | |
| "nv": { | |
| "lpips": 2.961059739371032e-19, | |
| "ssim": 1.0, | |
| "psnr": Infinity, | |
| "clip": 1.000000002980232, | |
| "coverage": NaN, | |
| "n_views_total": 120, | |
| "n_objects": 5 | |
| }, | |
| "geometry": { | |
| "cd_l1": 0.0, | |
| "f01": 1.0, | |
| "f02": 1.0, | |
| "f05": 1.0, | |
| "normal_consistency": 1.0, | |
| "vol_iou": 1.0, | |
| "cd_l1_rel": 0.0, | |
| "f01_rel": 1.0 | |
| } | |
| }, | |
| "excluded": [], | |
| "failed": {} | |
| }, | |
| "toys_equiv": { | |
| "objects": [ | |
| "airplane_000", | |
| "airplane_004" | |
| ], | |
| "max_abs_diff_appearance": 3.1044086051812414e-10, | |
| "max_abs_diff_geometry": 1.5437602996826172e-05 | |
| }, | |
| "ours_fb150_4v_10": { | |
| "means": { | |
| "n_objects": 10, | |
| "iv": { | |
| "lpips": 0.07327520195394754, | |
| "ssim": 0.9049470454454422, | |
| "psnr": 23.259419202804565, | |
| "clip": 0.9382741689682007, | |
| "coverage": 0.9812214121407739, | |
| "n_views_total": 40, | |
| "n_objects": 10 | |
| }, | |
| "nv": { | |
| "lpips": 0.08182250504951485, | |
| "ssim": 0.9016392203552979, | |
| "psnr": 22.729140621685644, | |
| "clip": 0.9302215895457575, | |
| "coverage": 0.9743186795478029, | |
| "n_views_total": 185, | |
| "n_objects": 10 | |
| }, | |
| "geometry": { | |
| "cd_l1": 0.010491380095481872, | |
| "f01": 0.7524213850498199, | |
| "f02": 0.8737186014652252, | |
| "f05": 0.9575369596481323, | |
| "normal_consistency": 0.8958532571792602, | |
| "vol_iou": 0.662994354883604, | |
| "cd_l1_rel": 0.011657089303787265, | |
| "f01_rel": 0.723338270187378 | |
| } | |
| }, | |
| "objects": [ | |
| "FaGr_s000001_butternutsquash1", | |
| "FaGr_s000002_yellowmustardbottle5", | |
| "FaGr_s000003_blueteacup4", | |
| "FaGr_s000004_toypictureviewer4", | |
| "FaGr_s000005_peppergrinder4", | |
| "FaGr_s000005_phillipsheadscrewdriver0", | |
| "FaGr_s000006_bluesanitizerbottle1", | |
| "FaGr_s000006_yellowdogcan5", | |
| "FaGr_s000007_coffeemaker0", | |
| "FaGr_s000008_britishtelephonebooth0" | |
| ] | |
| }, | |
| "ceil_fb150_4v": { | |
| "means": { | |
| "n_objects": 150, | |
| "iv": { | |
| "lpips": 0.14496873109911879, | |
| "ssim": 0.8604318048556646, | |
| "psnr": 18.509435766537983, | |
| "clip": 0.8836117126544316, | |
| "coverage": 0.9980876749307664, | |
| "n_views_total": 600, | |
| "n_objects": 150 | |
| }, | |
| "nv": { | |
| "lpips": 0.14438312600245357, | |
| "ssim": 0.8614764633895826, | |
| "psnr": 18.428407900478234, | |
| "clip": 0.8806010282023343, | |
| "coverage": 0.9979836521053852, | |
| "n_views_total": 2798, | |
| "n_objects": 150 | |
| } | |
| }, | |
| "excluded": [], | |
| "failed": {}, | |
| "n_scored": 150, | |
| "iv_front_only_1v": { | |
| "lpips": 0.1476941823710998, | |
| "ssim": 0.8565389076868694, | |
| "psnr": 18.354039103190104 | |
| } | |
| }, | |
| "ceil_omni300_4v_n60": { | |
| "means": { | |
| "n_objects": 57, | |
| "iv": { | |
| "lpips": 0.07808759794243726, | |
| "ssim": 0.9191342414471141, | |
| "psnr": 23.012761559402733, | |
| "clip": 0.9550255814142394, | |
| "coverage": 0.9789419838654099, | |
| "n_views_total": 228, | |
| "n_objects": 57 | |
| }, | |
| "nv": { | |
| "lpips": 0.08205401141343541, | |
| "ssim": 0.9165169060753103, | |
| "psnr": 22.73387204973321, | |
| "clip": 0.951842261249559, | |
| "coverage": 0.9780890347120065, | |
| "n_views_total": 228, | |
| "n_objects": 57 | |
| } | |
| }, | |
| "excluded": [ | |
| "beauty_blender_004", | |
| "insole_003", | |
| "ornaments_010" | |
| ], | |
| "failed": {}, | |
| "n_scored": 60 | |
| } | |
| } |