| { |
| "title": "Addition conditioning: anchor frame vs captions", |
| "ckpt": "text_both_audiofix_scratch_0811 @ epoch028-global_step112500", |
| "split": "test_first50_fixed.jsonl (first 10 with all arms shown)", |
| "grid": { |
| "size": 256, |
| "fps": 16, |
| "frames": 81, |
| "sr": 16000, |
| "duration_s": 5.0625 |
| }, |
| "arms": { |
| "bare": { |
| "label": "bare noun", |
| "anchor": false, |
| "prompt": "object name only" |
| }, |
| "withref": { |
| "label": "+ first frame", |
| "anchor": true, |
| "prompt": "object name only" |
| }, |
| "short": { |
| "label": "short caption", |
| "anchor": false, |
| "prompt": "3-8 words" |
| }, |
| "long": { |
| "label": "long caption", |
| "anchor": false, |
| "prompt": "8-18 words" |
| }, |
| "refshort": { |
| "label": "first frame + short", |
| "anchor": true, |
| "prompt": "3-8 words" |
| }, |
| "reflong": { |
| "label": "first frame + long", |
| "anchor": true, |
| "prompt": "8-18 words" |
| } |
| }, |
| "aggregate": { |
| "withref": { |
| "n": 50, |
| "gap": 2.03, |
| "median": 2.25, |
| "moved": "18/50" |
| }, |
| "refshort": { |
| "n": 50, |
| "gap": 2.13, |
| "median": 1.9, |
| "moved": "18/50" |
| }, |
| "reflong": { |
| "n": 50, |
| "gap": 4.02, |
| "median": 4.05, |
| "moved": "13/50" |
| }, |
| "bare": { |
| "n": 50, |
| "gap": 2.64, |
| "median": 2.85, |
| "moved": "17/50" |
| }, |
| "short": { |
| "n": 50, |
| "gap": 3.37, |
| "median": 4.1, |
| "moved": "16/50" |
| }, |
| "long": { |
| "n": 50, |
| "gap": 5.49, |
| "median": 4.95, |
| "moved": "10/50" |
| } |
| }, |
| "entries": [ |
| { |
| "video_id": "NU2oWhlEWss", |
| "object": "vehicles", |
| "caption_short": "White and black articulated bus", |
| "caption_long": "white and black articulated bus with 'COLT45' advertisement, moving slowly on a road.", |
| "clips": { |
| "input": "videos/NU2oWhlEWss_input.mp4", |
| "gt": "videos/NU2oWhlEWss_gt.mp4", |
| "anchor": "videos/NU2oWhlEWss_anchor.jpg", |
| "withref": "videos/NU2oWhlEWss_withref.mp4", |
| "refshort": "videos/NU2oWhlEWss_refshort.mp4", |
| "reflong": "videos/NU2oWhlEWss_reflong.mp4", |
| "bare": "videos/NU2oWhlEWss_bare.mp4", |
| "short": "videos/NU2oWhlEWss_short.mp4", |
| "long": "videos/NU2oWhlEWss_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": -3.93, |
| "a_gap": -1.1 |
| }, |
| "refshort": { |
| "v_gap": -3.13, |
| "a_gap": -1.09 |
| }, |
| "reflong": { |
| "v_gap": -1.95, |
| "a_gap": -1.15 |
| }, |
| "bare": { |
| "v_gap": -5.54, |
| "a_gap": -1.09 |
| }, |
| "short": { |
| "v_gap": -4.24, |
| "a_gap": -1.09 |
| }, |
| "long": { |
| "v_gap": 17.2, |
| "a_gap": -1.15 |
| } |
| } |
| }, |
| { |
| "video_id": "X7t8wKNIAP4", |
| "object": "birds", |
| "caption_short": "Two gray pigeons with black wing bars", |
| "caption_long": "Two gray pigeons with black wing bars stand close together, one facing forward, the other turned slightly.", |
| "clips": { |
| "input": "videos/X7t8wKNIAP4_input.mp4", |
| "gt": "videos/X7t8wKNIAP4_gt.mp4", |
| "anchor": "videos/X7t8wKNIAP4_anchor.jpg", |
| "withref": "videos/X7t8wKNIAP4_withref.mp4", |
| "refshort": "videos/X7t8wKNIAP4_refshort.mp4", |
| "reflong": "videos/X7t8wKNIAP4_reflong.mp4", |
| "bare": "videos/X7t8wKNIAP4_bare.mp4", |
| "short": "videos/X7t8wKNIAP4_short.mp4", |
| "long": "videos/X7t8wKNIAP4_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": 6.44, |
| "a_gap": -1.28 |
| }, |
| "refshort": { |
| "v_gap": 7.42, |
| "a_gap": -1.23 |
| }, |
| "reflong": { |
| "v_gap": 10.58, |
| "a_gap": -1.7 |
| }, |
| "bare": { |
| "v_gap": 5.63, |
| "a_gap": -1.31 |
| }, |
| "short": { |
| "v_gap": 6.29, |
| "a_gap": -1.26 |
| }, |
| "long": { |
| "v_gap": 10.87, |
| "a_gap": -1.71 |
| } |
| } |
| }, |
| { |
| "video_id": "Rumslm1EQhU", |
| "object": "water", |
| "caption_short": "clear water in a toilet bowl", |
| "caption_long": "Clear water sits still in the white ceramic toilet bowl, reflecting light from above.", |
| "clips": { |
| "input": "videos/Rumslm1EQhU_input.mp4", |
| "gt": "videos/Rumslm1EQhU_gt.mp4", |
| "anchor": "videos/Rumslm1EQhU_anchor.jpg", |
| "withref": "videos/Rumslm1EQhU_withref.mp4", |
| "refshort": "videos/Rumslm1EQhU_refshort.mp4", |
| "reflong": "videos/Rumslm1EQhU_reflong.mp4", |
| "bare": "videos/Rumslm1EQhU_bare.mp4", |
| "short": "videos/Rumslm1EQhU_short.mp4", |
| "long": "videos/Rumslm1EQhU_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": -24.05, |
| "a_gap": 0.61 |
| }, |
| "refshort": { |
| "v_gap": -24.08, |
| "a_gap": 0.6 |
| }, |
| "reflong": { |
| "v_gap": -20.52, |
| "a_gap": 0.55 |
| }, |
| "bare": { |
| "v_gap": -22.8, |
| "a_gap": 0.62 |
| }, |
| "short": { |
| "v_gap": -22.33, |
| "a_gap": 0.6 |
| }, |
| "long": { |
| "v_gap": -16.61, |
| "a_gap": 0.56 |
| } |
| } |
| }, |
| { |
| "video_id": "65vCgp9aKGU_000116", |
| "object": "birds", |
| "caption_short": "white cockatiels with yellow crests", |
| "caption_long": "White cockatiels with yellow crests perched on wooden bars, some moving slightly.", |
| "clips": { |
| "input": "videos/65vCgp9aKGU_000116_input.mp4", |
| "gt": "videos/65vCgp9aKGU_000116_gt.mp4", |
| "anchor": "videos/65vCgp9aKGU_000116_anchor.jpg", |
| "withref": "videos/65vCgp9aKGU_000116_withref.mp4", |
| "refshort": "videos/65vCgp9aKGU_000116_refshort.mp4", |
| "reflong": "videos/65vCgp9aKGU_000116_reflong.mp4", |
| "bare": "videos/65vCgp9aKGU_000116_bare.mp4", |
| "short": "videos/65vCgp9aKGU_000116_short.mp4", |
| "long": "videos/65vCgp9aKGU_000116_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": 11.89, |
| "a_gap": 0.63 |
| }, |
| "refshort": { |
| "v_gap": 12.05, |
| "a_gap": 0.85 |
| }, |
| "reflong": { |
| "v_gap": 13.88, |
| "a_gap": 1.04 |
| }, |
| "bare": { |
| "v_gap": 10.27, |
| "a_gap": 0.64 |
| }, |
| "short": { |
| "v_gap": 10.42, |
| "a_gap": 0.86 |
| }, |
| "long": { |
| "v_gap": 11.89, |
| "a_gap": 1.03 |
| } |
| } |
| }, |
| { |
| "video_id": "hVnvazN3U0s_000060", |
| "object": "motorboat engine", |
| "caption_short": "No visible motorboat engine", |
| "caption_long": "No visible motorboat engine in the provided frames.", |
| "clips": { |
| "input": "videos/hVnvazN3U0s_000060_input.mp4", |
| "gt": "videos/hVnvazN3U0s_000060_gt.mp4", |
| "anchor": "videos/hVnvazN3U0s_000060_anchor.jpg", |
| "withref": "videos/hVnvazN3U0s_000060_withref.mp4", |
| "refshort": "videos/hVnvazN3U0s_000060_refshort.mp4", |
| "reflong": "videos/hVnvazN3U0s_000060_reflong.mp4", |
| "bare": "videos/hVnvazN3U0s_000060_bare.mp4", |
| "short": "videos/hVnvazN3U0s_000060_short.mp4", |
| "long": "videos/hVnvazN3U0s_000060_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": -12.22, |
| "a_gap": -0.24 |
| }, |
| "refshort": { |
| "v_gap": -10.76, |
| "a_gap": -0.02 |
| }, |
| "reflong": { |
| "v_gap": 12.78, |
| "a_gap": -0.0 |
| }, |
| "bare": { |
| "v_gap": -11.12, |
| "a_gap": -0.15 |
| }, |
| "short": { |
| "v_gap": -9.92, |
| "a_gap": 0.09 |
| }, |
| "long": { |
| "v_gap": -8.41, |
| "a_gap": 0.1 |
| } |
| } |
| }, |
| { |
| "video_id": "u_8JdjQOo-o_000151", |
| "object": "bagpipe", |
| "caption_short": "black bagpipe with tassels", |
| "caption_long": "Black bagpipe with tassels, held by player, bag inflated, drones and chanter visible.", |
| "clips": { |
| "input": "videos/u_8JdjQOo-o_000151_input.mp4", |
| "gt": "videos/u_8JdjQOo-o_000151_gt.mp4", |
| "anchor": "videos/u_8JdjQOo-o_000151_anchor.jpg", |
| "withref": "videos/u_8JdjQOo-o_000151_withref.mp4", |
| "refshort": "videos/u_8JdjQOo-o_000151_refshort.mp4", |
| "reflong": "videos/u_8JdjQOo-o_000151_reflong.mp4", |
| "bare": "videos/u_8JdjQOo-o_000151_bare.mp4", |
| "short": "videos/u_8JdjQOo-o_000151_short.mp4", |
| "long": "videos/u_8JdjQOo-o_000151_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": 6.7, |
| "a_gap": -0.28 |
| }, |
| "refshort": { |
| "v_gap": 6.82, |
| "a_gap": -1.08 |
| }, |
| "reflong": { |
| "v_gap": 7.41, |
| "a_gap": -0.27 |
| }, |
| "bare": { |
| "v_gap": 5.77, |
| "a_gap": -0.3 |
| }, |
| "short": { |
| "v_gap": 5.86, |
| "a_gap": -1.04 |
| }, |
| "long": { |
| "v_gap": 6.64, |
| "a_gap": -0.12 |
| } |
| } |
| }, |
| { |
| "video_id": "TbTUbDC-IuQ_000249", |
| "object": "bass line", |
| "caption_short": "shiny silver steel drum", |
| "caption_long": "large, shiny silver steel drum being played with purple-tipped mallets, producing resonant bass tones.", |
| "clips": { |
| "input": "videos/TbTUbDC-IuQ_000249_input.mp4", |
| "gt": "videos/TbTUbDC-IuQ_000249_gt.mp4", |
| "anchor": "videos/TbTUbDC-IuQ_000249_anchor.jpg", |
| "withref": "videos/TbTUbDC-IuQ_000249_withref.mp4", |
| "refshort": "videos/TbTUbDC-IuQ_000249_refshort.mp4", |
| "reflong": "videos/TbTUbDC-IuQ_000249_reflong.mp4", |
| "bare": "videos/TbTUbDC-IuQ_000249_bare.mp4", |
| "short": "videos/TbTUbDC-IuQ_000249_short.mp4", |
| "long": "videos/TbTUbDC-IuQ_000249_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": 10.89, |
| "a_gap": 0.16 |
| }, |
| "refshort": { |
| "v_gap": 13.58, |
| "a_gap": 0.58 |
| }, |
| "reflong": { |
| "v_gap": 16.45, |
| "a_gap": 0.49 |
| }, |
| "bare": { |
| "v_gap": 3.88, |
| "a_gap": 0.15 |
| }, |
| "short": { |
| "v_gap": 7.32, |
| "a_gap": 0.6 |
| }, |
| "long": { |
| "v_gap": 11.12, |
| "a_gap": 0.48 |
| } |
| } |
| }, |
| { |
| "video_id": "GwA7hMcmJ6Q", |
| "object": "piano", |
| "caption_short": "black grand piano with glossy finish", |
| "caption_long": "black grand piano with a glossy finish, reflecting light on its smooth surface.", |
| "clips": { |
| "input": "videos/GwA7hMcmJ6Q_input.mp4", |
| "gt": "videos/GwA7hMcmJ6Q_gt.mp4", |
| "anchor": "videos/GwA7hMcmJ6Q_anchor.jpg", |
| "withref": "videos/GwA7hMcmJ6Q_withref.mp4", |
| "refshort": "videos/GwA7hMcmJ6Q_refshort.mp4", |
| "reflong": "videos/GwA7hMcmJ6Q_reflong.mp4", |
| "bare": "videos/GwA7hMcmJ6Q_bare.mp4", |
| "short": "videos/GwA7hMcmJ6Q_short.mp4", |
| "long": "videos/GwA7hMcmJ6Q_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": 0.26, |
| "a_gap": -0.29 |
| }, |
| "refshort": { |
| "v_gap": 0.16, |
| "a_gap": -0.58 |
| }, |
| "reflong": { |
| "v_gap": 0.07, |
| "a_gap": -0.18 |
| }, |
| "bare": { |
| "v_gap": 2.93, |
| "a_gap": -0.26 |
| }, |
| "short": { |
| "v_gap": 2.95, |
| "a_gap": -0.55 |
| }, |
| "long": { |
| "v_gap": 3.39, |
| "a_gap": -0.22 |
| } |
| } |
| }, |
| { |
| "video_id": "884QLkT8sBQ_000098", |
| "object": "musician", |
| "caption_short": "Young boy playing tabla drums", |
| "caption_long": "young boy in white shirt plays tabla drums with focused expression and precise hand movements.", |
| "clips": { |
| "input": "videos/884QLkT8sBQ_000098_input.mp4", |
| "gt": "videos/884QLkT8sBQ_000098_gt.mp4", |
| "anchor": "videos/884QLkT8sBQ_000098_anchor.jpg", |
| "withref": "videos/884QLkT8sBQ_000098_withref.mp4", |
| "refshort": "videos/884QLkT8sBQ_000098_refshort.mp4", |
| "reflong": "videos/884QLkT8sBQ_000098_reflong.mp4", |
| "bare": "videos/884QLkT8sBQ_000098_bare.mp4", |
| "short": "videos/884QLkT8sBQ_000098_short.mp4", |
| "long": "videos/884QLkT8sBQ_000098_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": 2.15, |
| "a_gap": -0.35 |
| }, |
| "refshort": { |
| "v_gap": 1.12, |
| "a_gap": -0.67 |
| }, |
| "reflong": { |
| "v_gap": 3.87, |
| "a_gap": -0.66 |
| }, |
| "bare": { |
| "v_gap": 2.32, |
| "a_gap": -0.49 |
| }, |
| "short": { |
| "v_gap": 6.46, |
| "a_gap": -0.78 |
| }, |
| "long": { |
| "v_gap": 6.73, |
| "a_gap": -0.83 |
| } |
| } |
| }, |
| { |
| "video_id": "NhWaVKzBX_g_000056", |
| "object": "powerful engine", |
| "caption_short": "Tall slender rocket engine", |
| "caption_long": "tall, slender rocket engine with a dark body and bright exhaust trail propels upward.", |
| "clips": { |
| "input": "videos/NhWaVKzBX_g_000056_input.mp4", |
| "gt": "videos/NhWaVKzBX_g_000056_gt.mp4", |
| "anchor": "videos/NhWaVKzBX_g_000056_anchor.jpg", |
| "withref": "videos/NhWaVKzBX_g_000056_withref.mp4", |
| "refshort": "videos/NhWaVKzBX_g_000056_refshort.mp4", |
| "reflong": "videos/NhWaVKzBX_g_000056_reflong.mp4", |
| "bare": "videos/NhWaVKzBX_g_000056_bare.mp4", |
| "short": "videos/NhWaVKzBX_g_000056_short.mp4", |
| "long": "videos/NhWaVKzBX_g_000056_long.mp4" |
| }, |
| "metrics": { |
| "withref": { |
| "v_gap": 9.48, |
| "a_gap": -0.08 |
| }, |
| "refshort": { |
| "v_gap": 9.67, |
| "a_gap": -0.14 |
| }, |
| "reflong": { |
| "v_gap": 9.77, |
| "a_gap": -0.1 |
| }, |
| "bare": { |
| "v_gap": 9.03, |
| "a_gap": 0.11 |
| }, |
| "short": { |
| "v_gap": 8.92, |
| "a_gap": -0.03 |
| }, |
| "long": { |
| "v_gap": 8.87, |
| "a_gap": -0.1 |
| } |
| } |
| } |
| ] |
| } |
