avio-verify / items.json
WitneyWW's picture
verify page v1 (items 711b321fc2bd5fc4, page 2026-11-14_live_v2-711b321f)
22ae8f9 verified
Raw History Blame Contribute Delete
168 kB
{ "tag": "2026-11-14_live_v2", "items_sha256": "711b321fc2bd5fc4", "page_version": "2026-11-14_live_v2-711b321f", "cards": [ { "id": "A.TEST.1.o_e569f315dd1e", "kind": "A", "clip": "TEST.1", "version": "TEST", "task": "removal", "video_id": "9rXAmXuAOw8_000000", "instruction": "Remove the crow.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.25, V=0.75", "src": "media/i_6537882e_src.mp4", "ref": null, "output": "media/o_e569f315dd1e.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "The crow's distorted vocalizations remain audible from 0.0s to 3.9s despite the crow being removed, and large brown vertical smears appear beside the post (0.0s-4.3s) and above it (4.3s-5.0s).", "evidence": { "source_target_visible": "A black crow is perched on a brown wooden post throughout the entire clip (0.0s to 5.0s).", "source_target_audible": "The crow makes distorted, robotic-sounding vocalizations/talking sounds synchronized with its beak opening at 0.0s-0.7s, 0.9s-2.3s, 2.4s-3.9s, and 4.7s-5.0s.", "source_other_sounds": "Faint background synth/electronic music throughout (0.0s to 5.0s).", "edited_visual": "The crow is removed, but brown blurry vertical smears appear next to the post (0.0s to 4.3s) and directly above the post (4.3s to 5.0s).", "edited_audio": "The crow's vocalizations are still audible and heavily distorted/muffled at 0.0s-0.7s, 0.9s-2.3s, and 2.4s-3.9s. Background synth remains." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_486d854b14c0.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "The crow's legs/claws remain visible on top of the wooden post from 0.0s to 5.0s, and the crow's vocalizations were not removed from the audio track (0.0s-5.0s).", "evidence": { "source_target_visible": "An animated crow is perched on a wooden post in the center from 0.0s to 5.0s.", "source_target_audible": "The crow makes animated talking/vocalizing sounds synced with its beak movement throughout 0.0s to 5.0s.", "source_other_sounds": "none", "edited_visual": "The crow's body is removed, but its claws/legs remain on top of the post throughout 0.0s to 5.0s.", "edited_audio": "The audio is completely unchanged; the crow's vocalizations remain throughout 0.0s to 5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_4458e43c484d.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "A deformed remnant of the crow and its legs remain visible on the post from 0.0s to 5.0s, and the wooden post has severe ghosting artifacts from 4.3s to 5.0s. The crow's audio remains audible throughout the entire clip (0.0s to 5.0s).", "evidence": { "source_target_visible": "The crow is visible sitting on a wooden post throughout the entire clip (0.0s to 5.0s).", "source_target_audible": "The crow makes distorted, cartoonish vocalizations/caws throughout the clip (0.0s to 5.0s).", "source_other_sounds": "none", "edited_visual": "The crow's body is mostly gone, but its legs and a deformed gray remnant of its body remain on the post throughout (0.0s to 5.0s). The post shifts and leaves a ghosting artifact at 4.3s-5.0s.", "edited_audio": "The crow's vocalizations remain audible throughout (0.0s to 5.0s), though slightly altered/distorted compared to the source." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_9rXAmXuAOw8_000000.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_6d1491f85064.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "The crow's vocalizations remain audible throughout (00:00-00:05), and the wooden post was erroneously removed from the visuals (00:00-00:05).", "evidence": { "source_target_visible": "The crow is visible perched on a wooden post in the center of the frame from 00:00.000 to 00:05.000.", "source_target_audible": "The crow emits distorted vocalizations/caws throughout the clip (00:00.000 to 00:05.000).", "source_other_sounds": "Upbeat background music plays throughout the clip.", "edited_visual": "The crow is removed, and the brown wooden post it was perched on is also completely removed from 00:00.000 to 00:05.000.", "edited_audio": "The audio is unchanged from SOURCE; both the crow's vocalizations and the background music remain from 00:00.000 to 00:05.000." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } } ] }, { "id": "A.TEST.2.o_bd0b653cf4c6", "kind": "A", "clip": "TEST.2", "version": "TEST", "task": "removal", "video_id": "BKVrl0ZP0lI_000030", "instruction": "Remove the train.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.00, V=0.50", "src": "media/i_0bacd674_src.mp4", "ref": null, "output": "media/o_bd0b653cf4c6.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "The train's audio remains as a heavily distorted, muffled residual sound (00:00.0-00:05.0), and the inpainted background replaces the train with a flickering, implausible gray barrier structure.", "evidence": { "source_target_visible": "Freight train cars moving across the middle ground from 00:00.0 to 00:05.0.", "source_target_audible": "Loud clanking and rumbling of the moving train from 00:00.0 to 00:05.0.", "source_other_sounds": "none", "edited_visual": "Train cars are replaced by a gray fence-like structure, with flickering/smearing along the track line from 00:00.0 to 00:05.0.", "edited_audio": "Muffled, distorted, watery residual clanking of the train persists throughout 00:00.0 to 00:05.0." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_8f2eaf2cf37d.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The train's loud clattering audio was not removed at all (0.0s-5.0s), and the inpainted background where the train was is a heavily blurred, smeared patch (0.0s-5.0s).", "evidence": { "source_target_visible": "Freight train cars moving left to right across the middle ground from 0.0s to 5.0s.", "source_target_audible": "Loud, rhythmic clattering and rumbling of the train on the tracks from 0.0s to 5.0s.", "source_other_sounds": "none", "edited_visual": "The train is removed from 0.0s to 5.0s, replaced by a blurry, smeared green and grey patch in the middle ground.", "edited_audio": "The loud train clattering and rumbling remains completely unchanged from 0.0s to 5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_094173fb4ac4.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Major", "failure_notes": "Acoustic removal failed as muffled train rumble persists throughout 0.0s-5.0s, and audio preservation failed due to major audio artifacts and hallucinated vocal/cheering sounds introduced around 1.8s-2.5s and 3.8s-4.4s with no visible source.", "evidence": { "source_target_visible": "Freight train cars moving left to right across the middle ground from 0.0s to 5.0s.", "source_target_audible": "Loud, rhythmic clattering and rumbling of train wheels on tracks from 0.0s to 5.0s.", "source_other_sounds": "none", "edited_visual": "The train is completely removed from 0.0s to 5.0s, replaced by a static fence and trees. Background grass and trees remain unchanged.", "edited_audio": "The train track noise is muffled and altered, and replaced with unnaturally spliced/distorted sounds resembling crowd cheering or muffled shouting around 1.8s-2.5s and 3.8s-4.4s, with residual muffled train rumbling throughout 0.0s-5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_BKVrl0ZP0lI_000030.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_aa629818bf77.mp4", "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Major", "failure_notes": "Inpainted train area has a blurry dark smudge on the right (0.0s-5.0s), and the audio is replaced with unrelated, heavily distorted crowd/voice noise throughout (0.0s-5.0s).", "evidence": { "source_target_visible": "Freight train cars moving horizontally across the middle ground from 0.0s to 5.0s.", "source_target_audible": "Loud rhythmic train clattering and rumbling from 0.0s to 5.0s.", "source_other_sounds": "none", "edited_visual": "The train is removed from 0.0s to 5.0s, replaced with blurry trees and a dark smudge on the right; the foreground grass is preserved.", "edited_audio": "The train clattering is replaced by loud, distorted noise/cheering and indistinct vocalizations from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" } } ] }, { "id": "A.TEST.3.o_b7a92cb60c81", "kind": "A", "clip": "TEST.3", "version": "TEST", "task": "removal", "video_id": "ykr3HGPai0g", "instruction": "Remove the birds.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.50, V=0.25", "src": "media/i_33f66d13_src.mp4", "ref": null, "output": "media/o_b7a92cb60c81.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The rooster remains visible from 0.0-1.6s and leaves large blurry orange smears from 1.6-5.0s, destroying background wire fencing. Bird clucking sounds remain audible from 0.5-4.0s.", "evidence": { "source_target_visible": "Rooster on left, white hen on right, and background chickens visible 0.0-5.0s.", "source_target_audible": "Birds chirping and clucking throughout 0.0-5.0s.", "source_other_sounds": "General outdoor/coop ambience 0.0-5.0s.", "edited_visual": "White hen removed 0.0-5.0s. Rooster on left remains fully visible 0.0-1.6s, then dissolves into severe orange/brown blurry smears 1.6-5.0s. Background chickens partially remain.", "edited_audio": "Some high-pitched chirping attenuated, but bird clucking and vocalizations remain audible around 0.5-4.0s with muffled/filtered quality." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_e6261ddda9f3.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The brown rooster remains partially visible on the left throughout 0.0s-5.0s, and the inpainted area on the right is a blurry smear that obscures the background cage (0.0s-5.0s). Additionally, the bird sounds were not removed from the audio track (0.0s-5.0s).", "evidence": { "source_target_visible": "Two chickens (a brown rooster and a white hen) are visible in the foreground from 0.0s to 5.0s, with other birds visible in the background cages.", "source_target_audible": "Continuous bird chirping, clucking, and vocalizations throughout 0.0s to 5.0s.", "source_other_sounds": "Brief metallic cage or movement noise at 0.0s-0.5s, general outdoor ambience.", "edited_visual": "The white hen and the rooster's dark tail feathers are replaced by a blurry, smeared brown patch (0.0s-5.0s), while the brown body and head of the rooster on the left remain visible throughout 0.0s-5.0s.", "edited_audio": "The audio track is unchanged; all bird chirping and clucking remains audible from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_d562eb55bf22.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The birds are not fully removed visually (parts of the rooster and hen remain at 0.0s-1.6s and 3.8s-5.0s, and the whole scene turns into a blurry smudge at 1.6s-3.8s), and none of the bird sounds were removed from the audio track (0.0s-5.0s).", "evidence": { "source_target_visible": "Two large chickens (a brown rooster on the left and a white hen on the right) are in the foreground from 0.0s to 5.0s, with other birds visible in the background cages.", "source_target_audible": "Birds chirping, clucking, and flapping wings throughout 0.0s to 5.0s.", "source_other_sounds": "General outdoor/coop ambient noise from 0.0s to 5.0s.", "edited_visual": "The foreground birds are partially and messily inpainted: parts of the brown rooster and white hen remain visible from 0.0s to 1.6s and 3.8s to 5.0s, while from 1.6s to 3.8s the frame becomes a severe blurry smudge. Background birds also remain visible from 0.0s to 1.5s and 4.3s to 5.0s.", "edited_audio": "The audio is completely unchanged from SOURCE; all bird chirping, clucking, and flapping sounds remain from 0.0s to 5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_ykr3HGPai0g.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_fad4fc5001c1.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "Visual removal fails (0.0s-5.0s) as the white chicken, background birds, and the rooster's black tail remain, leaving a major blurry smear where the rooster's body was. Acoustic removal fails (0.0s-5.0s) as all bird sounds remain untouched.", "evidence": { "source_target_visible": "Two large chickens/roosters in the foreground and others in the background cage from 0.0s to 5.0s.", "source_target_audible": "Birds chirping and chickens clucking throughout 0.0s to 5.0s.", "source_other_sounds": "General ambient background noise.", "edited_visual": "Only the orange body of the left rooster is replaced by a blurry smear from 0.0s to 5.0s; its black tail feathers and the entire white chicken remain visible.", "edited_audio": "The audio is unchanged from SOURCE; all bird chirping and clucking remain audible from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" } } ] }, { "id": "A.TEST.4.o_603afd0063f2", "kind": "A", "clip": "TEST.4", "version": "TEST", "task": "removal", "video_id": "137Af5KGX88_000026", "instruction": "Remove the birds.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.50, V=1.00", "src": "media/i_8c06e1d6_src.mp4", "ref": null, "output": "media/o_603afd0063f2.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "none", "evidence": { "source_target_visible": "A black and white magpie is perched on the roof of the wooden birdhouse from 00:00.00 to 00:03.875, then hops down onto the metal railing in the bottom right from 00:03.875 to 00:05.00.", "source_target_audible": "A bird call and wing/clatter sounds around 00:03.00 to 00:04.20.", "source_other_sounds": "Outdoor city traffic/wind ambience throughout 00:00.00 to 00:05.00.", "edited_visual": "The magpie is completely removed from the birdhouse roof and the railing throughout 00:00.00 to 00:05.00; the background and birdhouse remain static.", "edited_audio": "The bird call at 00:03.00-00:04.00 is removed, leaving the outdoor ambience and a brief rustling sound around 00:03.00." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_ac6da099a66d.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The bird vocalization at 3.0s is still audible in the edited clip, and the entire first frame at 0.0s suffers from severe whole-frame blurring/smearing.", "evidence": { "source_target_visible": "A magpie is perched on the birdhouse roof from 0.0s to 3.8s, then hops down to the railing until 5.0s.", "source_target_audible": "Bird calls/caws at 3.0s and 3.8s-4.2s, plus hopping/claw sounds around 3.8s.", "source_other_sounds": "Outdoor city/wind ambience throughout.", "edited_visual": "The bird is removed throughout. The first frame (0.0s) is heavily blurred/smeared across the whole frame, and the inpainted roof/railing area has minor blurriness.", "edited_audio": "The bird calls at 3.8s-4.2s are removed, but the bird call/chirp at 3.0s remains audible. Background ambience is preserved." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_e14a5874f402.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "Acoustic removal failed as the bird calls remain audible at 3.0s and 3.8-4.2s despite the bird being visually removed.", "evidence": { "source_target_visible": "A black and white magpie is perched on the wooden birdhouse roof (0.0-3.8s) and hops down to the metal railing (3.8-5.0s).", "source_target_audible": "Bird calls/caws at 3.0s and 3.8-4.2s, along with faint rustling/hopping sounds around 3.8s.", "source_other_sounds": "Continuous background urban ambience and wind.", "edited_visual": "The magpie is completely removed throughout the 0.0-5.0s clip, with the birdhouse roof and sky inpainted. A brief yellow tint appears in the sky at 0.0s.", "edited_audio": "The audio track is unchanged from the source; the bird calls at 3.0s and 3.8-4.2s remain clearly audible." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_137Af5KGX88_000026.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_a82d190a69fa.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "Visual removal fails due to a prominent blurry dark ghost/smudge remaining where the bird's tail and body were from 0.2s to 3.0s and briefly at 3.8s-4.0s. Acoustic removal fails completely as the bird calls remain clearly audible from 3.0s to 4.2s with no visible bird.", "evidence": { "source_target_visible": "A black and white bird (magpie) is visible throughout the clip (0.0s-5.0s), perched on top of the birdhouse and then hopping down to the railing around 3.8s.", "source_target_audible": "Bird calls/caws are heard around 3.0s-4.2s, along with brief fluttering/hopping sounds at 3.8s.", "source_other_sounds": "Continuous outdoor background ambient noise (wind/traffic hum) throughout the clip.", "edited_visual": "The bird is partially removed, leaving a blurry dark/grey ghosting smudge where its tail and body were on top of the birdhouse from 0.2s to 3.0s, and a brief smudge on the railing around 3.8s-4.0s.", "edited_audio": "The audio is completely unchanged from SOURCE; the bird calls at 3.0s-4.2s and ambient noise remain intact." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } } ] }, { "id": "A.TEST.5.o_ab733b929067", "kind": "A", "clip": "TEST.5", "version": "TEST", "task": "removal", "video_id": "Cbo-v02GXDc", "instruction": "Remove the birds.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.25, V=0.75", "src": "media/i_55876245_src.mp4", "ref": null, "output": "media/o_ab733b929067.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The upper bird's body and red legs remain visible from 0.0s to 3.5s, and a red remnant of the lower bird persists from 4.0s to 5.0s. Additionally, the inpainted region is a blurry texture blob throughout, and the entire frame degrades into a blurry green smear from 3.6s to 5.0s.", "evidence": { "source_target_visible": "Two birds are visible perched in the tree branches: the upper bird from 0.0s to 3.2s, and the lower bird from 0.0s to 5.0s.", "source_target_audible": "A bird call/squawk is heard around 1.8s to 2.2s, along with background bird chirps.", "source_other_sounds": "Loud continuous insect/cicada buzzing throughout the clip (0.0s-5.0s).", "edited_visual": "The lower bird is removed and replaced with blurry green foliage texture, though a red spot (remnant of its beak/legs) remains around 4.0s-5.0s. The upper bird's body, tail, and red legs remain clearly visible from 0.0s to 3.5s. From 3.6s to 5.0s, the entire frame becomes a blurry green smear.", "edited_audio": "The bird squawk at 1.8s-2.2s is removed. The insect buzzing remains, though slightly altered/filtered." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_7d5c9eae0293.mp4", "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The upper bird fails to be removed from 0.0s to 3.1s, and the inpainted area where the lower bird was (0.0s-5.0s) is a blurry, flickering green smear that also distorts surrounding foliage.", "evidence": { "source_target_visible": "Two birds are perched on tree branches: an upper bird visible from 0.0s to 3.1s, and a lower bird visible throughout the clip (0.0s to 5.0s).", "source_target_audible": "Bird calls/squawks are audible around 1.8s to 2.2s and fainter chirps around 2.8s to 3.2s.", "source_other_sounds": "Continuous loud background insect/cicada buzzing throughout the clip.", "edited_visual": "The lower bird is removed and replaced with blurry, flickering green texture, while the upper bird remains visible from 0.0s to 3.1s.", "edited_audio": "The bird calls are removed, leaving only the continuous background insect buzzing, which sounds slightly muffled/filtered." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_659dae4383ef.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "The upper-left bird's body and legs remain visible from 0.0s to 3.1s, and the lower bird is replaced by an implausible blurry brown blob from 3.0s to 5.0s. Bird calls are still audible (especially 3.8s-4.2s) and the audio track is heavily distorted with synthetic screeching noises replacing the original insect ambience throughout 0.0s-5.0s.", "evidence": { "source_target_visible": "Two birds are visible perched on branches: one in the upper left from 0.0s to 3.1s, and one in the lower right throughout 0.0s to 5.0s.", "source_target_audible": "Loud bird calls and squawks throughout 0.0s to 5.0s, especially sharp calls around 1.8s-2.5s and 3.8s-4.5s.", "source_other_sounds": "Continuous high-pitched insect/cicada buzzing ambience throughout.", "edited_visual": "The lower-right bird is replaced by a blurry brownish clump (3.0s-5.0s), while the upper-left bird's body and orange legs remain clearly visible from 0.0s to 3.1s (only its head is erased).", "edited_audio": "The audio is replaced with synthetic, distorted bird-like screeches and whooshing noises (0.0s-5.0s); bird calls remain around 3.8s-4.2s, and the original insect ambience is altered." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_Cbo-v02GXDc.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_85612eb19faa.mp4", "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "failure_notes": "The upper bird was not removed and remains visible from 0.0s to 3.8s.", "evidence": { "source_target_visible": "Two birds are perched in the tree branches: an upper bird from 0.0s to 3.2s (feet visible until 3.8s) and a lower bird from 0.0s to 5.0s.", "source_target_audible": "Bird calls/chirps are heard throughout the clip (e.g., around 2.0s and 3.0s), along with wing flapping at 1.6s-1.9s.", "source_other_sounds": "Continuous high-frequency insect/forest ambience throughout.", "edited_visual": "The lower bird is removed and replaced with foliage throughout 0.0s-5.0s, but the upper bird remains completely visible from 0.0s to 3.8s.", "edited_audio": "The bird calls and wing flaps are removed, leaving a slightly muffled insect ambience throughout 0.0s-5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" } } ] }, { "id": "A.TEST2.2.o_cdd4e3011941", "kind": "A", "clip": "TEST2.2", "version": "TEST2", "task": "removal", "video_id": "8hT_Cg551_0_000030", "instruction": "Remove the Canada goose.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.25, V=1.00", "src": "media/i_69875217_src.mp4", "ref": null, "output": "media/o_cdd4e3011941.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "The target goose's vocalizations were not removed from the audio track (audible at 0.8s, 2.0s, 3.0s, and 3.8s), resulting in disembodied goose sounds with no visible source.", "evidence": { "source_target_visible": "The Canada goose is visible on the left side of the nest throughout the entire clip (0.0s-5.0s).", "source_target_audible": "The goose makes low grunting and honking vocalizations intermittently around 0.8s, 2.0s, 3.0s, and 3.8s.", "source_other_sounds": "Background wind and outdoor ambience throughout (0.0s-5.0s).", "edited_visual": "The goose is completely removed and replaced with nest/grass texture throughout (0.0s-5.0s); the rest of the scene is unchanged.", "edited_audio": "The audio is unchanged from SOURCE; the goose's grunting and honking vocalizations remain clearly audible at 0.8s, 2.0s, 3.0s, and 3.8s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_1cc023ce388d.mp4", "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "Throughout 00:00.000-00:05.000, the inpainted region where the goose was is a blurry smear, and the top-left text and surrounding nest texture are heavily warped and degraded.", "evidence": { "source_target_visible": "Visible on the left side of the nest throughout 00:00.000-00:05.000.", "source_target_audible": "Honking and grunting calls heard around 00:00.900-00:04.200.", "source_other_sounds": "Background wind and ambient outdoor noise throughout.", "edited_visual": "The goose is removed (00:00.000-00:05.000), leaving a blurry, smeared texture over the nest and warping the top-left overlay text.", "edited_audio": "Goose calls are removed; background ambient noise remains throughout." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_fa92cd15ea2a.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "Target goose grunts/honks remain audible at 00:00.9, 00:01.9, 00:02.9, and 00:04.0, and the inpainted region on the left of the nest is a blurry smear throughout 00:00.0-00:05.0.", "evidence": { "source_target_visible": "A Canada goose is perched on the left side of the nest throughout 00:00.000-00:05.000.", "source_target_audible": "The goose makes low grunting and honking vocalizations around 00:00.9, 00:01.9, 00:02.9, and 00:04.0.", "source_other_sounds": "Outdoor wind/ambience noise throughout.", "edited_visual": "The goose is removed from 00:00.000-00:05.000, leaving a blurry brownish-grey patch on the left of the nest.", "edited_audio": "The goose vocalizations are still audible (attenuated/muffled) around 00:00.9, 00:01.9, 00:02.9, and 00:04.0, along with background noise." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_8hT_Cg551_0_000030.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_d32883065d4f.mp4", "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "Visual removal fails because a dark, blurry remnant of the goose's head appears in the nest from 00:02.625 to 00:05.000, and the inpainted area on the left is a blurry smear (00:00.000-00:05.000).", "evidence": { "source_target_visible": "A Canada goose is visible on the left side of the nest throughout the 00:00.000-00:05.000 clip.", "source_target_audible": "The goose makes low grunting and honking sounds at 00:00.800-00:01.200, 00:01.800-00:02.300, 00:02.900-00:03.300, and 00:03.800-00:04.300.", "source_other_sounds": "Continuous background wind/static ambience throughout the clip.", "edited_visual": "The goose is mostly removed, replaced by a blurry nest texture. However, from 00:02.625 to 00:05.000, a dark remnant/ghost of the goose's head and neck appears in the center of the nest where the head moved in the source.", "edited_audio": "The goose's vocalizations are largely removed, leaving mostly background static, though very faint muffled remnants/dips occur around 00:02.000 and 00:04.000." }, "consistency_verifiable": "No", "consistency_label": "NA" } } ] }, { "id": "A.TEST2.3.o_105085974621", "kind": "A", "clip": "TEST2.3", "version": "TEST2", "task": "removal", "video_id": "5mGW11vXkJA", "instruction": "Remove the machine.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.00, V=0.75", "src": "media/i_61bcfedf_src.mp4", "ref": null, "output": "media/o_105085974621.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Uncertain", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "its shadow is still on the desk?" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "The machine's motor sounds were not removed from the audio track (00:00.00-00:05.00), leaving mechanical sounds with no visible moving source.", "evidence": { "source_target_visible": "Delta robot arm visible at top center from 00:00.00 to 00:01.75; metal frame/bed visible throughout.", "source_target_audible": "Mechanical motor humming and whirring from 00:00.00 to 00:05.00.", "source_other_sounds": "none", "edited_visual": "The moving robot arm is removed from 00:00.00 to 00:01.75, while the static frame, bed, and paper remain unchanged.", "edited_audio": "Audio is unchanged; mechanical motor sounds remain throughout from 00:00.00 to 00:05.00." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_4d100081e62b.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "Minor", "failure_notes": "Only the moving effector arm was removed while the rest of the machine frame, rods, and base remain visible (0.0-5.0s), and the machine's motor whirring sound remains audible throughout (0.0-5.0s).", "evidence": { "source_target_visible": "A delta 3D printer/plotter machine is visible throughout 0.0-5.0s, with its moving effector arms visible at the top from 0.0s to 1.75s, and the base/frame/rods visible throughout.", "source_target_audible": "Loud mechanical stepper motor whirring and humming from 0.0s to 5.0s.", "source_other_sounds": "none", "edited_visual": "The moving arm of the machine was removed (0.0-1.75s) with smeared inpainting on the paper/background at 0.0-0.125s, while the rest of the machine (vertical rods, metal bed, knobs, frame) remains visible throughout 0.0-5.0s.", "edited_audio": "The mechanical whirring/humming sound of the machine remains audible throughout 0.0-5.0s, slightly muffled/filtered." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_085b080bf1e7.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "Minor", "failure_notes": "Only the moving arms were removed while the rest of the machine frame/bed remains (0.0-5.0s), and the machine's motor sounds are still audible throughout (0.0-5.0s) despite no visible moving parts.", "evidence": { "source_target_visible": "The machine (a delta robot/plotter with moving arms at the top and a metal frame/bed) is visible throughout 0.0-5.0s; the moving arms draw on paper and retract upwards from 0.0-1.7s.", "source_target_audible": "Motor whining and mechanical whirring/humming from the machine throughout 0.0-5.0s, louder during movement at 0.0-1.2s.", "source_other_sounds": "none", "edited_visual": "The moving arms at the top are removed (0.0-1.7s) and the drawing is already completed at 0.0s, though the rest of the machine frame and bed remain visible throughout 0.0-5.0s.", "edited_audio": "The mechanical motor whining and humming are still clearly audible throughout 0.0-5.0s, slightly muffled." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_5mGW11vXkJA.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_cfbf262951ce.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "Minor", "failure_notes": "The machine's metal bed, frame, and rods remain visible throughout 0.0-5.0s, while the unrelated sheet of paper and drawing were erroneously removed (0.0-5.0s). A residual mechanical hum also persists across 0.0-5.0s.", "evidence": { "source_target_visible": "The delta 3D printer/drawing machine (arms, frame, metal bed) is visible throughout 0.0-5.0s, with the moving head retracting upward between 0.0s and 1.7s.", "source_target_audible": "Mechanical stepper motor whining from 0.0s to 1.7s, and continuous machine fan/motor humming throughout 0.0-5.0s.", "source_other_sounds": "none", "edited_visual": "From 0.0-5.0s, the moving arms and the white paper with the orange drawing are removed, but the rest of the machine (metal bed, vertical rods, frame, knobs, green wires) remains completely visible.", "edited_audio": "The loud motor whining (0.0-1.7s) is reduced, but a continuous low mechanical hum/buzz persists throughout 0.0-5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" } } ] }, { "id": "A.TEST2.4.o_0c6ec550246e", "kind": "A", "clip": "TEST2.4", "version": "TEST2", "task": "removal", "video_id": "qQCmr6xbkzg_000090", "instruction": "Remove the vehicle.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.00, V=1.00", "src": "media/i_164eb464_src.mp4", "ref": null, "output": "media/o_0c6ec550246e.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Major", "failure_notes": "Acoustic removal fails as distorted, muffled engine revving and tire screeching persist from 0.0s to 5.0s despite the vehicle being visually absent.", "evidence": { "source_target_visible": "The silver car is on the road throughout 0.0-5.0s, generating tire smoke from 2.5s.", "source_target_audible": "Loud engine revving and tire screeching from 0.0s to 5.0s.", "source_other_sounds": "Faint outdoor ambience.", "edited_visual": "The car and smoke are removed (0.0-5.0s). The inpainted road area shows slight blurring and broken dashed line markings from 2.5s to 5.0s.", "edited_audio": "The car engine and tire screeching remain audible throughout 0.0-5.0s, though heavily filtered, muffled, and distorted with phasey artifacts." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_138c7f5f303b.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "Major", "failure_notes": "Residual muffled engine and tire screeching sounds remain throughout 0.0s-5.0s with heavy phasey audio artifacts, and slight visual blurring/smoke remnants appear on the road from 2.5s to 5.0s.", "evidence": { "source_target_visible": "A white car is visible on the road from 0.0s to 5.0s, doing a burnout with tire smoke.", "source_target_audible": "Loud engine revving and tire screeching from 0.0s to 5.0s.", "source_other_sounds": "none", "edited_visual": "The car is removed from 0.0s to 5.0s, leaving a slightly blurry road patch and faint smoke remnants around 3.0s-5.0s.", "edited_audio": "Engine sounds are mostly gone, replaced by muffled, phasey noise and residual tire screeching/engine whine throughout 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_50529ba2e1a8.mp4", "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "The car's loud engine revving and tire screeching are not removed from the audio track (0.0-5.0s), leaving the sound of a burnout on an empty road.", "evidence": { "source_target_visible": "The silver car is visible on the road throughout the clip (0.0-5.0s), producing tire smoke from 2.5s to 5.0s.", "source_target_audible": "Loud engine revving and tire screeching from the car throughout the clip (0.0-5.0s).", "source_other_sounds": "none", "edited_visual": "The car and most of the smoke are removed (0.0-5.0s), with a slight blurry smear on the road and field edge from 2.5s to 5.0s where the smoke was.", "edited_audio": "The car engine revving and tire screeching remain clearly audible throughout the clip (0.0-5.0s)." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_qQCmr6xbkzg_000090.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_cb197de378ae.mp4", "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "The vehicle's tire smoke remains visible on the empty road from 2.5s to 5.0s, and the loud engine revving and tire screeching audio remains throughout the entire 0.0s-5.0s clip.", "evidence": { "source_target_visible": "A white car is visible on the road from 0.0s to 5.0s, producing tire smoke starting around 2.5s.", "source_target_audible": "Loud car engine revving and tire screeching sounds are audible from 0.0s to 5.0s.", "source_other_sounds": "none", "edited_visual": "The car is removed from 0.0s to 5.0s, but the tire smoke it produced remains on the road from 2.5s to 5.0s.", "edited_audio": "The car engine revving and tire screeching sounds remain unchanged from 0.0s to 5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" } } ] }, { "id": "A.TEST2.5.o_978a9b87b140", "kind": "A", "clip": "TEST2.5", "version": "TEST2", "task": "removal", "video_id": "DGd846T0mus", "instruction": "Remove the someone chewing food.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "pseudo-target: A=0.00, V=0.50", "src": "media/i_d15d3504_src.mp4", "ref": null, "output": "media/o_978a9b87b140.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Uncertain", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "Chewing sounds were not removed from the audio (00.0s-05.0s). Visually, the unrelated salad in the bowl was erroneously erased and replaced with a blurry smear (00.0s-05.0s), leaving the fork stabbing empty air from 02.0s-05.0s.", "evidence": { "source_target_visible": "not visible (only the salad in the bowl and a fork entering at 02.0s are visible).", "source_target_audible": "Loud crunching and chewing sounds throughout 00.0s-05.0s.", "source_other_sounds": "Subtle handling and fork piercing salad around 02.3s-03.5s.", "edited_visual": "The entire salad in the bowl was removed (00.0s-05.0s), leaving an empty blurred bowl interior; the fork still enters at 02.0s and stabs into the empty space.", "edited_audio": "Unchanged; the loud chewing and crunching sounds remain present throughout 00.0s-05.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [ { "method": "AVIO (ours)", "row": "ours_refwarm_ep019", "file": "media/o_1c562d7c7930.mp4", "gemini": { "V": "Uncertain", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The chicken in the salad and the fork are severely smeared and distorted throughout the entire clip (0.0s-5.0s), damaging the unrelated visual scene.", "evidence": { "source_target_visible": "Not visible directly; only a fork stabbing chicken in a salad is visible from 0.0s to 5.0s.", "source_target_audible": "Loud chewing, crunching, and mouth sounds throughout (0.0s-5.0s).", "source_other_sounds": "Fork piercing food around 2.0s-3.0s and faint room ambience.", "edited_visual": "The chicken pieces in the salad are heavily smeared and inpainted throughout (0.0s-5.0s), and the fork entering at 2.0s is distorted and ghosted.", "edited_audio": "Chewing sounds are completely removed, leaving only quiet ambient room noise from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "InstructAV2AV FT", "row": "ft_remove_s2500", "file": "media/o_e96961b63564.mp4", "gemini": { "V": "Uncertain", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "Major", "failure_notes": "The person chewing is off-screen in SOURCE, but the edit wrongly removed the visible fork from 0.8s-5.0s (P_visual Fail). In audio, muffled chewing/crunching remnants remain throughout and a synthetic squeak artifact occurs at 4.0s (A and P_audio Fail).", "evidence": { "source_target_visible": "not visible (only a fork poking into chicken in a salad bowl is visible from 0.8s to 5.0s)", "source_target_audible": "loud chewing and crunching sounds throughout 0.0s to 5.0s", "source_other_sounds": "faint room ambience and slight rustling/poking of the fork in the salad around 2.0s-3.5s", "edited_visual": "the fork poking into the salad from 0.8s to 5.0s is completely removed; the salad bowl remains static", "edited_audio": "chewing sounds are attenuated/muffled, with residual crunching and a sharp squeak/artifact around 4.0s" }, "consistency_verifiable": "No", "consistency_label": "NA" } }, { "method": "Cascade (AVIO video + CAVEdit-FT audio)", "row": "cascade_avio_video_cavft_audio", "file": "media/cas_DGd846T0mus.mp4", "gemini": null }, { "method": "InstructAV2AV (zero-shot)", "row": "InstructAV2AV", "file": "media/o_065462030f3f.mp4", "gemini": { "V": "Uncertain", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The target chewing sounds remain audible throughout the entire clip (0.0s-5.0s). Visually, the non-target fork was mistakenly removed (0.2s-5.0s), leaving a major blurry smear over the chicken.", "evidence": { "source_target_visible": "not visible (person chewing is off-screen; only a fork stabbing chicken in a salad is visible from 0.2s to 5.0s).", "source_target_audible": "Loud crunching and chewing sounds throughout the clip (0.0s to 5.0s).", "source_other_sounds": "Slight sound of the fork poking the food around 0.2s-0.3s, mixed with chewing.", "edited_visual": "The fork visible from 0.2s to 5.0s in SOURCE was erroneously removed, leaving a blurry, smeared brown patch over the chicken piece on the left (0.2s to 5.0s).", "edited_audio": "Chewing sounds are still clearly audible throughout (0.0s to 5.0s), though slightly muffled/filtered compared to SOURCE." }, "consistency_verifiable": "No", "consistency_label": "NA" } } ] }, { "id": "C.TEST.11.o_97487861c45b", "kind": "C", "clip": "TEST.11", "version": "TEST", "task": "removal", "video_id": "rD6Nmmy_uR8", "instruction": "Remove the dog.", "method_row": "ft_remove_s2500", "method": "InstructAV2AV FT", "why": "A (argon V=1 A=0)", "src": "media/i_8094ca41_src.mp4", "ref": null, "output": "media/o_97487861c45b.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Major", "failure_notes": "Acoustic removal failed as remnants of the dog barks remain audible as muffled, distorted sounds around 1.0s, 1.9s, and 4.0s.", "evidence": { "source_target_visible": "Two white dogs are visible on the tiled patio from 0.0s to 5.0s.", "source_target_audible": "The dogs bark/yelp sharply around 1.0s, 1.9s, and 4.0s.", "source_other_sounds": "Background bird chirping and general outdoor ambience throughout.", "edited_visual": "Both dogs are completely removed from 0.0s to 5.0s, replaced with reconstructed floor tiles and sunlight.", "edited_audio": "The sharp barks at 1.0s, 1.9s, and 4.0s are muffled/attenuated into distorted vocal/groan-like artifacts, while bird chirping remains." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "C.TEST.13.o_f80dc0b3dcd1", "kind": "C", "clip": "TEST.13", "version": "TEST", "task": "removal", "video_id": "pMqSXizkeCY", "instruction": "Remove the birds.", "method_row": "ft_remove_s2500", "method": "InstructAV2AV FT", "why": "V (argon V=0 A=1)", "src": "media/i_e14e9683_src.mp4", "ref": null, "output": "media/o_f80dc0b3dcd1.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "Dark birds remain visible at the top-center and top-left throughout 0.0s-5.0s, and the inpainted central region is a large, implausible blurry smear (0.0s-5.0s). The audio introduces unnatural human gasping and vocal artifacts across the entire clip (0.0s-5.0s).", "evidence": { "source_target_visible": "Multiple white and dark birds (pigeons) are visible in the center and foreground from 0.0s to 5.0s.", "source_target_audible": "Bird cooing and wing fluttering sounds are audible throughout 0.0s to 5.0s.", "source_other_sounds": "General ambient background noise from 0.0s to 5.0s.", "edited_visual": "The foreground white birds are replaced by a large, blurry, brownish-grey smear from 0.0s to 5.0s, while dark birds remain partially visible in the top-left and top-center (especially 4.5s-5.0s).", "edited_audio": "Bird cooing and fluttering are replaced with unnatural human-like sighing, gasping, and muffled vocalizations from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "C.TEST.14.o_af82aaeb67ec", "kind": "C", "clip": "TEST.14", "version": "TEST", "task": "removal", "video_id": "J4yP-QsdjSo_000110", "instruction": "Remove the guitar.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "V (argon V=1 A=1)", "src": "media/i_a09d0dc2_src.mp4", "ref": null, "output": "media/o_af82aaeb67ec.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "From 00:00 to 00:05, the removal of the guitar leaves a massive, implausible blurry smear across the center of the frame that erases the person's hands, torso, and right leg.", "evidence": { "source_target_visible": "The bass guitar is visible in the foreground being played by a person from 00:00 to 00:05.", "source_target_audible": "Prominent bass guitar notes and riffs are audible throughout the clip (00:00-00:05).", "source_other_sounds": "Background music track with drums and rhythm instruments (00:00-00:05).", "edited_visual": "The guitar is removed, but the person's torso, arms, hands, and right leg are replaced by a large, blurry beige and dark smear (00:00-00:05).", "edited_audio": "The foreground bass guitar riffs are largely removed, leaving the muffled background music track (00:00-00:05)." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "C.TEST.5.o_ab733b929067", "kind": "C", "clip": "TEST.5", "version": "TEST", "task": "removal", "video_id": "Cbo-v02GXDc", "instruction": "Remove the birds.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "V A (argon V=0 A=1)", "src": "media/i_55876245_src.mp4", "ref": null, "output": "media/o_ab733b929067.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The upper bird's body and red legs remain visible from 0.0s to 3.5s, and a red remnant of the lower bird persists from 4.0s to 5.0s. Additionally, the inpainted region is a blurry texture blob throughout, and the entire frame degrades into a blurry green smear from 3.6s to 5.0s.", "evidence": { "source_target_visible": "Two birds are visible perched in the tree branches: the upper bird from 0.0s to 3.2s, and the lower bird from 0.0s to 5.0s.", "source_target_audible": "A bird call/squawk is heard around 1.8s to 2.2s, along with background bird chirps.", "source_other_sounds": "Loud continuous insect/cicada buzzing throughout the clip (0.0s-5.0s).", "edited_visual": "The lower bird is removed and replaced with blurry green foliage texture, though a red spot (remnant of its beak/legs) remains around 4.0s-5.0s. The upper bird's body, tail, and red legs remain clearly visible from 0.0s to 3.5s. From 3.6s to 5.0s, the entire frame becomes a blurry green smear.", "edited_audio": "The bird squawk at 1.8s-2.2s is removed. The insect buzzing remains, though slightly altered/filtered." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "C.TEST2.1.o_e32d4da9219c", "kind": "C", "clip": "TEST2.1", "version": "TEST2", "task": "removal", "video_id": "-8e8LB54vC8_000117", "instruction": "Remove the alarm clock.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "A (argon V=1 A=0)", "src": "media/i_a39d297f_src.mp4", "ref": null, "output": "media/o_e32d4da9219c.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": true, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The person's hands and forearms were erroneously removed along with the clock across the entire clip (00:00.000-00:05.000) with initial frame blurring at 00:00.000, and the clock's button click sound remains audible at 00:04.200 despite no visible source.", "evidence": { "source_target_visible": "Alarm clock held in hands from 00:00.000 to 00:05.000.", "source_target_audible": "Electronic beeping throughout (00:00.000-00:04.200) and a button click at 00:04.200.", "source_other_sounds": "Faint handling/movement noises.", "edited_visual": "The alarm clock AND the person's hands/forearms are completely removed throughout (00:00.000-00:05.000), leaving a static, frozen background with initial blurring at 00:00.000.", "edited_audio": "Beeping is removed, but the click sound from turning off the alarm clock remains at 00:04.200." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "C.TEST2.13.o_2caf63bbe82b", "kind": "C", "clip": "TEST2.13", "version": "TEST2", "task": "removal", "video_id": "IS9j-UR7QIk_000027", "instruction": "Remove the siren.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "V (argon V=1 A=0)", "src": "media/i_ae0f31b4_src.mp4", "ref": null, "output": "media/o_2caf63bbe82b.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The siren audio remains audible throughout the clip (0.0s-5.0s), and the visual edit severely damages the entire fire truck and background with a massive smeared texture blob from 0.0s to 5.0s.", "evidence": { "source_target_visible": "Flashing red siren lights on the top and front of the fire truck are visible from 0.0s to 5.0s.", "source_target_audible": "Loud wailing siren sound throughout the entire clip (0.0s to 5.0s).", "source_other_sounds": "Street and traffic noise, engine rumble, and a bell ringing throughout (0.0s to 5.0s).", "edited_visual": "The entire fire truck and surrounding street area are replaced by a severe, smeared vertical texture artifact from 0.0s to 5.0s, though the lower headlights and some red truck parts remain.", "edited_audio": "The wailing siren is still clearly audible throughout (0.0s to 5.0s), though slightly muffled and distorted between 2.0s and 4.5s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "C.TEST2.14.o_22c74d2c6762", "kind": "C", "clip": "TEST2.14", "version": "TEST2", "task": "removal", "video_id": "Komjd9MRmF8_000030", "instruction": "Remove the goat.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "A (argon V=0 A=1)", "src": "media/i_f27e0a1b_src.mp4", "ref": null, "output": "media/o_22c74d2c6762.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The goat's body and legs remain visible throughout 0.0s-5.0s, and the region where its head was removed is a blurry smear (0.0s-5.0s).", "evidence": { "source_target_visible": "A black goat with long white ears is visible in the foreground from 0.0s to 5.0s.", "source_target_audible": "The goat bleats around 0.8s-1.5s, 2.0s-2.5s, and 4.0s-5.0s.", "source_other_sounds": "Outdoor ambience and bird chirping throughout 0.0s-5.0s.", "edited_visual": "Only the goat's head and white ears are blurred out; its black body and legs remain completely visible from 0.0s to 5.0s.", "edited_audio": "The goat bleating sounds are removed, while the background ambience and bird sounds remain." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "C.TEST.18.o_fe7780fe35c7", "kind": "C", "clip": "TEST.18", "version": "TEST", "task": "addition", "video_id": "b7eT98wQRtc_000030", "instruction": "Add the engine.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "V A (argon V=0 A=0)", "src": "media/i_e39214ce_src.mp4", "ref": "media/i_e39214ce_ref.jpg", "output": "media/o_fe7780fe35c7.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The added engine severely warps and melts between 0.1s and 0.6s, the whole video loses sharpness and looks degraded throughout (0.0s-5.0s), and the engine sound is only an isolated burst at 0.0s-0.5s rather than a continuous boat engine sound.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Water splashing and wind noise throughout (0.0s-5.0s).", "edited_visual": "An engine is inserted at the back of the boat from 0.0s to 0.5s, but it quickly distorts and melts into blurry artifacts from 0.1s to 0.6s. The entire video is noticeably degraded, blurry, and painterly throughout (0.0s-5.0s).", "edited_audio": "A brief engine revving sound is added at the very beginning (0.0s-0.5s), while the rest of the clip (0.5s-5.0s) retains the water and wind sounds from the source." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "C.TEST.18.o_f634fa678878", "kind": "C", "clip": "TEST.18", "version": "TEST", "task": "addition", "video_id": "b7eT98wQRtc_000030", "instruction": "Add the engine.", "method_row": "ft_add_s2500_ff", "method": "InstructAV2AV FT", "why": "V (argon V=0 A=1)", "src": "media/i_e39214ce_src.mp4", "ref": "media/i_e39214ce_ref.jpg", "output": "media/o_f634fa678878.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "At 0.6-0.9s the driver is severely blurred and warped, and from 0.8-5.0s a large deformed boat-like object is hallucinated floating on the water ahead.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Water splashing and wind noise throughout (0.0-5.0s).", "edited_visual": "An engine matching the reference is added in the back of the boat from 0.0-0.5s. As the camera pans forward at 0.6s, the person driving is severely blurred/distorted, and a deformed boat/engine blob appears floating on the water in front of the boat from 0.8-5.0s.", "edited_audio": "A loud motor/engine roar is added throughout (0.0-5.0s) over the water/wind noise." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST.19.o_17cbd0913e34", "kind": "C", "clip": "TEST.19", "version": "TEST", "task": "addition", "video_id": "BjHG334T3bE", "instruction": "Add the stringed instrument.", "method_row": "ft_add_s2500_ff", "method": "InstructAV2AV FT", "why": "A (argon V=1 A=1)", "src": "media/i_a8779f6f_src.mp4", "ref": "media/i_a8779f6f_ref.jpg", "output": "media/o_17cbd0913e34.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "none", "evidence": { "source_target_visible": "Remnants of tuning pegs visible at 0.0-5.0s, otherwise not visible.", "source_target_audible": "not audible", "source_other_sounds": "Distorted low-frequency ambient music/hum from 0.0 to 5.0s.", "edited_visual": "A cello is added in the man's hands from 0.0 to 5.0s, matching the reference image while keeping the person and background unchanged.", "edited_audio": "Plucked cello notes are added from 0.0 to 5.0s on top of the existing background audio." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST.22.o_51e177dc424b", "kind": "C", "clip": "TEST.22", "version": "TEST", "task": "addition", "video_id": "QDqBJwoQcNY_000080", "instruction": "Add the guinea fowl.", "method_row": "ft_add_s2500_ff", "method": "InstructAV2AV FT", "why": "V (argon V=0 A=1)", "src": "media/i_d942d215_src.mp4", "ref": "media/i_d942d215_ref.jpg", "output": "media/o_51e177dc424b.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The added guinea fowl has a grotesque, heavily deformed head with unnatural white eyes and morphing facial features throughout 00:00.1-00:05.0.", "evidence": { "source_target_visible": "not visible (only an inpainted greyish texture where the bird should be)", "source_target_audible": "not audible", "source_other_sounds": "outdoor ambience, faint distant bird chirping around 00:01-00:03", "edited_visual": "A guinea fowl is added in the center from 00:00 to 00:05, matching the speckled body in the reference, though its head and face are severely deformed with bizarre large white eyes and morphing red wattles/beak throughout 00:00.1-00:05.0.", "edited_audio": "Loud, harsh squawking calls are added around 00:02.9-00:03.2 and 00:03.7-00:04.2 on top of the preserved background ambience." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST.8.o_d54e76aad886", "kind": "C", "clip": "TEST.8", "version": "TEST", "task": "addition", "video_id": "cmG3XZ0NG_A", "instruction": "Add the bird.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "A (argon V=1 A=0)", "src": "media/i_09a423dc_src.mp4", "ref": "media/i_09a423dc_ref.jpg", "output": "media/o_d54e76aad886.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "Audio was not newly added in EDITED; the target duck sounds were already present in SOURCE from 00.0s to 05.0s.", "evidence": { "source_target_visible": "Partially inpainted ghostly remnant of a duck visible on the left from 01.7s to 04.0s.", "source_target_audible": "Duck quacking is already audible throughout SOURCE (e.g., 01.0s, 02.0s, 03.0s).", "source_other_sounds": "Ambient outdoor noise.", "edited_visual": "Several ducks are present from 00.0s to 05.0s, moving around and flapping wings on the gravel.", "edited_audio": "Identical to SOURCE; duck quacks and ambient noise remain unchanged from 00.0s to 05.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST.9.o_a9ed7382d65a", "kind": "C", "clip": "TEST.9", "version": "TEST", "task": "addition", "video_id": "j-f_5hAsrp0_000062", "instruction": "Add the birds.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "V A (argon V=0 A=0)", "src": "media/i_b9ab6fad_src.mp4", "ref": "media/i_b9ab6fad_ref.jpg", "output": "media/o_a9ed7382d65a.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "", "core": true, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "Visual addition and preservation fail because the wire changes orientation initially (0.0-1.5s) and the bird flies off at 0.75s only to ghost back into existence around 1.3-1.8s. Acoustic addition and consistency fail because no new bird or wing-flap sounds were added to match the foreground bird's actions.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "Background bird chirping is already audible throughout 0.0-5.0s, though no specific foreground bird is visible.", "source_other_sounds": "Ambient bird chirping and insect sounds throughout 0.0-5.0s.", "edited_visual": "A swallow is perched on the wire at 0.0s, flies away around 0.6s-1.0s, and then a ghostly bird fades back into existence on the wire between 1.3s and 1.8s, while the wire itself warps and shifts orientation from 0.0s to 1.8s.", "edited_audio": "The audio track is identical to SOURCE; no new bird calls or wing-flapping sounds were added." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "C.TEST2.17.o_7454de6c39e3", "kind": "C", "clip": "TEST2.17", "version": "TEST2", "task": "addition", "video_id": "5TdzFtSeBDI_000130", "instruction": "Add the percussion instrument.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "A (argon V=1 A=0)", "src": "media/i_0aaf7a8f_src.mp4", "ref": "media/i_0aaf7a8f_ref.jpg", "output": "media/o_7454de6c39e3.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "The audio in EDITED is identical to SOURCE from 0.0s to 5.0s, meaning no new target sound was added.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "The marimba chords and notes are already audible throughout SOURCE (0.0-5.0s).", "source_other_sounds": "Vibraphone music played by the left musician (0.0-5.0s).", "edited_visual": "A large wooden marimba is added in front of the right musician (0.0-5.0s), though the keys look slightly warped and blurry. Background and people are preserved.", "edited_audio": "The audio is identical to SOURCE (0.0-5.0s); no new sound was added because the marimba sound was already present in SOURCE." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST2.17.o_4ccbb0cd6d71", "kind": "C", "clip": "TEST2.17", "version": "TEST2", "task": "addition", "video_id": "5TdzFtSeBDI_000130", "instruction": "Add the percussion instrument.", "method_row": "ft_add_s2500_ff", "method": "InstructAV2AV FT", "why": "V (argon V=0 A=1)", "src": "media/i_0aaf7a8f_src.mp4", "ref": "media/i_0aaf7a8f_ref.jpg", "output": "media/o_4ccbb0cd6d71.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "At 0.125s-5.0s, the central music stand morphs into a giant brown lid attached to the added marimba, and an unrelated person appears in the bottom-left corner from 1.0s-5.0s.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Vibraphone music played by the person on the left throughout 0.0-5.0s.", "edited_visual": "A marimba is added on the right from 0.0-5.0s, but at 0.125s the music stand in the center morphs into a large piano-lid-like object attached to the marimba, and a hallucinated person appears in the bottom left corner from 1.0-5.0s.", "edited_audio": "Marimba notes are added at 0.0-1.0s and 4.0-5.0s, while the original vibraphone chords remain audible throughout 0.0-5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST2.18.o_c716e7af7ead", "kind": "C", "clip": "TEST2.18", "version": "TEST2", "task": "addition", "video_id": "IzJSE_bLQTc_000142", "instruction": "Add the birds.", "method_row": "ft_add_s2500_ff", "method": "InstructAV2AV FT", "why": "V (argon V=0 A=1)", "src": "media/i_c35e55b2_src.mp4", "ref": "media/i_c35e55b2_ref.jpg", "output": "media/o_c716e7af7ead.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "At 0.0-0.2s, the bird abruptly morphs from the reference appearance into a different dark bird with a longer tail.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Background bird chirping and outdoor ambience throughout 0.0-5.0s.", "edited_visual": "A bird is added on the branch, starting as the reference bird at 0.0s, then morphing into a different darker bird with a longer tail from 0.1s to 5.0s. It opens its beak to call around 3.5-4.3s.", "edited_audio": "Additional bird calls are added throughout 0.0-5.0s on top of the background ambience." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST2.20.o_d6deb7dbb2b8", "kind": "C", "clip": "TEST2.20", "version": "TEST2", "task": "addition", "video_id": "SPyyUMPdkCA_000018", "instruction": "Add the birds.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "V (argon V=0 A=1)", "src": "media/i_bfe10fbf_src.mp4", "ref": "media/i_bfe10fbf_ref.jpg", "output": "media/o_d6deb7dbb2b8.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Uncertain", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "don't certain about the sound needed" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The inserted birds suffer from severe warping and melting around their heads and beaks throughout 0.0-5.0s, and the entire background tarp and upper-right structure become blurry and distorted.", "evidence": { "source_target_visible": "Only bird legs/inpainting remnants are faintly visible on the tarp throughout 0.0-5.0s; the full birds are not visible.", "source_target_audible": "not audible", "source_other_sounds": "Outdoor ambience with wind, water sounds, and distant indistinct voices/chatter around 1.0-3.0s.", "edited_visual": "Two black-headed gulls are added on the tarp from 0.0-5.0s, matching the reference image, though their heads and beaks warp/melt into each other and the background is slightly degraded/warped.", "edited_audio": "Bird chirps/calls are added throughout 0.0-5.0s (e.g., at 0.0-1.0s, 3.0-4.8s) while the background ambience and distant voices are preserved." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST2.6.o_b76479ca3f5c", "kind": "C", "clip": "TEST2.6", "version": "TEST2", "task": "addition", "video_id": "5kAjqqBLuYc_000030", "instruction": "Add the horse.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "A (argon V=1 A=0)", "src": "media/i_59bd5385_src.mp4", "ref": "media/i_59bd5385_ref.jpg", "output": "media/o_b76479ca3f5c.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "Acoustic addition fails because the horse galloping sound (0.0s-3.0s) was already present as a residual in the SOURCE audio track and no new sound was added.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "Galloping hoofbeats are already faintly audible in SOURCE from 0.0s to 3.0s despite the horse being removed from the video.", "source_other_sounds": "Outdoor wind ambience and birds chirping (3.5s-5.0s).", "edited_visual": "A white horse with a rider in red and white is added galloping away from the camera across the field from 0.0s to 3.6s, matching the reference image. Another distant rider briefly appears near the jump around 3.5s-3.9s. Background and scene dissolve transition remain preserved.", "edited_audio": "The audio track in EDITED is identical to SOURCE, containing the same galloping sounds (0.0s-3.0s), wind ambience, and birds chirping (3.5s-5.0s)." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "C.TEST2.7.o_32313b659f3e", "kind": "C", "clip": "TEST2.7", "version": "TEST2", "task": "addition", "video_id": "3W6ovRy3YH8_000260", "instruction": "Add the sewing machine.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "A (argon V=1 A=0)", "src": "media/i_ef78dc43_src.mp4", "ref": "media/i_ef78dc43_ref.jpg", "output": "media/o_32313b659f3e.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Uncertain", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "Source already contains sound of sewing machine" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "The target sewing machine sound was already present in SOURCE (0.0s-5.0s) and was not newly added in EDITED.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "Sewing machine motor whirring and stitching sounds are already audible from 0.0s to 5.0s, with louder bursts at 2.0s and 3.0s.", "source_other_sounds": "Faint fabric handling and mechanical clicks throughout.", "edited_visual": "A sewing machine and a person's hands operating it are added from 0.0s to 5.0s, replacing the blurry inpainted region in SOURCE.", "edited_audio": "The audio track is identical to SOURCE, containing the same sewing machine sounds from 0.0s to 5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "B.TEST.11.o_4d57c54885c4", "kind": "B", "clip": "TEST.11", "version": "TEST", "task": "removal", "video_id": "rD6Nmmy_uR8", "instruction": "Remove the dog.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_8094ca41_src.mp4", "ref": null, "output": "media/o_4d57c54885c4.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "failure_notes": "Minor visual blurring and grout warping appear in the inpainted floor region from 0.0s to 0.6s.", "evidence": { "source_target_visible": "Two white dogs are visible on the tiled patio from 0.0s to 5.0s.", "source_target_audible": "The front dog barks loudly around 0.9s, 1.9s, and 4.0s, with panting throughout.", "source_other_sounds": "Background bird chirping and outdoor ambience from 0.0s to 5.0s.", "edited_visual": "Both dogs are removed throughout 0.0s to 5.0s. The tiled floor is inpainted, though with noticeable blurring and warping of grout lines between 0.0s and 0.6s.", "edited_audio": "The loud dog barks are removed, leaving muffled panting/scuffling and background bird chirping from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST.11.o_af957296b4a4", "kind": "B", "clip": "TEST.11", "version": "TEST", "task": "removal", "video_id": "rD6Nmmy_uR8", "instruction": "Remove the dog.", "method_row": "cascade_avio_video_cavft_audio", "method": "Cascade (AVIO video + CAVEdit-FT audio)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_8094ca41_src.mp4", "ref": null, "output": "media/o_af957296b4a4.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "none", "evidence": { "source_target_visible": "Two small white dogs are visible on the tiled floor throughout the entire clip (0.0s - 5.0s).", "source_target_audible": "A dog barks sharply at around 0.9s - 1.1s and 1.8s - 2.0s.", "source_other_sounds": "Birds chirping and outdoor ambience throughout (0.0s - 5.0s), along with some handling/scuffing noises at 0.0s - 0.5s and 4.0s.", "edited_visual": "Both dogs are completely removed throughout (0.0s - 5.0s), replaced by the red tiled floor. Some slight warping and blurring is visible on the reconstructed tiles, especially from 0.0s to 0.5s. The sunlight patch on the right is also removed.", "edited_audio": "The dog barks at 0.9s and 1.8s are completely removed. The background birds chirping and ambient sounds remain intact throughout (0.0s - 5.0s)." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST.12.o_c7e66c5fc626", "kind": "B", "clip": "TEST.12", "version": "TEST", "task": "removal", "video_id": "Gs5QIBsD6bE", "instruction": "Remove the power tool.", "method_row": "cascade_avio_video_cavft_audio", "method": "Cascade (AVIO video + CAVEdit-FT audio)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_87abc64b_src.mp4", "ref": null, "output": "media/o_c7e66c5fc626.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "None", "artifact_audio": "Major", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": true, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "The loud blowing sound of the removed tool remains fully audible throughout 0.0s-4.0s with no visible source.", "evidence": { "source_target_visible": "A yellow hairdryer/power tool held by a hand on the right side from 0.0s to 3.6s.", "source_target_audible": "Loud blowing and whirring sound from 0.0s to 3.6s, followed by a switch click and thud around 3.9s.", "source_other_sounds": "Faint room ambience after 4.0s.", "edited_visual": "The yellow tool and hand are completely removed from 0.0s to 3.6s, revealing the clay stove rim and floor behind it.", "edited_audio": "The audio is completely unchanged; the loud blowing sound of the tool remains from 0.0s to 4.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST.13.o_e83da9e95d73", "kind": "B", "clip": "TEST.13", "version": "TEST", "task": "removal", "video_id": "pMqSXizkeCY", "instruction": "Remove the birds.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_e14e9683_src.mp4", "ref": null, "output": "media/o_e83da9e95d73.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "A bird remains visible in the top-left corner from 0.0s to 4.5s, the blue bowl was unintentionally removed, and the central inpainted region is a large blurry smear (0.0s-5.0s). The bird sounds were not removed from the audio track (0.0s-5.0s).", "evidence": { "source_target_visible": "Multiple white and dark birds (pigeons) are visible in the center, foreground, and top left from 0.0s to 5.0s.", "source_target_audible": "Faint bird cooing, chirping, and wing movement sounds are audible throughout 0.0s-5.0s, especially around 2.0s-4.0s.", "source_other_sounds": "Background room ambience and a clattering noise around 0.5s-1.5s.", "edited_visual": "The central white birds and the blue bowl are removed and replaced with a large blurry smear from 0.0s to 5.0s. However, a black-and-white bird remains visible in the top-left corner from 0.0s to 4.5s.", "edited_audio": "The audio is completely unchanged from SOURCE, retaining all bird cooing and movement sounds from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST.13.o_a6474137d232", "kind": "B", "clip": "TEST.13", "version": "TEST", "task": "removal", "video_id": "pMqSXizkeCY", "instruction": "Remove the birds.", "method_row": "cascade_avio_video_cavft_audio", "method": "Cascade (AVIO video + CAVEdit-FT audio)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_e14e9683_src.mp4", "ref": null, "output": "media/o_a6474137d232.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "A bird on the far left edge remains visible from 00:00.000-00:04.250, the blue feeding bowl is unintentionally removed, and the central inpainted region is a large blurry smear throughout 00:00.000-00:05.000.", "evidence": { "source_target_visible": "Multiple white fantail pigeons and dark pigeons are visible in the center and left foreground throughout 00:00.000-00:05.000.", "source_target_audible": "Pigeon cooing and wing/feather rustling are audible throughout 00:00.000-00:05.000, especially cooing at 00:01.800-00:03.500.", "source_other_sounds": "General ambient background noise and distant clattering.", "edited_visual": "The central white pigeons and the blue bowl are replaced by a large, blurry, pinkish-grey smudged patch from 00:00.000-00:05.000. A black-and-white bird remains partially visible on the far left edge from 00:00.000-00:04.250.", "edited_audio": "Pigeon cooing is removed, leaving muffled ambient scraping and clicking sounds throughout 00:00.000-00:05.000." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST.13.o_f80dc0b3dcd1", "kind": "B", "clip": "TEST.13", "version": "TEST", "task": "removal", "video_id": "pMqSXizkeCY", "instruction": "Remove the birds.", "method_row": "ft_remove_s2500", "method": "InstructAV2AV FT", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_e14e9683_src.mp4", "ref": null, "output": "media/o_f80dc0b3dcd1.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "Dark birds remain visible at the top-center and top-left throughout 0.0s-5.0s, and the inpainted central region is a large, implausible blurry smear (0.0s-5.0s). The audio introduces unnatural human gasping and vocal artifacts across the entire clip (0.0s-5.0s).", "evidence": { "source_target_visible": "Multiple white and dark birds (pigeons) are visible in the center and foreground from 0.0s to 5.0s.", "source_target_audible": "Bird cooing and wing fluttering sounds are audible throughout 0.0s to 5.0s.", "source_other_sounds": "General ambient background noise from 0.0s to 5.0s.", "edited_visual": "The foreground white birds are replaced by a large, blurry, brownish-grey smear from 0.0s to 5.0s, while dark birds remain partially visible in the top-left and top-center (especially 4.5s-5.0s).", "edited_audio": "Bird cooing and fluttering are replaced with unnatural human-like sighing, gasping, and muffled vocalizations from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST.14.o_d5343d463f6b", "kind": "B", "clip": "TEST.14", "version": "TEST", "task": "removal", "video_id": "J4yP-QsdjSo_000110", "instruction": "Remove the guitar.", "method_row": "ft_remove_s2500", "method": "InstructAV2AV FT", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_a09d0dc2_src.mp4", "ref": null, "output": "media/o_d5343d463f6b.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The body of the guitar remains visible on the left throughout 0.0-5.0s, the inpainted neck area is a blurry deformed smear, and the person's motion is frozen from 0.2-5.0s. Guitar and bass sounds remain audible throughout 0.0-5.0s while the player's hands are frozen.", "evidence": { "source_target_visible": "A wooden bass guitar is visible in the foreground held by a seated person throughout 0.0-5.0s.", "source_target_audible": "Bass guitar notes being played throughout 0.0-5.0s.", "source_other_sounds": "Background backing track with drums/percussion and keyboard chords throughout 0.0-5.0s.", "edited_visual": "The neck and headstock of the guitar are removed (0.0-5.0s), leaving a blurry distorted smear where the neck was, while the wooden body of the guitar on the left remains visible (0.0-5.0s). The video appears mostly frozen after the first fraction of a second.", "edited_audio": "The bass guitar notes are still audible, though slightly altered/muffled in places, and an acoustic guitar strumming/picking sound is heard along with the backing track throughout 0.0-5.0s." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST.15.o_32930325a6d7", "kind": "B", "clip": "TEST.15", "version": "TEST", "task": "removal", "video_id": "o9Kx9y3J1HE_000030", "instruction": "Remove the speedboat.", "method_row": "ft_remove_s2500", "method": "InstructAV2AV FT", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_48813e33_src.mp4", "ref": null, "output": "media/o_32930325a6d7.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "The boat's white frame remains on the right edge and the inpainted water is a blurry smear throughout 0.0s-5.0s. The boat engine noise is still audible, heavily distorted, and mixed with garbled vocal artifacts around 1.0s-2.0s.", "evidence": { "source_target_visible": "The red bow and white windshield frame of the speedboat are visible in the bottom right corner from 0.0s to 5.0s.", "source_target_audible": "Loud motorboat engine roar and hull slapping water from 0.0s to 5.0s.", "source_other_sounds": "Heavy wind noise against the microphone throughout 0.0s to 5.0s.", "edited_visual": "The red bow is replaced by a blurry grey water smear, while the white windshield frame on the far right edge remains visible from 0.0s to 5.0s.", "edited_audio": "The boat engine and wind sounds are muffled and distorted, with an unrelated garbled voice-like artifact audible around 1.0s to 2.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST2.1.o_e32d4da9219c", "kind": "B", "clip": "TEST2.1", "version": "TEST2", "task": "removal", "video_id": "-8e8LB54vC8_000117", "instruction": "Remove the alarm clock.", "method_row": "GT", "method": "Pseudo-target (GT)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_a39d297f_src.mp4", "ref": null, "output": "media/o_e32d4da9219c.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "Pass", "core": true, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The person's hands and forearms were erroneously removed along with the clock across the entire clip (00:00.000-00:05.000) with initial frame blurring at 00:00.000, and the clock's button click sound remains audible at 00:04.200 despite no visible source.", "evidence": { "source_target_visible": "Alarm clock held in hands from 00:00.000 to 00:05.000.", "source_target_audible": "Electronic beeping throughout (00:00.000-00:04.200) and a button click at 00:04.200.", "source_other_sounds": "Faint handling/movement noises.", "edited_visual": "The alarm clock AND the person's hands/forearms are completely removed throughout (00:00.000-00:05.000), leaving a static, frozen background with initial blurring at 00:00.000.", "edited_audio": "Beeping is removed, but the click sound from turning off the alarm clock remains at 00:04.200." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST2.11.o_168e1f4ce7ba", "kind": "B", "clip": "TEST2.11", "version": "TEST2", "task": "removal", "video_id": "9qpsyjmda5Q_000138", "instruction": "Remove the birds.", "method_row": "cascade_avio_video_cavft_audio", "method": "Cascade (AVIO video + CAVEdit-FT audio)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_3247a3a0_src.mp4", "ref": null, "output": "media/o_168e1f4ce7ba.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": true, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "The bird calls were not removed from the audio track (0.0s to 5.0s).", "evidence": { "source_target_visible": "A bird is visible flying against a blue sky throughout the entire clip (0.0s to 5.0s).", "source_target_audible": "Loud bird calls and chirps are audible throughout the clip (0.0s to 5.0s).", "source_other_sounds": "none", "edited_visual": "The bird is completely removed and replaced with a blue sky from 0.0s to 5.0s, with a slight brightness flicker at 0.0s.", "edited_audio": "The audio track is unchanged; the bird calls remain audible from 0.0s to 5.0s." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST2.11.o_b31b2bcd0b13", "kind": "B", "clip": "TEST2.11", "version": "TEST2", "task": "removal", "video_id": "9qpsyjmda5Q_000138", "instruction": "Remove the birds.", "method_row": "ft_remove_s2500", "method": "InstructAV2AV FT", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_3247a3a0_src.mp4", "ref": null, "output": "media/o_b31b2bcd0b13.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "Minor", "failure_notes": "The target bird was not visually removed and remains visible for the entire clip (0.0-5.0s), with minor audio filtering artifacts at 0.0-1.3s.", "evidence": { "source_target_visible": "A bird is visible flying against a blue sky throughout the entire clip (0.0-5.0s).", "source_target_audible": "Bird calls and chirping are audible throughout the clip (0.0-5.0s).", "source_other_sounds": "None.", "edited_visual": "The bird remains fully visible throughout the entire clip (0.0-5.0s) with only slight color/texture blurring.", "edited_audio": "Bird chirps are removed, but muffled low-frequency bursts remain at 0.0-0.3s and 1.0-1.3s." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST2.14.o_952e0e48e831", "kind": "B", "clip": "TEST2.14", "version": "TEST2", "task": "removal", "video_id": "Komjd9MRmF8_000030", "instruction": "Remove the goat.", "method_row": "ft_remove_s2500", "method": "InstructAV2AV FT", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_f27e0a1b_src.mp4", "ref": null, "output": "media/o_952e0e48e831.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "Fail", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Pass", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "Fail", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "From 00:00.000 to 00:05.000, the entire video frame is destroyed into a blurry smear, destroying background preservation and plausible reconstruction. In the audio, attenuated and muffled goat vocalizations remain around 00:01.000, 00:02.000, and 00:04.000, and background ambience/bird sounds are heavily muffled.", "evidence": { "source_target_visible": "A black and white goat is visible in the foreground from 00:00.000 to 00:05.000.", "source_target_audible": "Goat bleating and grunting sounds are audible around 00:01.000, 00:02.000, and 00:04.000.", "source_other_sounds": "Outdoor wind/ambience and faint bird chirps throughout 00:00.000 to 00:05.000.", "edited_visual": "The goat is removed, but the entire frame (including the background fence and ground) is replaced by a blurry, smeared gray/brown texture from 00:00.000 to 00:05.000.", "edited_audio": "The audio is muffled and distorted, but faint remnants of the goat bleats/grunts are still audible around 00:01.000, 00:02.000, and 00:04.000, while bird chirps and clear ambience are degraded." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST.18.o_fe7780fe35c7", "kind": "B", "clip": "TEST.18", "version": "TEST", "task": "addition", "video_id": "b7eT98wQRtc_000030", "instruction": "Add the engine.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_e39214ce_src.mp4", "ref": "media/i_e39214ce_ref.jpg", "output": "media/o_fe7780fe35c7.mp4", "votes": [ { "rater": "TEST-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "failure_notes": "The added engine severely warps and melts between 0.1s and 0.6s, the whole video loses sharpness and looks degraded throughout (0.0s-5.0s), and the engine sound is only an isolated burst at 0.0s-0.5s rather than a continuous boat engine sound.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Water splashing and wind noise throughout (0.0s-5.0s).", "edited_visual": "An engine is inserted at the back of the boat from 0.0s to 0.5s, but it quickly distorts and melts into blurry artifacts from 0.1s to 0.6s. The entire video is noticeably degraded, blurry, and painterly throughout (0.0s-5.0s).", "edited_audio": "A brief engine revving sound is added at the very beginning (0.0s-0.5s), while the rest of the clip (0.5s-5.0s) retains the water and wind sounds from the source." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST.20.o_34babd8eab9f", "kind": "B", "clip": "TEST.20", "version": "TEST", "task": "addition", "video_id": "EvE34eomY60", "instruction": "Add the frogs.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_cb30a68e_src.mp4", "ref": "media/i_cb30a68e_ref.jpg", "output": "media/o_34babd8eab9f.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "failure_notes": "No frog sounds were added to the audio track (0.0-5.0s), and the frog's visible throat/body movement around 1.2-2.5s produces no corresponding sound.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Nighttime insect and cicada buzzing throughout (0.0-5.0s).", "edited_visual": "A frog matching the reference image is added in the center (0.0-5.0s) with slight breathing motion around 1.2-2.5s; background remains the same.", "edited_audio": "Identical to SOURCE; no new frog sounds added (0.0-5.0s)." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST.22.o_4902afd48541", "kind": "B", "clip": "TEST.22", "version": "TEST", "task": "addition", "video_id": "QDqBJwoQcNY_000080", "instruction": "Add the guinea fowl.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_d942d215_src.mp4", "ref": "media/i_d942d215_ref.jpg", "output": "media/o_4902afd48541.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The added guinea fowl from 0.0s-5.0s is a blurry, deformed texture patch rather than a coherent object, and no new sound for the guinea fowl was added to the audio track.", "evidence": { "source_target_visible": "In SOURCE, a blurry grey shape (inpainted artifact) is in the center from 0.0s to 5.0s, no clear guinea fowl.", "source_target_audible": "Soft chirping/peeping sounds are audible in the background around 1.8s-3.5s.", "source_other_sounds": "Background wind/ambience and faint chick chirping.", "edited_visual": "A spotted guinea fowl (and a small dark chick underneath) replaces the blurry grey patch from 0.0s to 5.0s, though the textures look somewhat blurry and static/warped compared to the reference.", "edited_audio": "The audio is identical to SOURCE, containing only the background ambience and faint chirps; no new distinct guinea fowl sound was added." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST.22.o_ecb7248d96e2", "kind": "B", "clip": "TEST.22", "version": "TEST", "task": "addition", "video_id": "QDqBJwoQcNY_000080", "instruction": "Add the guinea fowl.", "method_row": "cascade_avio_video_cavft_audio", "method": "Cascade (AVIO video + CAVEdit-FT audio)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_d942d215_src.mp4", "ref": "media/i_d942d215_ref.jpg", "output": "media/o_ecb7248d96e2.mp4", "votes": [ { "rater": "TEST-R3", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST-R1", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "From 0.0s to 5.0s, the added guinea fowl is a static, headless mass of feathers that warps slightly and does not animate to match the added calling sounds.", "evidence": { "source_target_visible": "In SOURCE, a blurry, inpainted grey speckled patch is visible where the bird should be from 0.0s to 5.0s.", "source_target_audible": "not audible", "source_other_sounds": "Outdoor ambience with faint background bird chirps and rustling.", "edited_visual": "From 0.0s to 5.0s, a spotted guinea fowl body is added over the central patch, though it is static and lacks a visible head, blending somewhat awkwardly into the straw.", "edited_audio": "From 0.0s to 5.0s, rhythmic clucking/calling sounds of a fowl are added over the original background ambience." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST2.16.o_eefebb06a11d", "kind": "B", "clip": "TEST2.16", "version": "TEST2", "task": "addition", "video_id": "R0vY9M9jSZ4", "instruction": "Add the vehicle engine.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_1ea07dae_src.mp4", "ref": "media/i_1ea07dae_ref.jpg", "output": "media/o_eefebb06a11d.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "Minor", "failure_notes": "none", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Continuous high-frequency hissing noise from 0.0s to 5.0s.", "edited_visual": "The vehicle engine from the reference image is added into the empty engine bay from 0.0s to 5.0s. Some minor warping/shimmering occurs around 4.5s-5.0s.", "edited_audio": "A loud, harsh static and engine idling/whirring sound is added throughout 0.0s to 5.0s, masking the original background hiss." }, "consistency_verifiable": "No", "consistency_label": "NA" }, "extra_outputs": [] }, { "id": "B.TEST2.18.o_9b8cf844323d", "kind": "B", "clip": "TEST2.18", "version": "TEST2", "task": "addition", "video_id": "IzJSE_bLQTc_000142", "instruction": "Add the birds.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_c35e55b2_src.mp4", "ref": "media/i_c35e55b2_ref.jpg", "output": "media/o_9b8cf844323d.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "No new bird sounds were added in the edited audio track (0.0-5.0s); the audio is unchanged from the source.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "bird chirps audible throughout (0.0-5.0s) in the background", "source_other_sounds": "outdoor ambience and background bird calls (0.0-5.0s)", "edited_visual": "A bird matching the reference image is added on the branch throughout 0.0-5.0s, moving its head slightly.", "edited_audio": "Audio track is identical to SOURCE (0.0-5.0s), no new sound was added." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "B.TEST2.18.o_4e807a446e06", "kind": "B", "clip": "TEST2.18", "version": "TEST2", "task": "addition", "video_id": "IzJSE_bLQTc_000142", "instruction": "Add the birds.", "method_row": "cascade_avio_video_cavft_audio", "method": "Cascade (AVIO video + CAVEdit-FT audio)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_c35e55b2_src.mp4", "ref": "media/i_c35e55b2_ref.jpg", "output": "media/o_4e807a446e06.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Uncertain", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "some strange sound" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "failure_notes": "none", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Background bird chirping and outdoor ambience throughout 0.0-5.0s.", "edited_visual": "A bird matching the reference image is added on the branch from 0.0-5.0s, moving its head slightly. The background leaves and branch are preserved.", "edited_audio": "Loud bird calls/chirps are added at 0.0-0.5s, 0.9-1.5s, 1.9-2.2s, and 2.8-3.8s, while the original background ambience remains." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "B.TEST2.20.o_d6deb7dbb2b8", "kind": "B", "clip": "TEST2.20", "version": "TEST2", "task": "addition", "video_id": "SPyyUMPdkCA_000018", "instruction": "Add the birds.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_bfe10fbf_src.mp4", "ref": "media/i_bfe10fbf_ref.jpg", "output": "media/o_d6deb7dbb2b8.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Uncertain", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "don't certain about the sound needed" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "The inserted birds suffer from severe warping and melting around their heads and beaks throughout 0.0-5.0s, and the entire background tarp and upper-right structure become blurry and distorted.", "evidence": { "source_target_visible": "Only bird legs/inpainting remnants are faintly visible on the tarp throughout 0.0-5.0s; the full birds are not visible.", "source_target_audible": "not audible", "source_other_sounds": "Outdoor ambience with wind, water sounds, and distant indistinct voices/chatter around 1.0-3.0s.", "edited_visual": "Two black-headed gulls are added on the tarp from 0.0-5.0s, matching the reference image, though their heads and beaks warp/melt into each other and the background is slightly degraded/warped.", "edited_audio": "Bird chirps/calls are added throughout 0.0-5.0s (e.g., at 0.0-1.0s, 3.0-4.8s) while the background ambience and distant voices are preserved." }, "consistency_verifiable": "Yes", "consistency_label": "Pass" }, "extra_outputs": [] }, { "id": "B.TEST2.20.o_b73a94743592", "kind": "B", "clip": "TEST2.20", "version": "TEST2", "task": "addition", "video_id": "SPyyUMPdkCA_000018", "instruction": "Add the birds.", "method_row": "ft_add_s2500_ff", "method": "InstructAV2AV FT", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_bfe10fbf_src.mp4", "ref": "media/i_bfe10fbf_ref.jpg", "output": "media/o_b73a94743592.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R2", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Major", "artifact_audio": "None", "failure_notes": "At 0.125s-5.0s, the two birds from the reference image collapse and morph into a single grotesquely deformed bird, and loud bird squawks at 1.8s-4.5s do not match coherent beak movements.", "evidence": { "source_target_visible": "Only bird legs/inpainting remnants are faintly visible on the tarp throughout 0.0-5.0s; the full birds are not visible.", "source_target_audible": "not audible", "source_other_sounds": "Outdoor ambience, water/wind noise, and distant human chatter/laughter (around 1.0-3.0s).", "edited_visual": "At 0.0s, two black-headed gulls matching the reference appear, but immediately at 0.125s they merge into a single deformed, mutated bird-like creature that persists until 5.0s. Background tarp and water remain mostly preserved.", "edited_audio": "Bird squawks/calls are added around 1.8-4.5s over the preserved background ambience and distant voices." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST2.21.o_3f22dde6cb7a", "kind": "B", "clip": "TEST2.21", "version": "TEST2", "task": "addition", "video_id": "faFZzzkn5Qs_000030", "instruction": "Add the birds.", "method_row": "ours_refwarm_ep019", "method": "AVIO (ours)", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_bdba6d51_src.mp4", "ref": "media/i_bdba6d51_ref.jpg", "output": "media/o_3f22dde6cb7a.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "NA", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R4", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R2", "V": "Fail", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Pass", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "The bird on the right vanishes by 0.7s while the background branches warp, and a severely deformed, stretched bird-like blob morphs on the left from 1.0s to 5.0s. The audio introduces heavy static noise throughout, and bird chirps occur around 0.8-1.5s when no intact bird is visible.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Outdoor wind and faint ambient noise throughout.", "edited_visual": "At 0.0-0.7s, the reference bird appears on the right, then dissolves while the branches shift. From 1.0-5.0s, a heavily distorted, stretched bird-like shape morphs into existence on the left side.", "edited_audio": "Bird chirping is added throughout the clip, accompanied by a loud broadband static hiss that alters the original wind ambience." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] }, { "id": "B.TEST2.21.o_d69034ce7f90", "kind": "B", "clip": "TEST2.21", "version": "TEST2", "task": "addition", "video_id": "faFZzzkn5Qs_000030", "instruction": "Add the birds.", "method_row": "ft_add_s2500_ff", "method": "InstructAV2AV FT", "why": "4 raters split 2-2 on core/V/A", "src": "media/i_bdba6d51_src.mp4", "ref": "media/i_bdba6d51_ref.jpg", "output": "media/o_d69034ce7f90.mp4", "votes": [ { "rater": "TEST2-R3", "V": "Fail", "A": "Fail", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": false, "comment": "" }, { "rater": "TEST2-R1", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "Minor", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R4", "V": "Pass", "A": "Pass", "P_visual": "Pass", "P_audio": "Pass", "artifact_visual": "None", "artifact_audio": "None", "recon_plausible": "", "core": true, "comment": "" }, { "rater": "TEST2-R2", "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "NA", "artifact_visual": "None", "artifact_audio": "Minor", "recon_plausible": "", "core": false, "comment": "" } ], "gemini": { "V": "Fail", "A": "Fail", "P_visual": "Fail", "P_audio": "Fail", "artifact_visual": "Major", "artifact_audio": "Major", "failure_notes": "At 0.25s, the reference bird disappears and is replaced by an unrelated large black bird while background branches warp. Instead of bird calls, distorted speech-like noises play at 0.8-1.2s, 1.8-2.3s, and 3.8-4.2s.", "evidence": { "source_target_visible": "not visible", "source_target_audible": "not audible", "source_other_sounds": "Outdoor wind ambience and faint background chirping.", "edited_visual": "From 0.0-0.25s, the small light-colored bird from the reference image appears on the right. At 0.25s, it vanishes and a completely different, much larger black bird appears on the left and stays until 5.0s, while background branches shift.", "edited_audio": "Muffled, distorted, speech-like garbled sounds are added around 0.8-1.2s, 1.8-2.3s, and 3.8-4.2s alongside wind noise." }, "consistency_verifiable": "Yes", "consistency_label": "Fail" }, "extra_outputs": [] } ] }
{
"tag": "2026-11-14_live_v2",
"items_sha256": "711b321fc2bd5fc4",
"page_version": "2026-11-14_live_v2-711b321f",
"cards": [
{
"id": "A.TEST.1.o_e569f315dd1e",
"kind": "A",
"clip": "TEST.1",
"version": "TEST",
"task": "removal",
"video_id": "9rXAmXuAOw8_000000",
"instruction": "Remove the crow.",
"method_row": "GT",
"method": "Pseudo-target (GT)",
"why": "pseudo-target: A=0.25, V=0.75",
"src": "media/i_6537882e_src.mp4",
"ref": null,
"output": "media/o_e569f315dd1e.mp4",
"votes": [
{
"rater": "TEST-R3",
"V": "Pass",
"A": "Fail",
"P_visual": "Pass",
"P_audio": "Fail",
"artifact_visual": "Major",
"artifact_audio": "None",
"recon_plausible": "Fail",
"core": false,
"comment": ""
},
{
"rater": "TEST-R4",
"V": "Pass",
"A": "Fail",
"P_visual": "Pass",
"P_audio": "NA",
"artifact_visual": "Major",
"artifact_audio": "Minor",
"recon_plausible": "Pass",
"core": false,
"comment": ""
},
{
"rater": "TEST-R2",
"V": "Pass",
"A": "Pass",
"P_visual": "Pass",
"P_audio": "Pass",
"artifact_visual": "None",
"artifact_audio": "Minor",
"recon_plausible": "Pass",
"core": true,
"comment": ""
},
{
"rater": "TEST-R1",
"V": "Fail",
"A": "Fail",
"P_visual": "Fail",
"P_audio": "Fail",
"artifact_visual": "Major",
"artifact_audio": "Minor",
"recon_plausible": "Fail",
"core": false,
"comment": ""
}
],
"gemini": {
"V": "Pass",
"A": "Fail",
"P_visual": "Fail",
"P_audio": "Pass",
"artifact_visual": "Major",
"artifact_audio": "Major",
"failure_notes": "The crow's distorted vocalizations remain audible from 0.0s to 3.9s despite the crow being removed, and large brown vertical smears appear beside the post (0.0s-4.3s) and above it (4.3s-5.0s).",
"evidence": {
"source_target_visible": "A black crow is perched on a brown wooden post throughout the entire clip (0.0s to 5.0s).",
"source_target_audible": "The crow makes distorted, robotic-sounding vocalizations/talking sounds synchronized with its beak opening at 0.0s-0.7s, 0.9s-2.3s, 2.4s-3.9s, and 4.7s-5.0s.",
"source_other_sounds": "Faint background synth/electronic music throughout (0.0s to 5.0s).",
"edited_visual": "The crow is removed, but brown blurry vertical smears appear next to the post (0.0s to 4.3s) and directly above the post (4.3s to 5.0s).",
"edited_audio": "The crow's vocalizations are still audible and heavily distorted/muffled at 0.0s-0.7s, 0.9s-2.3s, and 2.4s-3.9s. Background synth remains."
},
"consistency_verifiable": "Yes",
"consistency_label": "Fail"
},
"extra_outputs": [
{
"method": "AVIO (ours)",
"row": "ours_refwarm_ep019",
"file": "media/o_486d854b14c0.mp4",
"gemini": {
"V": "Fail",
"A": "Fail",
"P_visual": "Pass",
"P_audio": "NA",
"artifact_visual": "Minor",
"artifact_audio": "None",
"failure_notes": "The crow's legs/claws remain visible on top of the wooden post from 0.0s to 5.0s, and the crow's vocalizations were not removed from the audio track (0.0s-5.0s).",