| { |
| "model": "gemini-3.8-flash", |
| "experiment": "shaina_fresh_test", |
| "tasks": { |
| "t1": { |
| "scores": { |
| "n": 10, |
| "accuracy": 0.4, |
| "ci95": [ |
| 0.1682, |
| 0.6873 |
| ], |
| "balanced_accuracy": 0.4167, |
| "cohen_kappa": 0.2405 |
| }, |
| "items": [ |
| { |
| "id": "t1_RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352", |
| "clips": [ |
| "RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "neither", |
| "both", |
| "the video", |
| "the audio" |
| ], |
| "gt_answer": "the audio", |
| "pred_answer": "neither", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"neither\"}\n```" |
| }, |
| { |
| "id": "t1_RVFA__seed_vc__77f__main__id01296_B7gb-0xbMn0_00024__id06068", |
| "clips": [ |
| "RVFA__seed_vc__77f__main__id01296_B7gb-0xbMn0_00024__id06068.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "the audio", |
| "both", |
| "the video", |
| "neither" |
| ], |
| "gt_answer": "the audio", |
| "pred_answer": "the audio", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"the audio\"}\n```" |
| }, |
| { |
| "id": "t1_RVRA__real__77f__vocoded__id05623_HbnUHzNq8XI_00003__id00739", |
| "clips": [ |
| "RVRA__real__77f__vocoded__id05623_HbnUHzNq8XI_00003__id00739.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "neither", |
| "both", |
| "the audio", |
| "the video" |
| ], |
| "gt_answer": "neither", |
| "pred_answer": "neither", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"neither\"}\n```" |
| }, |
| { |
| "id": "t1_RVRA__real__153f__vocoded__id02959_hp-EccEnxCQ_00026__id08803", |
| "clips": [ |
| "RVRA__real__153f__vocoded__id02959_hp-EccEnxCQ_00026__id08803.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "the video", |
| "both", |
| "neither", |
| "the audio" |
| ], |
| "gt_answer": "neither", |
| "pred_answer": "neither", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"neither\"}\n```" |
| }, |
| { |
| "id": "t1_FVFA__s2v__153f__main__id03084_BQIuTs2QZUU_00021__id08748", |
| "clips": [ |
| "FVFA__s2v__153f__main__id03084_BQIuTs2QZUU_00021__id08748.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "the video", |
| "both", |
| "neither", |
| "the audio" |
| ], |
| "gt_answer": "both", |
| "pred_answer": "the video", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"the video\"}\n```" |
| }, |
| { |
| "id": "t1_FVFA__animate__77f__main__id05162_NpL77H2Shgk_00043__id03606", |
| "clips": [ |
| "FVFA__animate__77f__main__id05162_NpL77H2Shgk_00043__id03606.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "the audio", |
| "neither", |
| "both", |
| "the video" |
| ], |
| "gt_answer": "both", |
| "pred_answer": "neither", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"neither\"}\n```" |
| }, |
| { |
| "id": "t1_FVFA__s2v__77f__main__id06390_17l4_o3k29c_00007__id07683", |
| "clips": [ |
| "FVFA__s2v__77f__main__id06390_17l4_o3k29c_00007__id07683.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "neither", |
| "both", |
| "the audio", |
| "the video" |
| ], |
| "gt_answer": "both", |
| "pred_answer": "neither", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"neither\"}\n```" |
| }, |
| { |
| "id": "t1_FVFA__facefusion__153f__main__id07786_bFZAH657cFs_00051__id07683", |
| "clips": [ |
| "FVFA__facefusion__153f__main__id07786_bFZAH657cFs_00051__id07683.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "neither", |
| "both", |
| "the audio", |
| "the video" |
| ], |
| "gt_answer": "both", |
| "pred_answer": "the audio", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"the audio\"}\n```" |
| }, |
| { |
| "id": "t1_FVFA__facefusion__77f__main__id05670_8I_YYqRsihM_00041__id04900", |
| "clips": [ |
| "FVFA__facefusion__77f__main__id05670_8I_YYqRsihM_00041__id04900.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "the video", |
| "both", |
| "neither", |
| "the audio" |
| ], |
| "gt_answer": "both", |
| "pred_answer": "neither", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"neither\"}\n```" |
| }, |
| { |
| "id": "t1_RVFA__seed_vc__77f__main__id01858_HBSDURviflg_00015__id01156", |
| "clips": [ |
| "RVFA__seed_vc__77f__main__id01858_HBSDURviflg_00015__id01156.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.", |
| "options": [ |
| "neither", |
| "the audio", |
| "both", |
| "the video" |
| ], |
| "gt_answer": "the audio", |
| "pred_answer": "the audio", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"the audio\"}\n```" |
| } |
| ] |
| }, |
| "t2": { |
| "scores": { |
| "n": 10, |
| "accuracy": 0.2, |
| "ci95": [ |
| 0.0567, |
| 0.5098 |
| ], |
| "balanced_accuracy": 0.25, |
| "cohen_kappa": 0.0 |
| }, |
| "items": [ |
| { |
| "id": "t2_FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444", |
| "clips": [ |
| "FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "full-body swap", |
| "fully generated", |
| "real", |
| "face swap" |
| ], |
| "gt_answer": "fully generated", |
| "pred_answer": "real", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"real\"}\n```" |
| }, |
| { |
| "id": "t2_FVRA__facefusion__77f__main__id08940_B5cvspWteMI_00009__id03946", |
| "clips": [ |
| "FVRA__facefusion__77f__main__id08940_B5cvspWteMI_00009__id03946.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "face swap", |
| "full-body swap", |
| "fully generated", |
| "real" |
| ], |
| "gt_answer": "face swap", |
| "pred_answer": "face swap", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"face swap\"}\n```" |
| }, |
| { |
| "id": "t2_RVRA__real__77f__main__id05444_QIxNYNGaVg0_00040__id08406", |
| "clips": [ |
| "RVRA__real__77f__main__id05444_QIxNYNGaVg0_00040__id08406.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "full-body swap", |
| "face swap", |
| "real", |
| "fully generated" |
| ], |
| "gt_answer": "real", |
| "pred_answer": "face swap", |
| "correct": false, |
| "raw": "{\"answer\": \"face swap\"}" |
| }, |
| { |
| "id": "t2_FVRA__animate__77f__main__id07054_gLIAqAQ40LQ_00053__id08610", |
| "clips": [ |
| "FVRA__animate__77f__main__id07054_gLIAqAQ40LQ_00053__id08610.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "fully generated", |
| "real", |
| "full-body swap", |
| "face swap" |
| ], |
| "gt_answer": "full-body swap", |
| "pred_answer": "face swap", |
| "correct": false, |
| "raw": "{\"answer\": \"face swap\"}" |
| }, |
| { |
| "id": "t2_RVRA__real__77f__main__id00761_C_ZBK3E8bJk_00027__id07616", |
| "clips": [ |
| "RVRA__real__77f__main__id00761_C_ZBK3E8bJk_00027__id07616.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "real", |
| "fully generated", |
| "face swap", |
| "full-body swap" |
| ], |
| "gt_answer": "real", |
| "pred_answer": "real", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"real\"}\n```" |
| }, |
| { |
| "id": "t2_FVRA__facefusion__77f__main__id06480_YQD4AsloKcY_00076__id01534", |
| "clips": [ |
| "FVRA__facefusion__77f__main__id06480_YQD4AsloKcY_00076__id01534.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "real", |
| "face swap", |
| "fully generated", |
| "full-body swap" |
| ], |
| "gt_answer": "face swap", |
| "pred_answer": "real", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"real\"}\n```" |
| }, |
| { |
| "id": "t2_FVRA__s2v__77f__main__id03854_yfQGoMSBDx0_00151__id00509", |
| "clips": [ |
| "FVRA__s2v__77f__main__id03854_yfQGoMSBDx0_00151__id00509.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "face swap", |
| "full-body swap", |
| "real", |
| "fully generated" |
| ], |
| "gt_answer": "fully generated", |
| "pred_answer": "face swap", |
| "correct": false, |
| "raw": "{\"answer\": \"face swap\"}" |
| }, |
| { |
| "id": "t2_FVRA__animate__77f__main__id02959_h9Qwwv-DApg_00016__id03078", |
| "clips": [ |
| "FVRA__animate__77f__main__id02959_h9Qwwv-DApg_00016__id03078.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "face swap", |
| "fully generated", |
| "full-body swap", |
| "real" |
| ], |
| "gt_answer": "full-body swap", |
| "pred_answer": "face swap", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"face swap\"}\n```" |
| }, |
| { |
| "id": "t2_FVRA__s2v__77f__main__id02359_hoPN5dCzSC8_00093__id06987", |
| "clips": [ |
| "FVRA__s2v__77f__main__id02359_hoPN5dCzSC8_00093__id06987.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "fully generated", |
| "full-body swap", |
| "real", |
| "face swap" |
| ], |
| "gt_answer": "fully generated", |
| "pred_answer": "real", |
| "correct": false, |
| "raw": "{\"answer\": \"real\"}" |
| }, |
| { |
| "id": "t2_FVRA__animate__77f__main__id03681_Szvjxba6BTg_00066__id01664", |
| "clips": [ |
| "FVRA__animate__77f__main__id03681_Szvjxba6BTg_00066__id01664.mp4" |
| ], |
| "part": null, |
| "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.", |
| "options": [ |
| "real", |
| "full-body swap", |
| "face swap", |
| "fully generated" |
| ], |
| "gt_answer": "full-body swap", |
| "pred_answer": "real", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"real\"}\n```" |
| } |
| ] |
| }, |
| "t3": { |
| "scores": { |
| "n": 10, |
| "accuracy": 0.5, |
| "ci95": [ |
| 0.2366, |
| 0.7634 |
| ], |
| "balanced_accuracy": 0.3542, |
| "cohen_kappa": 0.2647 |
| }, |
| "items": [ |
| { |
| "id": "t3_voice_A_RVRA__real__77f__vocoded__id00948_PCy1KCIsNxo_00024__id04278", |
| "clips": [ |
| "RVRA__real__77f__vocoded__id00948_PCy1KCIsNxo_00024__id04278.mp4", |
| "RVRA__real__77f__main__id00948_z-P1KSBdNU8_00128__id08581.mp4" |
| ], |
| "part": "voice", |
| "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.", |
| "options": [ |
| "fake of a different person", |
| "fake of this person", |
| "real, same person", |
| "real, different person" |
| ], |
| "gt_answer": "real, same person", |
| "pred_answer": "real, same person", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"real, same person\"}\n```" |
| }, |
| { |
| "id": "t3_face_A_RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564", |
| "clips": [ |
| "RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564.mp4", |
| "RVRA__real__77f__main__id05877_6QUcq_6B-LU_00011__id07768.mp4" |
| ], |
| "part": "face", |
| "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.", |
| "options": [ |
| "real, different person", |
| "real, same person", |
| "fake of this person", |
| "fake of a different person" |
| ], |
| "gt_answer": "real, same person", |
| "pred_answer": "real, same person", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"real, same person\"}\n```" |
| }, |
| { |
| "id": "t3_voice_B_RVRA__real__153f__vocoded__id00885_xiPMwAYISG0_00135__id05623", |
| "clips": [ |
| "RVRA__real__153f__vocoded__id00885_xiPMwAYISG0_00135__id05623.mp4", |
| "RVRA__real__153f__main__id05623_UXUHOpRirtg_00079__id03123.mp4" |
| ], |
| "part": "voice", |
| "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.", |
| "options": [ |
| "real, same person", |
| "fake of this person", |
| "real, different person", |
| "fake of a different person" |
| ], |
| "gt_answer": "real, different person", |
| "pred_answer": "real, same person", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"real, same person\"}\n```" |
| }, |
| { |
| "id": "t3_face_B_RVRA__real__153f__main__id06713_5qTLKWiq73k_00074__id05042", |
| "clips": [ |
| "RVRA__real__153f__main__id06713_5qTLKWiq73k_00074__id05042.mp4", |
| "RVRA__real__153f__main__id05042_K1HMS1K0kWI_00010__id00144.mp4" |
| ], |
| "part": "face", |
| "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.", |
| "options": [ |
| "real, same person", |
| "fake of a different person", |
| "fake of this person", |
| "real, different person" |
| ], |
| "gt_answer": "real, different person", |
| "pred_answer": "real, different person", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"real, different person\"}\n```" |
| }, |
| { |
| "id": "t3_voice_A_RVFA__seed_vc__153f__main__id04049_JCDeI7UwWnE_00080__id08096", |
| "clips": [ |
| "RVFA__seed_vc__153f__main__id04049_JCDeI7UwWnE_00080__id08096.mp4", |
| "RVRA__real__153f__main__id04049_Flu_4soH0mo_00003__id04528.mp4" |
| ], |
| "part": "voice", |
| "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.", |
| "options": [ |
| "fake of a different person", |
| "real, same person", |
| "real, different person", |
| "fake of this person" |
| ], |
| "gt_answer": "fake of a different person", |
| "pred_answer": "real, different person", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"real, different person\"}\n```" |
| }, |
| { |
| "id": "t3_voice_B_RVFA__seed_vc__77f__main__id07574_DbcyMh1l_Ng_00043__id06429", |
| "clips": [ |
| "RVFA__seed_vc__77f__main__id07574_DbcyMh1l_Ng_00043__id06429.mp4", |
| "RVRA__real__77f__main__id06429_0aTJ9yNMAx0_00002__id06427.mp4" |
| ], |
| "part": "voice", |
| "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.", |
| "options": [ |
| "fake of this person", |
| "real, different person", |
| "real, same person", |
| "fake of a different person" |
| ], |
| "gt_answer": "fake of this person", |
| "pred_answer": "fake of a different person", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"fake of a different person\"}\n```" |
| }, |
| { |
| "id": "t3_face_A_RVRA__real__153f__main__id00885_yMWJySOEuTI_00138__id00739", |
| "clips": [ |
| "RVRA__real__153f__main__id00885_yMWJySOEuTI_00138__id00739.mp4", |
| "RVRA__real__153f__main__id00885_xiPMwAYISG0_00135__id05623.mp4" |
| ], |
| "part": "face", |
| "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.", |
| "options": [ |
| "real, different person", |
| "fake of this person", |
| "real, same person", |
| "fake of a different person" |
| ], |
| "gt_answer": "real, same person", |
| "pred_answer": "real, different person", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"real, different person\"}\n```" |
| }, |
| { |
| "id": "t3_face_B_RVRA__real__77f__main__id08808_vQV6tYM_-3M_00053__id07697", |
| "clips": [ |
| "RVRA__real__77f__main__id08808_vQV6tYM_-3M_00053__id07697.mp4", |
| "RVRA__real__77f__main__id07697_diNcpyVMCk8_00047__id07630.mp4" |
| ], |
| "part": "face", |
| "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.", |
| "options": [ |
| "real, same person", |
| "fake of this person", |
| "fake of a different person", |
| "real, different person" |
| ], |
| "gt_answer": "real, different person", |
| "pred_answer": "real, different person", |
| "correct": true, |
| "raw": "```json\n{\"answer\": \"real, different person\"}\n```" |
| }, |
| { |
| "id": "t3_voice_B_RVFA__seed_vc__153f__main__id02359_QiCI0mefCx4_00030__id03078", |
| "clips": [ |
| "RVFA__seed_vc__153f__main__id02359_QiCI0mefCx4_00030__id03078.mp4", |
| "RVRA__real__153f__main__id03078_4XN_hNJCe9E_00031__id06987.mp4" |
| ], |
| "part": "voice", |
| "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.", |
| "options": [ |
| "real, same person", |
| "real, different person", |
| "fake of a different person", |
| "fake of this person" |
| ], |
| "gt_answer": "fake of this person", |
| "pred_answer": "real, different person", |
| "correct": false, |
| "raw": "```json\n{\"answer\": \"real, different person\"}\n```" |
| }, |
| { |
| "id": "t3_voice_A_RVRA__real__153f__vocoded__id01124_7Y-51ooBIWQ_00040__id09218", |
| "clips": [ |
| "RVRA__real__153f__vocoded__id01124_7Y-51ooBIWQ_00040__id09218.mp4", |
| "RVRA__real__153f__main__id01124_2hg7q9O8xLo_00015__id08748.mp4" |
| ], |
| "part": "voice", |
| "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.", |
| "options": [ |
| "real, same person", |
| "fake of this person", |
| "real, different person", |
| "fake of a different person" |
| ], |
| "gt_answer": "real, same person", |
| "pred_answer": "real, same person", |
| "correct": true, |
| "raw": "{\"answer\": \"real, same person\"}" |
| } |
| ] |
| } |
| } |
| } |
