Spaces:

Deep-Fake
/

mozaic-sample-explorer

Running

App Files Files Community

shainaraza commited on about 17 hours ago

Commit

264e8fb

verified ·

1 Parent(s): b454846

Upload folder using huggingface_hub

Browse files

Files changed (10) hide show

.gitattributes +6 -0
README.md +15 -4
clips/FVRA__animate__77f__main__id07839__QTWWQBdhWY_00029__id07683.mp4 +3 -0
clips/FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444.mp4 +3 -0
clips/RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352.mp4 +3 -0
clips/RVRA__real__77f__main__id05877_6QUcq_6B-LU_00011__id07768.mp4 +3 -0
clips/RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564.mp4 +3 -0
clips/RVRA__real__77f__main__id07839_gJgj1DxOfKA_00045__id07786.mp4 +3 -0
index.html +170 -17
samples.json +132 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+clips/FVRA__animate__77f__main__id07839__QTWWQBdhWY_00029__id07683.mp4 filter=lfs diff=lfs merge=lfs -text
+clips/FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444.mp4 filter=lfs diff=lfs merge=lfs -text
+clips/RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352.mp4 filter=lfs diff=lfs merge=lfs -text
+clips/RVRA__real__77f__main__id05877_6QUcq_6B-LU_00011__id07768.mp4 filter=lfs diff=lfs merge=lfs -text
+clips/RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564.mp4 filter=lfs diff=lfs merge=lfs -text
+clips/RVRA__real__77f__main__id07839_gJgj1DxOfKA_00045__id07786.mp4 filter=lfs diff=lfs merge=lfs -text

README.md CHANGED Viewed

@@ -1,10 +1,21 @@
 ---
-title: Mozaic Sample Explorer
-emoji: 💻
 colorFrom: indigo
-colorTo: pink
 sdk: static
 pinned: false
 ---
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

 ---
+title: MOZAIC Sample Explorer
+emoji: 🎭
 colorFrom: indigo
+colorTo: gray
 sdk: static
+app_file: index.html
 pinned: false
+short_description: Gemini zero-shot answers on MOZAIC deepfake samples
 ---
+# MOZAIC sample explorer
+A small pilot from MOZAIC, an audio-visual deepfake benchmark built on VoxCeleb2
+(code: https://github.com/VectorInstitute/mozaic). It shows Gemini's zero-shot answers on
+example questions from each task (T1 spot the fake, T2 manipulation type, T3 same person,
+T3.5 real same person), with the clips, the correct answers and the pilot-run scores.
+Clips are copied from the MOZAIC release unchanged (video stream untouched; audio
+re-encoded to AAC so browsers can play it). Fake clips are synthetic media made for
+detection research. The pilot is far too small for conclusions.

clips/FVRA__animate__77f__main__id07839__QTWWQBdhWY_00029__id07683.mp4 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:fb541e2410de06410bcd71789b65c745ba1b36abc98305252bb25630c731f405
+size 107958

clips/FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444.mp4 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:11a10bd416821b8601bfaebfc9fec4d7c1deb0e480874a4f5470605b111a0baf
+size 119994

clips/RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352.mp4 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:42ce21a3fa99eaa074a33dab7c90129065c8466a01a5ba46f1fcc019a37dd7bb
+size 260939

clips/RVRA__real__77f__main__id05877_6QUcq_6B-LU_00011__id07768.mp4 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:175f6a038574049752362c47d6f9cac10be0eb0633155529e0d2b44453a92f3a
+size 128751

clips/RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564.mp4 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d20f676dd5ce8f1339b5687fdcbf0d1dff4b9f62e35ea0820c3b5d1ad984c3da
+size 135135

clips/RVRA__real__77f__main__id07839_gJgj1DxOfKA_00045__id07786.mp4 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:51819f6865267482967c0b79c5e42203166c3b24c138f881961093c9674fb25f
+size 135567

index.html CHANGED Viewed

@@ -1,19 +1,172 @@
 <!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
 </html>

 <!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>MOZAIC Sample Explorer</title>
+<style>
+  :root {
+    --bg: #f7f7f8; --card: #ffffff; --text: #1b1c1f; --muted: #62656e; --line: #e2e3e7;
+    --accent: #4f46e5; --ok: #15803d; --bad: #b91c1c; --code: #f0f0f3;
+  }
+  @media (prefers-color-scheme: dark) {
+    :root:not([data-theme="light"]) {
+      --bg: #111214; --card: #1a1b1f; --text: #ececf0; --muted: #a0a3ad; --line: #2c2e34;
+      --accent: #8b85ff; --ok: #4ade80; --bad: #f87171; --code: #24262c;
+    }
+  }
+  :root[data-theme="dark"] {
+    --bg: #111214; --card: #1a1b1f; --text: #ececf0; --muted: #a0a3ad; --line: #2c2e34;
+    --accent: #8b85ff; --ok: #4ade80; --bad: #f87171; --code: #24262c;
+  }
+  * { box-sizing: border-box; }
+  body { margin: 0; background: var(--bg); color: var(--text);
+         font: 15px/1.55 system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; }
+  main { max-width: 1040px; margin: 0 auto; padding: 28px 16px 48px; }
+  h1 { font-size: 1.7rem; margin: 0 0 6px; }
+  .lede { color: var(--muted); margin: 0 0 4px; }
+  .fine { color: var(--muted); font-size: .82rem; margin: 0 0 20px; }
+  code { background: var(--code); padding: 1px 6px; border-radius: 5px; font-size: .88em; }
+  .tabs { display: flex; gap: 4px; border-bottom: 1px solid var(--line); margin-bottom: 18px; overflow-x: auto; }
+  .tabs button { background: none; border: 0; border-bottom: 2px solid transparent; color: var(--muted);
+                 padding: 9px 12px; font: inherit; cursor: pointer; white-space: nowrap; }
+  .tabs button[aria-selected="true"] { color: var(--accent); border-bottom-color: var(--accent); font-weight: 600; }
+  .task-q { font-weight: 600; margin: 0; }
+  .task-in { color: var(--muted); margin: 2px 0 14px; }
+  .note { color: var(--muted); font-size: .82rem; margin: 0 0 6px; }
+  .stats { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 10px; margin-bottom: 18px; }
+  .stat { background: var(--card); border: 1px solid var(--line); border-radius: 10px; padding: 10px 12px; }
+  .stat b { display: block; font-size: 1.45rem; font-variant-numeric: tabular-nums; }
+  .stat span { color: var(--muted); font-size: .82rem; }
+  .nav { display: flex; gap: 8px; align-items: center; margin-bottom: 14px; }
+  .nav select { flex: 1; min-width: 0; padding: 8px 10px; border-radius: 8px; border: 1px solid var(--line);
+                background: var(--card); color: var(--text); font: inherit; }
+  .nav button { padding: 8px 14px; border-radius: 8px; border: 1px solid var(--line); background: var(--card);
+                color: var(--text); font: inherit; cursor: pointer; }
+  .clips { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 12px; margin-bottom: 16px; }
+  figure { margin: 0; background: var(--card); border: 1px solid var(--line); border-radius: 10px; padding: 10px; }
+  figure video { width: 100%; max-height: 320px; background: #000; border-radius: 6px; display: block; }
+  figcaption { color: var(--muted); font-size: .84rem; margin-top: 6px; }
+  .grid { display: grid; grid-template-columns: 3fr 2fr; gap: 16px; }
+  .card { background: var(--card); border: 1px solid var(--line); border-radius: 10px; padding: 14px 16px; }
+  .card h3 { font-size: .8rem; text-transform: uppercase; letter-spacing: .04em; color: var(--muted); margin: 0 0 6px; }
+  blockquote { margin: 0 0 12px; padding-left: 12px; border-left: 3px solid var(--line); }
+  table { border-collapse: collapse; width: 100%; margin-top: 10px; }
+  th, td { text-align: left; padding: 7px 8px; border-bottom: 1px solid var(--line); }
+  th { color: var(--muted); font-weight: 500; font-size: .85rem; }
+  .ok { color: var(--ok); font-weight: 600; } .bad { color: var(--bad); font-weight: 600; }
+  details { margin-top: 14px; } summary { cursor: pointer; color: var(--muted); }
+  pre { background: var(--code); padding: 10px; border-radius: 8px; overflow-x: auto; font-size: .85rem; }
+  ul { margin: 0; padding-left: 18px; }
+  @media (max-width: 680px) {
+    .stats { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+    .grid { grid-template-columns: 1fr; }
+  }
+</style>
+</head>
+<body>
+<main>
+  <h1>MOZAIC sample explorer</h1>
+  <p class="lede">MOZAIC is an audio-visual deepfake benchmark built on VoxCeleb2. Every clip pairs a real or fake face
+    with a real or fake voice. This page shows a small pilot: <code id="model">…</code> answering MOZAIC questions
+    zero-shot, with example clips per task.</p>
+  <p class="fine">Research demo. Fake clips are synthetic media made for detection research; they do not show anything
+    the people in them said or did. A pilot this small is far too few for conclusions.</p>
+  <div class="tabs" role="tablist" id="tabs"></div>
+  <section id="panel"></section>
+</main>
+<script>
+const TASKS = {
+  t1:   ["T1 · Spot the fake", "Which part is fake: the video, the audio, both, or neither?",
+         "The model saw 32 frames and heard the audio."],
+  t2:   ["T2 · Manipulation type", "How was the video made?", "The model saw 32 frames only, no sound."],
+  t3:   ["T3 · Same person", "Is the face (or voice) in clip 1 the same person as in clip 2?",
+         "Face questions: frames of both clips, no sound. Voice questions: sound of both clips, no frames."],
+  t3_5: ["T3.5 · Real same person", "Is clip 1 a real recording of clip 2's person?",
+         "Face questions: frames of both clips, no sound. Voice questions: sound of both clips, no frames."],
+};
+const LABELS = { RVRA: "real video · real audio", RVFA: "real video · fake audio",
+                 FVRA: "fake video · real audio", FVFA: "fake video · fake audio" };
+const METHODS = { real: "camera (real)", facefusion: "FaceFusion face swap", s2v: "Wan2.2-S2V (audio-driven)",
+                  animate: "Wan2.2-Animate (video-driven)", seed_vc: "Seed-VC voice conversion" };
+const esc = s => String(s ?? "").replace(/[&<>"']/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
+function clipDesc(clip) {
+  const [label, method, frames, arm] = clip.split("__");
+  const parts = [LABELS[label] || label, METHODS[method] || method, frames.replace("f", " frames")];
+  if (arm !== "main") parts.push(`${arm} audio`);
+  return parts.join(" · ");
+}
+let DATA, current = { task: null, i: 0 };
+function renderTabs() {
+  const tabs = document.getElementById("tabs");
+  tabs.innerHTML = "";
+  for (const [key, [title]] of Object.entries(TASKS)) {
+    if (!DATA.tasks[key] || !DATA.tasks[key].items.length) continue;
+    const b = document.createElement("button");
+    b.role = "tab"; b.textContent = title;
+    b.setAttribute("aria-selected", key === current.task);
+    b.onclick = () => { current = { task: key, i: 0 }; renderTabs(); renderPanel(); };
+    tabs.appendChild(b);
+  }
+}
+function renderPanel() {
+  const key = current.task, task = DATA.tasks[key], s = task.scores, it = task.items[current.i];
+  const [, question, inputs] = TASKS[key];
+  const right = Math.round(s.accuracy * s.n);
+  const opts = task.items.map((x, i) =>
+    `<option value="${i}" ${i === current.i ? "selected" : ""}>${x.correct ? "✓" : "✗"}  #${i + 1}${x.part ? ` [${x.part}]` : ""}  ${esc(clipDesc(x.clips[0]))}</option>`).join("");
+  const clips = it.clips.map((c, n) => `
+    <figure><video controls preload="metadata" playsinline src="clips/${encodeURIComponent(c)}"></video>
+      <figcaption>Clip ${n + 1}${it.clips.length > 1 && n === 1 ? " (real reference)" : ""}</figcaption></figure>`).join("");
+  document.getElementById("panel").innerHTML = `
+    <p class="task-q">${esc(question)}</p>
+    <p class="task-in">${esc(inputs)}</p>
+    <p class="note">Scores over all ${s.n} questions Gemini answered in this pilot run; ${task.items.length} shown below.</p>
+    <div class="stats">
+      <div class="stat"><b>${right}/${s.n}</b><span>correct</span></div>
+      <div class="stat"><b>${s.accuracy.toFixed(2)}</b><span>accuracy (95% CI ${s.ci95[0].toFixed(2)}–${s.ci95[1].toFixed(2)})</span></div>
+      <div class="stat"><b>${s.balanced_accuracy.toFixed(2)}</b><span>balanced accuracy</span></div>
+      <div class="stat"><b>${s.cohen_kappa.toFixed(2)}</b><span>Cohen's κ</span></div>
+    </div>
+    <div class="nav">
+      <button id="prev" aria-label="Previous sample">◀</button>
+      <select id="pick" aria-label="Sample">${opts}</select>
+      <button id="next" aria-label="Next sample">▶</button>
+    </div>
+    <div class="clips">${clips}</div>
+    <div class="grid">
+      <div class="card">
+        <h3>Question</h3>
+        <blockquote>${esc(it.question)}</blockquote>
+        <div>Options, in the order shown: ${it.options.map(o => `<code>${esc(o)}</code>`).join(" · ")}</div>
+        <table>
+          <tr><th>Correct answer</th><th>Gemini answered</th><th></th></tr>
+          <tr><td><b>${esc(it.gt_answer)}</b></td><td><b>${esc(it.pred_answer ?? "— unreadable —")}</b></td>
+              <td class="${it.correct ? "ok" : "bad"}">${it.correct ? "✓ Correct" : "✗ Wrong"}</td></tr>
+        </table>
+      </div>
+      <div class="card">
+        <h3>About the clip(s), hidden from the model</h3>
+        <ul>${it.clips.map((c, n) => `<li><b>Clip ${n + 1}:</b> ${esc(clipDesc(c))}</li>`).join("")}</ul>
+        <details><summary>Gemini's raw reply</summary><pre>${esc((it.raw || "").trim())}</pre></details>
+      </div>
+    </div>`;
+  const go = d => { current.i = (current.i + d + task.items.length) % task.items.length; renderPanel(); };
+  document.getElementById("prev").onclick = () => go(-1);
+  document.getElementById("next").onclick = () => go(1);
+  document.getElementById("pick").onchange = e => { current.i = +e.target.value; renderPanel(); };
+}
+fetch("samples.json").then(r => r.json()).then(d => {
+  DATA = d;
+  document.getElementById("model").textContent = d.model;
+  current.task = Object.keys(TASKS).find(k => d.tasks[k] && d.tasks[k].items.length);
+  renderTabs(); renderPanel();
+}).catch(e => { document.getElementById("panel").textContent = "Could not load samples.json: " + e; });
+</script>
+</body>
 </html>

samples.json ADDED Viewed

	@@ -0,0 +1,132 @@

+{
+ "model": "gemini-3.8-flash",
+ "experiment": "shaina_fresh_test",
+ "tasks": {
+  "t1": {
+   "scores": {
+    "n": 10,
+    "accuracy": 0.2,
+    "ci95": [
+     0.0567,
+     0.5098
+    ],
+    "balanced_accuracy": 0.1875,
+    "cohen_kappa": -0.0256
+   },
+   "items": [
+    {
+     "id": "t1_RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352",
+     "clips": [
+      "RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352.mp4"
+     ],
+     "part": null,
+     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
+     "options": [
+      "neither",
+      "both",
+      "the video",
+      "the audio"
+     ],
+     "gt_answer": "the audio",
+     "pred_answer": "neither",
+     "correct": false,
+     "raw": "```json\n{\"answer\": \"neither\"}\n```"
+    }
+   ]
+  },
+  "t2": {
+   "scores": {
+    "n": 10,
+    "accuracy": 0.5,
+    "ci95": [
+     0.2366,
+     0.7634
+    ],
+    "balanced_accuracy": 0.5,
+    "cohen_kappa": 0.3333
+   },
+   "items": [
+    {
+     "id": "t2_FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444",
+     "clips": [
+      "FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444.mp4"
+     ],
+     "part": null,
+     "question": "How was the video in this clip made? This question is only about the video (what you see), not the sound. Normal processing such as brightness, colour or compression does not count.",
+     "options": [
+      "video-driven animation",
+      "audio-driven animation",
+      "real",
+      "face swap"
+     ],
+     "gt_answer": "audio-driven animation",
+     "pred_answer": "video-driven animation",
+     "correct": false,
+     "raw": "{\"answer\": \"video-driven animation\"}"
+    }
+   ]
+  },
+  "t3": {
+   "scores": {
+    "n": 10,
+    "accuracy": 0.7,
+    "ci95": [
+     0.3968,
+     0.8922
+    ],
+    "balanced_accuracy": 0.7084,
+    "cohen_kappa": 0.4
+   },
+   "items": [
+    {
+     "id": "t3_face_A_FVRA__animate__77f__main__id07839__QTWWQBdhWY_00029__id07683",
+     "clips": [
+      "FVRA__animate__77f__main__id07839__QTWWQBdhWY_00029__id07683.mp4",
+      "RVRA__real__77f__main__id07839_gJgj1DxOfKA_00045__id07786.mp4"
+     ],
+     "part": "face",
+     "question": "Clip 2 is a real recording of one person. Look only at the faces: is the face in clip 1 the same person as in clip 2? Face means the inner face: eyes, nose and mouth.",
+     "options": [
+      "yes",
+      "no"
+     ],
+     "gt_answer": "no",
+     "pred_answer": "no",
+     "correct": true,
+     "raw": "```json\n{\"answer\": \"no\"}\n```"
+    }
+   ]
+  },
+  "t3_5": {
+   "scores": {
+    "n": 10,
+    "accuracy": 0.8,
+    "ci95": [
+     0.4902,
+     0.9433
+    ],
+    "balanced_accuracy": 0.75,
+    "cohen_kappa": 0.5455
+   },
+   "items": [
+    {
+     "id": "t35_face_A_RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564",
+     "clips": [
+      "RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564.mp4",
+      "RVRA__real__77f__main__id05877_6QUcq_6B-LU_00011__id07768.mp4"
+     ],
+     "part": "face",
+     "question": "Clip 2 is a real recording of one person. Look only at the faces: is clip 1 a real recording of that same person's face? Answer no if the face in clip 1 belongs to someone else, or was swapped, changed or made by a computer, even if it looks like that person. A real recording that was only processed, without changing whose face it is, still counts as real. Face means the inner face: eyes, nose and mouth.",
+     "options": [
+      "yes",
+      "no"
+     ],
+     "gt_answer": "yes",
+     "pred_answer": "yes",
+     "correct": true,
+     "raw": "```json\n{\"answer\": \"yes\"}\n```"
+    }
+   ]
+  }
+ }
+}