audible audio, sentence-style moves/places, clearer sound and gaze questions, how-to panel

app.py

@@ -8,6 +8,7 @@ import argparse
 import csv
 import json
 import os
+import re
 from datetime import datetime
 from pathlib import Path
 
@@ -68,6 +69,48 @@ if RESP_CSV.exists():
         RESP_CSV.rename(RESP_DIR / f"responses_old_{datetime.now():%Y%m%d_%H%M%S}.csv")
 
 
+def place(anchor):
+    """`P02_counter.002` -> `the counter #2`; `person_01` -> `the wearer's hand`."""
+    if not anchor or anchor == "unknown":
+        return "an unknown place"
+    if anchor.startswith("person"):
+        return "the wearer's hand"
+    kind, _, num = re.sub(r"^P\d\d_", "", anchor).partition(".")
+    return f"the {kind.replace('_', ' ')}" + (f" #{int(num)}" if num.isdigit() else "")
+
+
+def where(rel, anchor):
+    if rel == "held_by":
+        return "held in the wearer's hand"
+    return {"inside": "inside", "on": "on", "hanging_on": "hanging on"}.get(rel, "at") + " " + place(anchor)
+
+
+def state_sentence(st):
+    a, b = st["interval_sec"]
+    w = where(st["relation"], st["anchor"])
+    if b is None:
+        return f"from {a:.1f}s until the end of the clip: {w}"
+    if abs(b - a) < 0.05:
+        return f"at {a:.1f}s: {w}"
+    return f"from {a:.1f}s to {b:.1f}s: {w}"
+
+
+def move_sentence(tr):
+    a, b = tr["interval_sec"]
+    when = f"at {a:.1f}s" if abs(b - a) < 0.05 else f"between {a:.1f}s and {b:.1f}s"
+    return (f"{when} it goes from “{where(tr['from']['relation'], tr['from']['anchor'])}” to "
+            f"“{where(tr['to']['relation'], tr['to']['anchor'])}” (action: {tr['event'].replace('_', ' ')})")
+
+
+def gaze_summary(it):
+    a, att = it["annotation"], it.get("attention") or {}
+    if a.get("looked_at"):
+        gap = att.get("prime_gap_sec")
+        return (f"Yes — first look at {att.get('first_gaze_sec')}s"
+                + (f", about {gap:.1f}s before the hand touches it" if isinstance(gap, (int, float)) else ""))
+    return "No — the memory found no look at this object before the hand touches it"
+
+
 def _join(xs):
     return "; ".join(f"({i + 1}) {x}" for i, x in enumerate(xs))
 
@@ -83,29 +126,41 @@ def dim_questions(it):
                      f"“{a.get('looks_like')}”. Is that what the object is and looks like?")
     q["localization"] = (f"2. Localization — do the red box (video + keyframes) and the green mask (keyframes) "
                          f"cover the {name}, and not a neighbouring object, the hand, or background?")
-    hist = a.get("history") or []
-    q["state_history"] = (f"3. Location history of the {name} — the memory says: {_join(hist)}. "
-                          f"At the end it is “{now.get('relation')} {now.get('at')}”. Are the places and times correct?")
-    moves = a.get("moves") or []
-    q["moves"] = (f"4. Moves — the memory records these moves of the {name}: {_join(moves)}. "
-                  "Are the from → to and the times correct, and is no move missing?"
+    hist = [state_sentence(x) for x in it.get("states") or []] or (a.get("history") or [])
+    q["state_history"] = (f"3. Where it is — the memory says the {name} is: {_join(hist)}. "
+                          "Are these places and times correct?")
+    moves = [move_sentence(x) for x in it.get("transitions") or []]
+    MOVE_DEF = "A move = the object changes where it is (picked up, put down, put into or taken out of something). "
+    q["moves"] = (f"4. Moves — {MOVE_DEF}The memory records for the {name}: {_join(moves)}. "
+                  "Is each move real, with the right start place, end place and time — and is no move missing?"
                   if moves else
-                  f"4. Moves — the memory records NO move for the {name}. Do you agree that this object was "
-                  "not moved from one place to another during the clip?")
-    snd = [f"{e['label']} [{e['t0']:.1f}–{e['t1']:.1f}s]" for e in ev.get("sound") or []]
-    sp = [f"“{e['label']}” [{e['t0']:.1f}–{e['t1']:.1f}s]" for e in ev.get("speech") or []]
-    q["sound_speech"] = ((f"5. Sounds — the memory links these sounds to the {name}: {_join(snd)}. "
-                          "Was each of them made by (or with) this object?"
+                  f"4. Moves — {MOVE_DEF}The memory records NO move for the {name}. Do you agree that this "
+                  "object stays where it is for the whole clip?")
+    snd = [f"“{e['label']}” at {e['t0']:.1f}–{e['t1']:.1f}s" for e in ev.get("sound") or []]
+    sp = [f"“{e['label']}” at {e['t0']:.1f}–{e['t1']:.1f}s" for e in ev.get("speech") or []]
+    LISTEN = "🔊 Turn the sound ON for this one (the audio is amplified; click the speaker icon if the player is muted). "
+    q["sound_speech"] = ((f"5. Sounds — {LISTEN}The memory links these sounds to the {name}: {_join(snd)}. "
+                          "Sound names are dataset categories (e.g. “stir / mix / whisk” = a stirring sound, "
+                          "“metal / plastic collision” = metal hitting plastic). For each: can you hear it at that "
+                          "time, and is it made by or with THIS object?"
                           if snd else
-                          f"5. Sounds — the memory links NO sound to the {name}. Do you agree that this object "
-                          "made no audible sound in the clip?")
+                          f"5. Sounds — {LISTEN}The memory links NO sound to the {name}. Do you agree that this "
+                          "object makes no audible sound in the clip?")
                          + (f" Speech heard in the clip: {_join(sp)} — is it transcribed correctly?" if sp else ""))
-    dwell = [f"{e['t0']:.1f}–{e['t1']:.1f}s" for e in ev.get("dwell") or []]
-    q["gaze"] = (f"6. Gaze — the memory says the wearer looked at the {name} (first at {att.get('first_gaze_sec')}s; "
-                 f"dwell {', '.join(dwell)}). During those times, is the red gaze dot on or right next to this object?"
+    GAZE_DEF = ("The red dot shows where the wearer is looking at every moment — it is always on screen. "
+                "This question is only about whether they looked at THIS object just before touching it. ")
+    gap = att.get("prime_gap_sec")
+    q["gaze"] = (f"6. Gaze before touching — {GAZE_DEF}The memory claims the wearer looked at the {name} at about "
+                 f"{att.get('first_gaze_sec')}s"
+                 + (f", {gap:.1f}s before the hand touches it" if isinstance(gap, (int, float)) else "")
+                 + (" (yellow bar on the timeline). Around that time, is the red dot on or very close to this object?"
+                    if it["window_sec"][0] <= (att.get("first_gaze_sec") or -1) <= it["window_sec"][1] else
+                    f". That moment is OUTSIDE this clip ({it['window_sec'][0]:.0f}–{it['window_sec'][1]:.0f}s), so it "
+                    "cannot be checked here — please answer “Can't tell”.")
                  if a.get("looked_at") else
-                 f"6. Gaze — the memory says the wearer did NOT look at the {name} before handling it. "
-                 "Do you agree — does the red gaze dot stay off this object before it is touched?")
+                 f"6. Gaze before touching — {GAZE_DEF}The memory claims the wearer did NOT look at the {name} before "
+                 "touching it (grabbed or put it down without looking). In the few seconds before the hand first "
+                 "touches this object, does the red dot stay away from it?")
     return [gr.update(label=q[k], value=None) for k, _ in DIMENSIONS]
 
 
@@ -188,19 +243,19 @@ def annotation_md(it):
 | **category** | {a.get('category')} |
 | **looks like** | {a.get('looks_like')} |
 | **color / material** | {a.get('color', '?')} / {a.get('material', '?')} |
-| **now** | {now.get('relation')} **{now.get('at')}** since {now.get('since_sec')}s{(' · contains: ' + str(now['contains'])) if now.get('contains') else ''} |
-| **looked at** | {a.get('looked_at')} — first gaze {att.get('first_gaze_sec')}s, {att.get('prime_gap_sec')}s before contact, total dwell {att.get('total_dwell_sec')}s |
+| **at the end of the clip** | {where(now.get('relation'), now.get('at'))} (since {now.get('since_sec')}s){(' · contains: ' + str(now['contains'])) if now.get('contains') else ''} |
+| **looked at before touching** | {gaze_summary(it)} |
 | **lifecycle** | first {lc.get('first_observed_sec')}s → last {lc.get('last_observed_sec')}s, {lc.get('final_status')} |
 | **confidence** | {it.get('confidence')} |
 
-### History (where it was)
-{fmt_list(a.get('history'))}
+### Where it is (location history)
+{fmt_list([state_sentence(x) for x in it.get('states') or []] or a.get('history'))}
 
-### Moves
-{fmt_list(a.get('moves'))}
+### Moves (changes of place)
+{fmt_list([move_sentence(x) for x in it.get('transitions') or []])}
 
-### Gaze dwell intervals (yellow lane; red dot = gaze point)
-{fmt_iv(ev.get('dwell'))}
+### Claimed look at the object before touching it (yellow lane; the red dot itself is always shown)
+{fmt_iv(ev.get('dwell')) if a.get('looked_at') else '- *(none claimed)*'}
 
 ### Narrations linked to this object (blue lane) — annotator text, NOT audio
 {fmt_iv(ev.get('narration')) if ev.get('narration') else fmt_list(a.get('said'))}
@@ -287,6 +342,19 @@ with gr.Blocks(title="Object-centric annotation eval") as demo:
                 ann_md = gr.Markdown()
         with gr.Accordion("Cross-check vs original HD-EPIC annotation (auto-generated, for reference)", open=False):
             xc_md = gr.Markdown()
+        with gr.Accordion("How to judge — read this once", open=True):
+            gr.Markdown(
+                "- You are checking an automatically generated **memory record of ONE object** against the video. "
+                "Each question states a claim from that record; answer whether the claim matches what you see and hear.\n"
+                "- **Correct** = fully right · **Partially correct** = right idea but a place, time or detail is off · "
+                "**Incorrect** = wrong · **Can't tell** = the video does not let you decide.\n"
+                "- **Times** are seconds of the full recording: read them from the boxed **VIDEO TIME** (top-right) "
+                "or the numbers on the bottom time axis. Small differences (under about 1 s) are fine.\n"
+                "- **Places** such as “the counter #2” or “the fridge #1” are the dataset's names for kitchen fixtures; "
+                "judge the kind of place (counter, fridge, drawer, hand …), you do not need to know which number it is.\n"
+                "- **Sound**: 🔊 turn the sound on. The audio has been amplified because the original recording is very quiet.\n"
+                "- **Gaze**: the red dot is where the wearer looks at each moment and is always visible. "
+                "The gaze question only asks whether they looked at this object just before touching it.")
         gr.Markdown("## Your judgement")
         radios = [gr.Radio(CHOICES, label=q) for _, q in DIMENSIONS]
         gr.Markdown("### Narrations — judge each one separately\n"