eval/analyze_tempcompass.py

#!/usr/bin/env python3
"""Analyze TempCompass evaluation data: direction & speed & attribute_change questions."""

import json
import re
import random
from collections import Counter, defaultdict

random.seed(42)

# ── Load data ────────────────────────────────────────────────────────
with open("/root/zwj/Eval/Evaluation/eval_tempcompass.json", "r") as f:
    data = json.load(f)

print(f"Total entries: {len(data)}")
dim_counts = Counter(item["dim"] for item in data)
print("Dimension distribution:")
for dim, cnt in dim_counts.most_common():
    print(f"  {dim}: {cnt}")

# ── Separate by dim ──────────────────────────────────────────────────
direction_qs = [item for item in data if item["dim"] == "direction"]
speed_qs = [item for item in data if item["dim"] == "speed"]
attr_change_qs = [item for item in data if item["dim"] == "attribute_change"]

print(f"\nDirection questions: {len(direction_qs)}")
print(f"Speed questions: {len(speed_qs)}")
print(f"Attribute change questions: {len(attr_change_qs)}")

# ═══════════════════════════════════════════════════════════════════════
# 1. DIRECTION QUESTIONS — sample 25
# ═══════════════════════════════════════════════════════════════════════
dir_sample = random.sample(direction_qs, min(25, len(direction_qs)))
print("\n" + "=" * 80)
print("SECTION 1: DIRECTION QUESTION SAMPLES (25)")
print("=" * 80)
for i, q in enumerate(dir_sample, 1):
    print(f"\n--- Direction Q#{i} (video_id={q['video_id']}, problem_id={q['problem_id']}) ---")
    print(f"  Question: {q['question'][:300]}")
    print(f"  Options:  {q['options']}")
    print(f"  Answer:   {q['answer'][:200]}")
    print(f"  Video:    {q['path']}")

# ═══════════════════════════════════════════════════════════════════════
# 2. SPEED QUESTIONS — sample 25
# ═══════════════════════════════════════════════════════════════════════
spd_sample = random.sample(speed_qs, min(25, len(speed_qs)))
print("\n" + "=" * 80)
print("SECTION 2: SPEED QUESTION SAMPLES (25)")
print("=" * 80)
for i, q in enumerate(spd_sample, 1):
    print(f"\n--- Speed Q#{i} (video_id={q['video_id']}, problem_id={q['problem_id']}) ---")
    print(f"  Question: {q['question'][:300]}")
    print(f"  Options:  {q['options']}")
    print(f"  Answer:   {q['answer'][:200]}")
    print(f"  Video:    {q['path']}")

# ═══════════════════════════════════════════════════════════════════════
# 3. ATTRIBUTE CHANGE QUESTIONS — sample 10
# ═══════════════════════════════════════════════════════════════════════
ac_sample = random.sample(attr_change_qs, min(10, len(attr_change_qs)))
print("\n" + "=" * 80)
print("SECTION 3: ATTRIBUTE CHANGE QUESTION SAMPLES (10)")
print("=" * 80)
for i, q in enumerate(ac_sample, 1):
    print(f"\n--- AttrChange Q#{i} (video_id={q['video_id']}, problem_id={q['problem_id']}) ---")
    print(f"  Question: {q['question'][:400]}")
    print(f"  Options:  {q['options']}")
    print(f"  Answer:   {q['answer'][:200]}")
    print(f"  Video:    {q['path']}")

# ═══════════════════════════════════════════════════════════════════════
# 4. DIRECTION SUBTYPE CATEGORIZATION
# ═══════════════════════════════════════════════════════════════════════
print("\n" + "=" * 80)
print("SECTION 4: DIRECTION QUESTION SUBTYPE ANALYSIS")
print("=" * 80)

def classify_direction(text):
    """Classify a direction question/answer into subtypes."""
    t = text.lower()
    subtypes = []

    # Left / Right
    if re.search(r'left.to.right|right.to.left|left|right|leftward|rightward', t):
        subtypes.append("left/right motion")
    # Up / Down
    if re.search(r'up.to.down|down.to.up|upward|downward|ascending|descending|rises|falls|going up|going down|moving up|moving down|top.to.bottom|bottom.to.top', t):
        subtypes.append("up/down motion")
    # Clockwise / counterclockwise / rotation
    if re.search(r'clockwise|counterclockwise|counter-clockwise|rotating|rotation|spinning|revolving|turning', t):
        subtypes.append("clockwise/rotation")
    # Toward / Away (camera-relative)
    if re.search(r'toward|away|closer|further|approaching|receding|coming closer|moving away|zooming in|zooming out|towards', t):
        subtypes.append("toward/away from camera")
    # Forward / Backward
    if re.search(r'forward|backward|forwards|backwards|ahead', t):
        subtypes.append("forward/backward")
    # Opening / Closing (e.g., door, lid)
    if re.search(r'opening|closing|open|close|unfold|fold', t):
        subtypes.append("opening/closing")
    # Pan / tilt (camera motion)
    if re.search(r'pan|tilt|camera.*(move|motion|shift)|zoom', t):
        subtypes.append("camera pan/tilt/zoom")
    # Inward / outward / expansion / contraction
    if re.search(r'inward|outward|expand|contract|spread|shrink', t):
        subtypes.append("inward/outward/expansion")

    if not subtypes:
        subtypes.append("other/unclassified")
    return subtypes

dir_subtype_counter = Counter()
dir_subtype_examples = defaultdict(list)

for q in direction_qs:
    # Use both question and answer text for classification
    combined = q["question"] + " " + q["answer"]
    subtypes = classify_direction(combined)
    for st in subtypes:
        dir_subtype_counter[st] += 1
        if len(dir_subtype_examples[st]) < 3:
            dir_subtype_examples[st].append(q)

print(f"\nTotal direction questions analyzed: {len(direction_qs)}")
print("\nSubtype counts (a question can belong to multiple subtypes):")
for st, cnt in dir_subtype_counter.most_common():
    print(f"  {st}: {cnt}")

print("\nRepresentative examples per subtype:")
for st in dir_subtype_counter.most_common():
    st_name = st[0]
    print(f"\n  >> Subtype: {st_name} ({st[1]} total)")
    for ex in dir_subtype_examples[st_name][:2]:
        print(f"     Video: {ex['video_id']} | Answer: {ex['answer'][:120]}")

# ── Also look at 'other/unclassified' to see what we missed ──
if "other/unclassified" in dir_subtype_examples:
    print("\n  Details of 'other/unclassified' direction questions:")
    for ex in dir_subtype_examples["other/unclassified"][:5]:
        print(f"     Q: {ex['question'][:200]}")
        print(f"     A: {ex['answer'][:200]}")
        print()

# ═══════════════════════════════════════════════════════════════════════
# 5. SPEED SUBTYPE CATEGORIZATION
# ═══════════════════════════════════════════════════════════════════════
print("\n" + "=" * 80)
print("SECTION 5: SPEED QUESTION SUBTYPE ANALYSIS")
print("=" * 80)

def classify_speed(text):
    """Classify a speed question/answer into subtypes."""
    t = text.lower()
    subtypes = []

    # Fast / Slow absolute
    if re.search(r'fast|rapid|quick|swiftly|brisk|speedy|high.speed|high speed', t):
        subtypes.append("fast (absolute)")
    if re.search(r'slow|sluggish|leisurely|unhurried|gentle pace|low.speed', t):
        subtypes.append("slow (absolute)")
    # Timelapse / slow-motion
    if re.search(r'timelapse|time.lapse|time-lapse|sped.up|fast.forward|accelerated.*(footage|video|playback)', t):
        subtypes.append("timelapse/sped-up")
    if re.search(r'slow.motion|slow-motion|slowed.down|decelerat', t):
        subtypes.append("slow-motion playback")
    # Normal speed / real-time
    if re.search(r'normal.speed|regular.speed|real.time|real-time|natural.pace|actual.speed|normal.pace|original.speed', t):
        subtypes.append("normal/real-time speed")
    # Acceleration / deceleration
    if re.search(r'accelerat|speed.*up|getting faster|increasing speed', t):
        subtypes.append("acceleration")
    if re.search(r'decelerat|slow.*down|getting slower|decreasing speed|slowing', t):
        subtypes.append("deceleration")
    # Comparative (faster vs slower)
    if re.search(r'faster|slower|more quickly|more slowly|compared|relative', t):
        subtypes.append("comparative speed")

    if not subtypes:
        subtypes.append("other/unclassified")
    return subtypes

spd_subtype_counter = Counter()
spd_subtype_examples = defaultdict(list)

for q in speed_qs:
    combined = q["question"] + " " + q["answer"]
    subtypes = classify_speed(combined)
    for st in subtypes:
        spd_subtype_counter[st] += 1
        if len(spd_subtype_examples[st]) < 3:
            spd_subtype_examples[st].append(q)

print(f"\nTotal speed questions analyzed: {len(speed_qs)}")
print("\nSubtype counts (a question can belong to multiple subtypes):")
for st, cnt in spd_subtype_counter.most_common():
    print(f"  {st}: {cnt}")

print("\nRepresentative examples per subtype:")
for st in spd_subtype_counter.most_common():
    st_name = st[0]
    print(f"\n  >> Subtype: {st_name} ({st[1]} total)")
    for ex in spd_subtype_examples[st_name][:2]:
        print(f"     Video: {ex['video_id']} | Answer: {ex['answer'][:150]}")

# ── Unclassified speed ──
if "other/unclassified" in spd_subtype_examples:
    print("\n  Details of 'other/unclassified' speed questions:")
    for ex in spd_subtype_examples["other/unclassified"][:5]:
        print(f"     Q: {ex['question'][:200]}")
        print(f"     A: {ex['answer'][:200]}")
        print()

# ═══════════════════════════════════════════════════════════════════════
# 6. VIDEO SOURCES ANALYSIS
# ═══════════════════════════════════════════════════════════════════════
print("\n" + "=" * 80)
print("SECTION 6: VIDEO SOURCES ANALYSIS")
print("=" * 80)

# Unique videos across direction + speed + attribute_change
all_relevant = direction_qs + speed_qs + attr_change_qs
video_ids_dir = set(q["video_id"] for q in direction_qs)
video_ids_spd = set(q["video_id"] for q in speed_qs)
video_ids_ac = set(q["video_id"] for q in attr_change_qs)
all_video_ids = set(q["video_id"] for q in data)

print(f"\nUnique videos (all dims): {len(all_video_ids)}")
print(f"Unique videos (direction): {len(video_ids_dir)}")
print(f"Unique videos (speed): {len(video_ids_spd)}")
print(f"Unique videos (attribute_change): {len(video_ids_ac)}")

# Analyze video_id patterns — are they numeric IDs (Pexels/YouTube)?
numeric_ids = [vid for vid in all_video_ids if vid.isdigit()]
text_ids = [vid for vid in all_video_ids if not vid.isdigit()]

print(f"\nNumeric video IDs: {len(numeric_ids)} (likely Pexels/stock footage)")
print(f"Text-based video IDs: {len(text_ids)}")
if text_ids:
    print(f"  Sample text IDs: {text_ids[:20]}")
if numeric_ids:
    print(f"  Sample numeric IDs: {sorted(numeric_ids)[:20]}")

# Analyze paths
paths = set(q["path"] for q in data)
path_prefixes = Counter()
for p in paths:
    parts = p.split("/")
    if len(parts) >= 3:
        path_prefixes["/".join(parts[:3])] += 1
    else:
        path_prefixes[p] += 1
print(f"\nPath prefixes:")
for pfx, cnt in path_prefixes.most_common(10):
    print(f"  {pfx}: {cnt}")

# ── Analyze what direction videos depict (from question content) ──
print("\n--- What direction videos depict (from question text) ---")
dir_video_subjects = Counter()
for q in direction_qs:
    t = q["question"].lower() + " " + q["answer"].lower()
    # Try to extract the main subject
    subjects = []
    if re.search(r'man|person|people|woman|girl|boy|child|baby', t):
        subjects.append("people")
    if re.search(r'car|vehicle|truck|bus|train|boat|ship|plane|airplane|helicopter|motorcycle|bicycle|bike', t):
        subjects.append("vehicle/transport")
    if re.search(r'ball|basketball|football|soccer|tennis', t):
        subjects.append("ball/sports")
    if re.search(r'animal|dog|cat|bird|fish|horse|insect|butterfly|bee', t):
        subjects.append("animal")
    if re.search(r'water|river|wave|ocean|rain|snow|cloud|smoke|fire|flame', t):
        subjects.append("nature/elements")
    if re.search(r'camera|drone|aerial', t):
        subjects.append("camera/drone")
    if re.search(r'robot|machine|mechanical', t):
        subjects.append("robot/machine")
    if not subjects:
        subjects.append("other")
    for s in subjects:
        dir_video_subjects[s] += 1

print("Direction video subject matter:")
for subj, cnt in dir_video_subjects.most_common():
    print(f"  {subj}: {cnt}")

# Same for speed videos
print("\n--- What speed videos depict ---")
spd_video_subjects = Counter()
for q in speed_qs:
    t = q["question"].lower() + " " + q["answer"].lower()
    subjects = []
    if re.search(r'man|person|people|woman|girl|boy|child|baby|walk|run', t):
        subjects.append("people")
    if re.search(r'car|vehicle|truck|bus|train|boat|ship|plane|airplane|helicopter|motorcycle|bicycle|bike|driv', t):
        subjects.append("vehicle/transport")
    if re.search(r'ball|basketball|football|soccer|tennis', t):
        subjects.append("ball/sports")
    if re.search(r'animal|dog|cat|bird|fish|horse|insect|butterfly|bee|turtle|snail', t):
        subjects.append("animal")
    if re.search(r'water|river|wave|ocean|rain|snow|cloud|smoke|fire|flame|wind', t):
        subjects.append("nature/elements")
    if re.search(r'camera|drone|aerial', t):
        subjects.append("camera/drone")
    if re.search(r'timelapse|time.lapse|time-lapse', t):
        subjects.append("timelapse content")
    if not subjects:
        subjects.append("other")
    for s in subjects:
        spd_video_subjects[s] += 1

print("Speed video subject matter:")
for subj, cnt in spd_video_subjects.most_common():
    print(f"  {subj}: {cnt}")

# ═══════════════════════════════════════════════════════════════════════
# SUMMARY
# ═══════════════════════════════════════════════════════════════════════
print("\n" + "=" * 80)
print("SUMMARY")
print("=" * 80)
print(f"Total dataset entries: {len(data)}")
print(f"Dimensions: {dict(dim_counts)}")
print(f"\nDirection questions: {len(direction_qs)} across {len(video_ids_dir)} unique videos")
print(f"  Top subtypes: {dir_subtype_counter.most_common(5)}")
print(f"\nSpeed questions: {len(speed_qs)} across {len(video_ids_spd)} unique videos")
print(f"  Top subtypes: {spd_subtype_counter.most_common(5)}")
print(f"\nAttribute change questions: {len(attr_change_qs)} across {len(video_ids_ac)} unique videos")

# Questions per video distribution
qs_per_video = Counter(q["video_id"] for q in data)
print(f"\nQuestions per video: min={min(qs_per_video.values())}, max={max(qs_per_video.values())}, "
      f"mean={sum(qs_per_video.values())/len(qs_per_video):.1f}")