#!/usr/bin/env python3 """Analyze TempCompass evaluation data: direction & speed & attribute_change questions.""" import json import re import random from collections import Counter, defaultdict random.seed(42) # ── Load data ──────────────────────────────────────────────────────── with open("/root/zwj/Eval/Evaluation/eval_tempcompass.json", "r") as f: data = json.load(f) print(f"Total entries: {len(data)}") dim_counts = Counter(item["dim"] for item in data) print("Dimension distribution:") for dim, cnt in dim_counts.most_common(): print(f" {dim}: {cnt}") # ── Separate by dim ────────────────────────────────────────────────── direction_qs = [item for item in data if item["dim"] == "direction"] speed_qs = [item for item in data if item["dim"] == "speed"] attr_change_qs = [item for item in data if item["dim"] == "attribute_change"] print(f"\nDirection questions: {len(direction_qs)}") print(f"Speed questions: {len(speed_qs)}") print(f"Attribute change questions: {len(attr_change_qs)}") # ═══════════════════════════════════════════════════════════════════════ # 1. DIRECTION QUESTIONS — sample 25 # ═══════════════════════════════════════════════════════════════════════ dir_sample = random.sample(direction_qs, min(25, len(direction_qs))) print("\n" + "=" * 80) print("SECTION 1: DIRECTION QUESTION SAMPLES (25)") print("=" * 80) for i, q in enumerate(dir_sample, 1): print(f"\n--- Direction Q#{i} (video_id={q['video_id']}, problem_id={q['problem_id']}) ---") print(f" Question: {q['question'][:300]}") print(f" Options: {q['options']}") print(f" Answer: {q['answer'][:200]}") print(f" Video: {q['path']}") # ═══════════════════════════════════════════════════════════════════════ # 2. SPEED QUESTIONS — sample 25 # ═══════════════════════════════════════════════════════════════════════ spd_sample = random.sample(speed_qs, min(25, len(speed_qs))) print("\n" + "=" * 80) print("SECTION 2: SPEED QUESTION SAMPLES (25)") print("=" * 80) for i, q in enumerate(spd_sample, 1): print(f"\n--- Speed Q#{i} (video_id={q['video_id']}, problem_id={q['problem_id']}) ---") print(f" Question: {q['question'][:300]}") print(f" Options: {q['options']}") print(f" Answer: {q['answer'][:200]}") print(f" Video: {q['path']}") # ═══════════════════════════════════════════════════════════════════════ # 3. ATTRIBUTE CHANGE QUESTIONS — sample 10 # ═══════════════════════════════════════════════════════════════════════ ac_sample = random.sample(attr_change_qs, min(10, len(attr_change_qs))) print("\n" + "=" * 80) print("SECTION 3: ATTRIBUTE CHANGE QUESTION SAMPLES (10)") print("=" * 80) for i, q in enumerate(ac_sample, 1): print(f"\n--- AttrChange Q#{i} (video_id={q['video_id']}, problem_id={q['problem_id']}) ---") print(f" Question: {q['question'][:400]}") print(f" Options: {q['options']}") print(f" Answer: {q['answer'][:200]}") print(f" Video: {q['path']}") # ═══════════════════════════════════════════════════════════════════════ # 4. DIRECTION SUBTYPE CATEGORIZATION # ═══════════════════════════════════════════════════════════════════════ print("\n" + "=" * 80) print("SECTION 4: DIRECTION QUESTION SUBTYPE ANALYSIS") print("=" * 80) def classify_direction(text): """Classify a direction question/answer into subtypes.""" t = text.lower() subtypes = [] # Left / Right if re.search(r'left.to.right|right.to.left|left|right|leftward|rightward', t): subtypes.append("left/right motion") # Up / Down if re.search(r'up.to.down|down.to.up|upward|downward|ascending|descending|rises|falls|going up|going down|moving up|moving down|top.to.bottom|bottom.to.top', t): subtypes.append("up/down motion") # Clockwise / counterclockwise / rotation if re.search(r'clockwise|counterclockwise|counter-clockwise|rotating|rotation|spinning|revolving|turning', t): subtypes.append("clockwise/rotation") # Toward / Away (camera-relative) if re.search(r'toward|away|closer|further|approaching|receding|coming closer|moving away|zooming in|zooming out|towards', t): subtypes.append("toward/away from camera") # Forward / Backward if re.search(r'forward|backward|forwards|backwards|ahead', t): subtypes.append("forward/backward") # Opening / Closing (e.g., door, lid) if re.search(r'opening|closing|open|close|unfold|fold', t): subtypes.append("opening/closing") # Pan / tilt (camera motion) if re.search(r'pan|tilt|camera.*(move|motion|shift)|zoom', t): subtypes.append("camera pan/tilt/zoom") # Inward / outward / expansion / contraction if re.search(r'inward|outward|expand|contract|spread|shrink', t): subtypes.append("inward/outward/expansion") if not subtypes: subtypes.append("other/unclassified") return subtypes dir_subtype_counter = Counter() dir_subtype_examples = defaultdict(list) for q in direction_qs: # Use both question and answer text for classification combined = q["question"] + " " + q["answer"] subtypes = classify_direction(combined) for st in subtypes: dir_subtype_counter[st] += 1 if len(dir_subtype_examples[st]) < 3: dir_subtype_examples[st].append(q) print(f"\nTotal direction questions analyzed: {len(direction_qs)}") print("\nSubtype counts (a question can belong to multiple subtypes):") for st, cnt in dir_subtype_counter.most_common(): print(f" {st}: {cnt}") print("\nRepresentative examples per subtype:") for st in dir_subtype_counter.most_common(): st_name = st[0] print(f"\n >> Subtype: {st_name} ({st[1]} total)") for ex in dir_subtype_examples[st_name][:2]: print(f" Video: {ex['video_id']} | Answer: {ex['answer'][:120]}") # ── Also look at 'other/unclassified' to see what we missed ── if "other/unclassified" in dir_subtype_examples: print("\n Details of 'other/unclassified' direction questions:") for ex in dir_subtype_examples["other/unclassified"][:5]: print(f" Q: {ex['question'][:200]}") print(f" A: {ex['answer'][:200]}") print() # ═══════════════════════════════════════════════════════════════════════ # 5. SPEED SUBTYPE CATEGORIZATION # ═══════════════════════════════════════════════════════════════════════ print("\n" + "=" * 80) print("SECTION 5: SPEED QUESTION SUBTYPE ANALYSIS") print("=" * 80) def classify_speed(text): """Classify a speed question/answer into subtypes.""" t = text.lower() subtypes = [] # Fast / Slow absolute if re.search(r'fast|rapid|quick|swiftly|brisk|speedy|high.speed|high speed', t): subtypes.append("fast (absolute)") if re.search(r'slow|sluggish|leisurely|unhurried|gentle pace|low.speed', t): subtypes.append("slow (absolute)") # Timelapse / slow-motion if re.search(r'timelapse|time.lapse|time-lapse|sped.up|fast.forward|accelerated.*(footage|video|playback)', t): subtypes.append("timelapse/sped-up") if re.search(r'slow.motion|slow-motion|slowed.down|decelerat', t): subtypes.append("slow-motion playback") # Normal speed / real-time if re.search(r'normal.speed|regular.speed|real.time|real-time|natural.pace|actual.speed|normal.pace|original.speed', t): subtypes.append("normal/real-time speed") # Acceleration / deceleration if re.search(r'accelerat|speed.*up|getting faster|increasing speed', t): subtypes.append("acceleration") if re.search(r'decelerat|slow.*down|getting slower|decreasing speed|slowing', t): subtypes.append("deceleration") # Comparative (faster vs slower) if re.search(r'faster|slower|more quickly|more slowly|compared|relative', t): subtypes.append("comparative speed") if not subtypes: subtypes.append("other/unclassified") return subtypes spd_subtype_counter = Counter() spd_subtype_examples = defaultdict(list) for q in speed_qs: combined = q["question"] + " " + q["answer"] subtypes = classify_speed(combined) for st in subtypes: spd_subtype_counter[st] += 1 if len(spd_subtype_examples[st]) < 3: spd_subtype_examples[st].append(q) print(f"\nTotal speed questions analyzed: {len(speed_qs)}") print("\nSubtype counts (a question can belong to multiple subtypes):") for st, cnt in spd_subtype_counter.most_common(): print(f" {st}: {cnt}") print("\nRepresentative examples per subtype:") for st in spd_subtype_counter.most_common(): st_name = st[0] print(f"\n >> Subtype: {st_name} ({st[1]} total)") for ex in spd_subtype_examples[st_name][:2]: print(f" Video: {ex['video_id']} | Answer: {ex['answer'][:150]}") # ── Unclassified speed ── if "other/unclassified" in spd_subtype_examples: print("\n Details of 'other/unclassified' speed questions:") for ex in spd_subtype_examples["other/unclassified"][:5]: print(f" Q: {ex['question'][:200]}") print(f" A: {ex['answer'][:200]}") print() # ═══════════════════════════════════════════════════════════════════════ # 6. VIDEO SOURCES ANALYSIS # ═══════════════════════════════════════════════════════════════════════ print("\n" + "=" * 80) print("SECTION 6: VIDEO SOURCES ANALYSIS") print("=" * 80) # Unique videos across direction + speed + attribute_change all_relevant = direction_qs + speed_qs + attr_change_qs video_ids_dir = set(q["video_id"] for q in direction_qs) video_ids_spd = set(q["video_id"] for q in speed_qs) video_ids_ac = set(q["video_id"] for q in attr_change_qs) all_video_ids = set(q["video_id"] for q in data) print(f"\nUnique videos (all dims): {len(all_video_ids)}") print(f"Unique videos (direction): {len(video_ids_dir)}") print(f"Unique videos (speed): {len(video_ids_spd)}") print(f"Unique videos (attribute_change): {len(video_ids_ac)}") # Analyze video_id patterns — are they numeric IDs (Pexels/YouTube)? numeric_ids = [vid for vid in all_video_ids if vid.isdigit()] text_ids = [vid for vid in all_video_ids if not vid.isdigit()] print(f"\nNumeric video IDs: {len(numeric_ids)} (likely Pexels/stock footage)") print(f"Text-based video IDs: {len(text_ids)}") if text_ids: print(f" Sample text IDs: {text_ids[:20]}") if numeric_ids: print(f" Sample numeric IDs: {sorted(numeric_ids)[:20]}") # Analyze paths paths = set(q["path"] for q in data) path_prefixes = Counter() for p in paths: parts = p.split("/") if len(parts) >= 3: path_prefixes["/".join(parts[:3])] += 1 else: path_prefixes[p] += 1 print(f"\nPath prefixes:") for pfx, cnt in path_prefixes.most_common(10): print(f" {pfx}: {cnt}") # ── Analyze what direction videos depict (from question content) ── print("\n--- What direction videos depict (from question text) ---") dir_video_subjects = Counter() for q in direction_qs: t = q["question"].lower() + " " + q["answer"].lower() # Try to extract the main subject subjects = [] if re.search(r'man|person|people|woman|girl|boy|child|baby', t): subjects.append("people") if re.search(r'car|vehicle|truck|bus|train|boat|ship|plane|airplane|helicopter|motorcycle|bicycle|bike', t): subjects.append("vehicle/transport") if re.search(r'ball|basketball|football|soccer|tennis', t): subjects.append("ball/sports") if re.search(r'animal|dog|cat|bird|fish|horse|insect|butterfly|bee', t): subjects.append("animal") if re.search(r'water|river|wave|ocean|rain|snow|cloud|smoke|fire|flame', t): subjects.append("nature/elements") if re.search(r'camera|drone|aerial', t): subjects.append("camera/drone") if re.search(r'robot|machine|mechanical', t): subjects.append("robot/machine") if not subjects: subjects.append("other") for s in subjects: dir_video_subjects[s] += 1 print("Direction video subject matter:") for subj, cnt in dir_video_subjects.most_common(): print(f" {subj}: {cnt}") # Same for speed videos print("\n--- What speed videos depict ---") spd_video_subjects = Counter() for q in speed_qs: t = q["question"].lower() + " " + q["answer"].lower() subjects = [] if re.search(r'man|person|people|woman|girl|boy|child|baby|walk|run', t): subjects.append("people") if re.search(r'car|vehicle|truck|bus|train|boat|ship|plane|airplane|helicopter|motorcycle|bicycle|bike|driv', t): subjects.append("vehicle/transport") if re.search(r'ball|basketball|football|soccer|tennis', t): subjects.append("ball/sports") if re.search(r'animal|dog|cat|bird|fish|horse|insect|butterfly|bee|turtle|snail', t): subjects.append("animal") if re.search(r'water|river|wave|ocean|rain|snow|cloud|smoke|fire|flame|wind', t): subjects.append("nature/elements") if re.search(r'camera|drone|aerial', t): subjects.append("camera/drone") if re.search(r'timelapse|time.lapse|time-lapse', t): subjects.append("timelapse content") if not subjects: subjects.append("other") for s in subjects: spd_video_subjects[s] += 1 print("Speed video subject matter:") for subj, cnt in spd_video_subjects.most_common(): print(f" {subj}: {cnt}") # ═══════════════════════════════════════════════════════════════════════ # SUMMARY # ═══════════════════════════════════════════════════════════════════════ print("\n" + "=" * 80) print("SUMMARY") print("=" * 80) print(f"Total dataset entries: {len(data)}") print(f"Dimensions: {dict(dim_counts)}") print(f"\nDirection questions: {len(direction_qs)} across {len(video_ids_dir)} unique videos") print(f" Top subtypes: {dir_subtype_counter.most_common(5)}") print(f"\nSpeed questions: {len(speed_qs)} across {len(video_ids_spd)} unique videos") print(f" Top subtypes: {spd_subtype_counter.most_common(5)}") print(f"\nAttribute change questions: {len(attr_change_qs)} across {len(video_ids_ac)} unique videos") # Questions per video distribution qs_per_video = Counter(q["video_id"] for q in data) print(f"\nQuestions per video: min={min(qs_per_video.values())}, max={max(qs_per_video.values())}, " f"mean={sum(qs_per_video.values())/len(qs_per_video):.1f}")