code-HFD / benchmark /session_data.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
fb0074a verified
Raw History Blame Contribute Delete
6.09 kB
"""Benchmark-only session selection and source slicing; no model-dependent filters."""
from collections import Counter, defaultdict
from functools import lru_cache
import json
import random
import numpy as np
from select_cohort import category
TASKS = ('turn-taking', 'interruption', 'stop', 'backchannel', 'hesitation', 'pause', 'third-party')
def task_of(session):
r = session[0]
if r.get('_third_party'):
return 'third-party'
return {'emergency-stop': 'stop'}.get(r.get('_session_interaction_mode'), r.get('_session_interaction_mode'))
def quotas(count):
# Six equally represented task families; stop is a subtype of interruption.
families = ['turn-taking', 'interruption', 'backchannel', 'hesitation', 'pause', 'third-party']
q = {k: count // 6 + int(i < count % 6) for i, k in enumerate(families)}
q['stop'] = q['interruption'] // 2
q['interruption'] -= q['stop']
return q
def candidate_order(raw, seed):
"""Natural validation sessions, stratified by task/turn count/voice/semantics."""
rng = random.Random(seed)
cells = defaultdict(list)
for i, session in enumerate(raw.sessions):
task = task_of(session)
if task not in TASKS or not 1 <= len(session) <= 4:
continue
# Include subtype/profile as a sampling stratum, not a separate task.
subtype = session[0].get('_session_disfluency_subtype', '')
if task == 'third-party':
subtype = raw.recipes[i-len(raw.raw_dataset)]['session_type']
cats = sorted(set(c for r in session for c in category(r['instruction'])))
seconds = sum(float(r['declared_frames']) / 120 for r in session)
key = (task, len(session), session[0]['_session_voice_name'], subtype,
cats[int(session[0]['row_id']) % len(cats)], int(np.searchsorted([15,30,60], seconds)))
cells[key].append(i)
for values in cells.values():
rng.shuffle(values)
keys = sorted(cells); rng.shuffle(keys)
order = []
while any(cells.values()):
for k in keys:
if cells[k]:
order.append(cells[k].pop())
return order
@lru_cache(maxsize=8192)
def receipt(path):
with open(path) as f:
return json.load(f)
def quality_asset(row):
"""Require actual verification receipts, not just a published accepted flag."""
try:
visual = receipt(row['visual_receipt'])
asr = receipt(row['asr_receipt'])
if visual.get('accepted') is not True or asr.get('accepted') is not True:
return False
if row.get('video_path'):
live = receipt(row['live_qa_receipt'])
if live.get('accepted') is not True:
return False
# A question must have observed-prefix evidence, not full-video-only review.
if not row.get('causal_qa_points') and not row.get('live_qa_windows'):
return False
return True
except (OSError, KeyError, ValueError):
return False
def motion_segments(source, records, controls):
"""GT-clock slices. Caption = final executed command, NEVER microphone concatenation.
`controls` includes the initial state; exported/predicted motion starts at
GT frame 1. Interrupted/Stop-target turns are excluded in their entirety.
Physical whole-session metrics still include those turns and their seams.
"""
steps = source['steps']
turns = np.asarray([s['turn_index'] for s in steps], dtype=np.int64)
turns = np.r_[turns, np.full(len(controls)-len(turns), turns[-1], dtype=np.int64)][1:]
labels = np.asarray(controls)[1:]
excluded = {}
if source['interaction_mode'] == 'interruption':
excluded.update({i: 'interrupted_by_next_command' for i in range(len(records)-1)})
for event in source.get('control_command_events', []):
excluded[int(event['turn_index'])] = 'stopped_before_completion'
result = []
for turn, record in enumerate(records):
voice = source['session_voice_name']
text = record['instruction_zh' if voice.endswith('_zh') else 'instruction'].strip()
targets = {s['latest_instruction_text'].strip() for s in steps
if int(s['turn_index']) == turn and s.get('latest_instruction_text', '').strip()}
if targets != {text}:
raise ValueError(f'Final executed command disagrees with formal supervision: turn={turn}, {targets!r} vs {text!r}')
frames = np.flatnonzero((turns == turn) & (labels == 1))
reason = excluded.get(turn)
if not len(frames):
reason = reason or 'no_motion_frames'
elif len(frames) < 50:
reason = reason or 'less_than_one_second'
if reason is None and np.any(np.diff(frames) != 1):
raise ValueError('Complete motion unexpectedly has disconnected intervals')
result.append(dict(turn_index=turn, motion_name=record['filename'], original_row_id=int(record['row_id']),
caption=record['instruction'].strip(), final_instruction=text,
caption_language='en', spoken_language='zh' if voice.endswith('_zh') else 'en',
semantic_categories=category(record['instruction']), eligible=reason is None, exclusion_reason=reason,
score_start=int(frames[0]) if len(frames) else None,
score_end=int(frames[-1]+1) if len(frames) else None))
return result, turns
def flattened_rows(rows):
"""Every eligible source occurrence has weight one, independent of its session."""
result = []
for row in rows:
if 'segments' not in row: # Old adapter tests / explicit legacy compatibility.
result.append(row); continue
for segment in row['segments']:
if segment['eligible']:
result.append(dict(row, **segment, parent_sample_id=row['sample_id'],
sample_id=f"{row['sample_id']}_turn{segment['turn_index']}"))
# Shuffle BEFORE retrieval batching; otherwise same-session items cluster.
random.Random(42).shuffle(result)
return result