"""Benchmark-only session selection and source slicing; no model-dependent filters.""" from collections import Counter, defaultdict from functools import lru_cache import json import random import numpy as np from select_cohort import category TASKS = ('turn-taking', 'interruption', 'stop', 'backchannel', 'hesitation', 'pause', 'third-party') def task_of(session): r = session[0] if r.get('_third_party'): return 'third-party' return {'emergency-stop': 'stop'}.get(r.get('_session_interaction_mode'), r.get('_session_interaction_mode')) def quotas(count): # Six equally represented task families; stop is a subtype of interruption. families = ['turn-taking', 'interruption', 'backchannel', 'hesitation', 'pause', 'third-party'] q = {k: count // 6 + int(i < count % 6) for i, k in enumerate(families)} q['stop'] = q['interruption'] // 2 q['interruption'] -= q['stop'] return q def candidate_order(raw, seed): """Natural validation sessions, stratified by task/turn count/voice/semantics.""" rng = random.Random(seed) cells = defaultdict(list) for i, session in enumerate(raw.sessions): task = task_of(session) if task not in TASKS or not 1 <= len(session) <= 4: continue # Include subtype/profile as a sampling stratum, not a separate task. subtype = session[0].get('_session_disfluency_subtype', '') if task == 'third-party': subtype = raw.recipes[i-len(raw.raw_dataset)]['session_type'] cats = sorted(set(c for r in session for c in category(r['instruction']))) seconds = sum(float(r['declared_frames']) / 120 for r in session) key = (task, len(session), session[0]['_session_voice_name'], subtype, cats[int(session[0]['row_id']) % len(cats)], int(np.searchsorted([15,30,60], seconds))) cells[key].append(i) for values in cells.values(): rng.shuffle(values) keys = sorted(cells); rng.shuffle(keys) order = [] while any(cells.values()): for k in keys: if cells[k]: order.append(cells[k].pop()) return order @lru_cache(maxsize=8192) def receipt(path): with open(path) as f: return json.load(f) def quality_asset(row): """Require actual verification receipts, not just a published accepted flag.""" try: visual = receipt(row['visual_receipt']) asr = receipt(row['asr_receipt']) if visual.get('accepted') is not True or asr.get('accepted') is not True: return False if row.get('video_path'): live = receipt(row['live_qa_receipt']) if live.get('accepted') is not True: return False # A question must have observed-prefix evidence, not full-video-only review. if not row.get('causal_qa_points') and not row.get('live_qa_windows'): return False return True except (OSError, KeyError, ValueError): return False def motion_segments(source, records, controls): """GT-clock slices. Caption = final executed command, NEVER microphone concatenation. `controls` includes the initial state; exported/predicted motion starts at GT frame 1. Interrupted/Stop-target turns are excluded in their entirety. Physical whole-session metrics still include those turns and their seams. """ steps = source['steps'] turns = np.asarray([s['turn_index'] for s in steps], dtype=np.int64) turns = np.r_[turns, np.full(len(controls)-len(turns), turns[-1], dtype=np.int64)][1:] labels = np.asarray(controls)[1:] excluded = {} if source['interaction_mode'] == 'interruption': excluded.update({i: 'interrupted_by_next_command' for i in range(len(records)-1)}) for event in source.get('control_command_events', []): excluded[int(event['turn_index'])] = 'stopped_before_completion' result = [] for turn, record in enumerate(records): voice = source['session_voice_name'] text = record['instruction_zh' if voice.endswith('_zh') else 'instruction'].strip() targets = {s['latest_instruction_text'].strip() for s in steps if int(s['turn_index']) == turn and s.get('latest_instruction_text', '').strip()} if targets != {text}: raise ValueError(f'Final executed command disagrees with formal supervision: turn={turn}, {targets!r} vs {text!r}') frames = np.flatnonzero((turns == turn) & (labels == 1)) reason = excluded.get(turn) if not len(frames): reason = reason or 'no_motion_frames' elif len(frames) < 50: reason = reason or 'less_than_one_second' if reason is None and np.any(np.diff(frames) != 1): raise ValueError('Complete motion unexpectedly has disconnected intervals') result.append(dict(turn_index=turn, motion_name=record['filename'], original_row_id=int(record['row_id']), caption=record['instruction'].strip(), final_instruction=text, caption_language='en', spoken_language='zh' if voice.endswith('_zh') else 'en', semantic_categories=category(record['instruction']), eligible=reason is None, exclusion_reason=reason, score_start=int(frames[0]) if len(frames) else None, score_end=int(frames[-1]+1) if len(frames) else None)) return result, turns def flattened_rows(rows): """Every eligible source occurrence has weight one, independent of its session.""" result = [] for row in rows: if 'segments' not in row: # Old adapter tests / explicit legacy compatibility. result.append(row); continue for segment in row['segments']: if segment['eligible']: result.append(dict(row, **segment, parent_sample_id=row['sample_id'], sample_id=f"{row['sample_id']}_turn{segment['turn_index']}")) # Shuffle BEFORE retrieval batching; otherwise same-session items cluster. random.Random(42).shuffle(result) return result