Download benchmark/session_data.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 6.09 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/session_data.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/session_data.py
-
curl -L -o session_data.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/session_data.py
6.09 kB
| """Benchmark-only session selection and source slicing; no model-dependent filters.""" | |
| from collections import Counter, defaultdict | |
| from functools import lru_cache | |
| import json | |
| import random | |
| import numpy as np | |
| from select_cohort import category | |
| TASKS = ('turn-taking', 'interruption', 'stop', 'backchannel', 'hesitation', 'pause', 'third-party') | |
| def task_of(session): | |
| r = session[0] | |
| if r.get('_third_party'): | |
| return 'third-party' | |
| return {'emergency-stop': 'stop'}.get(r.get('_session_interaction_mode'), r.get('_session_interaction_mode')) | |
| def quotas(count): | |
| # Six equally represented task families; stop is a subtype of interruption. | |
| families = ['turn-taking', 'interruption', 'backchannel', 'hesitation', 'pause', 'third-party'] | |
| q = {k: count // 6 + int(i < count % 6) for i, k in enumerate(families)} | |
| q['stop'] = q['interruption'] // 2 | |
| q['interruption'] -= q['stop'] | |
| return q | |
| def candidate_order(raw, seed): | |
| """Natural validation sessions, stratified by task/turn count/voice/semantics.""" | |
| rng = random.Random(seed) | |
| cells = defaultdict(list) | |
| for i, session in enumerate(raw.sessions): | |
| task = task_of(session) | |
| if task not in TASKS or not 1 <= len(session) <= 4: | |
| continue | |
| # Include subtype/profile as a sampling stratum, not a separate task. | |
| subtype = session[0].get('_session_disfluency_subtype', '') | |
| if task == 'third-party': | |
| subtype = raw.recipes[i-len(raw.raw_dataset)]['session_type'] | |
| cats = sorted(set(c for r in session for c in category(r['instruction']))) | |
| seconds = sum(float(r['declared_frames']) / 120 for r in session) | |
| key = (task, len(session), session[0]['_session_voice_name'], subtype, | |
| cats[int(session[0]['row_id']) % len(cats)], int(np.searchsorted([15,30,60], seconds))) | |
| cells[key].append(i) | |
| for values in cells.values(): | |
| rng.shuffle(values) | |
| keys = sorted(cells); rng.shuffle(keys) | |
| order = [] | |
| while any(cells.values()): | |
| for k in keys: | |
| if cells[k]: | |
| order.append(cells[k].pop()) | |
| return order | |
| def receipt(path): | |
| with open(path) as f: | |
| return json.load(f) | |
| def quality_asset(row): | |
| """Require actual verification receipts, not just a published accepted flag.""" | |
| try: | |
| visual = receipt(row['visual_receipt']) | |
| asr = receipt(row['asr_receipt']) | |
| if visual.get('accepted') is not True or asr.get('accepted') is not True: | |
| return False | |
| if row.get('video_path'): | |
| live = receipt(row['live_qa_receipt']) | |
| if live.get('accepted') is not True: | |
| return False | |
| # A question must have observed-prefix evidence, not full-video-only review. | |
| if not row.get('causal_qa_points') and not row.get('live_qa_windows'): | |
| return False | |
| return True | |
| except (OSError, KeyError, ValueError): | |
| return False | |
| def motion_segments(source, records, controls): | |
| """GT-clock slices. Caption = final executed command, NEVER microphone concatenation. | |
| `controls` includes the initial state; exported/predicted motion starts at | |
| GT frame 1. Interrupted/Stop-target turns are excluded in their entirety. | |
| Physical whole-session metrics still include those turns and their seams. | |
| """ | |
| steps = source['steps'] | |
| turns = np.asarray([s['turn_index'] for s in steps], dtype=np.int64) | |
| turns = np.r_[turns, np.full(len(controls)-len(turns), turns[-1], dtype=np.int64)][1:] | |
| labels = np.asarray(controls)[1:] | |
| excluded = {} | |
| if source['interaction_mode'] == 'interruption': | |
| excluded.update({i: 'interrupted_by_next_command' for i in range(len(records)-1)}) | |
| for event in source.get('control_command_events', []): | |
| excluded[int(event['turn_index'])] = 'stopped_before_completion' | |
| result = [] | |
| for turn, record in enumerate(records): | |
| voice = source['session_voice_name'] | |
| text = record['instruction_zh' if voice.endswith('_zh') else 'instruction'].strip() | |
| targets = {s['latest_instruction_text'].strip() for s in steps | |
| if int(s['turn_index']) == turn and s.get('latest_instruction_text', '').strip()} | |
| if targets != {text}: | |
| raise ValueError(f'Final executed command disagrees with formal supervision: turn={turn}, {targets!r} vs {text!r}') | |
| frames = np.flatnonzero((turns == turn) & (labels == 1)) | |
| reason = excluded.get(turn) | |
| if not len(frames): | |
| reason = reason or 'no_motion_frames' | |
| elif len(frames) < 50: | |
| reason = reason or 'less_than_one_second' | |
| if reason is None and np.any(np.diff(frames) != 1): | |
| raise ValueError('Complete motion unexpectedly has disconnected intervals') | |
| result.append(dict(turn_index=turn, motion_name=record['filename'], original_row_id=int(record['row_id']), | |
| caption=record['instruction'].strip(), final_instruction=text, | |
| caption_language='en', spoken_language='zh' if voice.endswith('_zh') else 'en', | |
| semantic_categories=category(record['instruction']), eligible=reason is None, exclusion_reason=reason, | |
| score_start=int(frames[0]) if len(frames) else None, | |
| score_end=int(frames[-1]+1) if len(frames) else None)) | |
| return result, turns | |
| def flattened_rows(rows): | |
| """Every eligible source occurrence has weight one, independent of its session.""" | |
| result = [] | |
| for row in rows: | |
| if 'segments' not in row: # Old adapter tests / explicit legacy compatibility. | |
| result.append(row); continue | |
| for segment in row['segments']: | |
| if segment['eligible']: | |
| result.append(dict(row, **segment, parent_sample_id=row['sample_id'], | |
| sample_id=f"{row['sample_id']}_turn{segment['turn_index']}")) | |
| # Shuffle BEFORE retrieval batching; otherwise same-session items cluster. | |
| random.Random(42).shuffle(result) | |
| return result | |