Download benchmark/session_overlay.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 6.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/session_overlay.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/session_overlay.py
-
curl -L -o session_overlay.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/session_overlay.py
6.5 kB
| """Attach reviewed external QA without changing a single motion/event target. | |
| The original benchmark's visual timestamp is the END of the spoken question. | |
| The visual stream starts at original frame zero. No history crop or time warp. | |
| This is a spoken HFD adaptation, not the original multiple-choice protocol. | |
| """ | |
| import json | |
| import math | |
| from pathlib import Path | |
| import sys | |
| import numpy as np | |
| from common import ROOT, sha256 | |
| sys.path[:0] = [str(ROOT), str(ROOT/'src')] | |
| from psi.data.seed.live_multimodal_overlay import ( | |
| available_intervals, continuous_audio, interaction_task, question_wave, | |
| safe_question_gain, seed_for, | |
| ) | |
| def timing(row, intervals, question_seconds, session_end, last_packet): | |
| end = float(row['question_time']) | |
| start = end - question_seconds | |
| first_reply = max(0, math.ceil((end-.6-1e-8)/.4)) | |
| final_reply = .6 + .4*(first_reply+2) | |
| if row['duration_seconds'] < session_end+.2 or final_reply > last_packet+1e-7: | |
| return None | |
| if not any(a-1e-7 <= start and final_reply <= b+1e-7 for a,b in intervals): | |
| return None | |
| return start, end | |
| def reviewed(row): | |
| """Human/machine provenance must say what was actually checked.""" | |
| review = json.loads(Path(row['visual_receipt']).read_text()) | |
| asr = json.loads(Path(row['asr_receipt']).read_text()) | |
| if row.get('direct_inspection_path'): | |
| path=Path(row['direct_inspection_path']) | |
| if sha256(path)!=row.get('direct_inspection_sha256'):return False | |
| inspection=json.loads(path.read_text()) | |
| matches=[r for r in inspection['rows'] if r['id']==row['asset_id']] | |
| if inspection.get('complete') is not True or len(matches)!=1 or matches[0].get('accepted') is not True: | |
| return False | |
| return bool(review.get('accepted') is True | |
| and review.get('video_sha256') == row['video_sha256'] | |
| and review.get('question') == row['original_question'] | |
| and review.get('reference') == row['original_answer'] | |
| and review.get('question_time') == row['question_time'] | |
| and review.get('observed_prefix_only') is True | |
| and asr.get('accepted') is True | |
| and asr.get('wave_sha256') == row['question_audio_sha256']) | |
| class ExternalOverlay: | |
| def __init__(self, policy, seed=42): | |
| self.policy, self.seed, self.by_voice = policy, seed, {} | |
| def compose(self, original, forced_mode='qa'): | |
| if forced_mode != 'qa': | |
| raise ValueError('External benchmark requires a visual question') | |
| source = dict(original) | |
| source['steps'] = [dict(s) for s in original['steps']] | |
| source['initial_abs_state'] = np.asarray(original['steps'][0]['states'], np.float32).reshape(-1).copy() | |
| voice = source['session_voice_name'] | |
| intervals = available_intervals(source, self.policy.motion_lead_seconds, self.policy.motion_tail_seconds) | |
| semantic = [s for s in source['steps'] if s['update_semantic_cache']] | |
| last_packet = .6+.4*int(semantic[-1]['semantic_window_index']) | |
| session_end = max(last_packet, (len(source['steps'])+30)/50.) | |
| pool = self.by_voice.get(voice, ()) | |
| rng = np.random.default_rng(seed_for(source['source_id'], self.seed)) | |
| relation = dict(task=interaction_task(source), mode='audio-only', voice=voice, | |
| fallback_reason='no_safe_external_question_interval') | |
| for i in rng.permutation(len(pool)): | |
| row = pool[int(i)] | |
| clip = question_wave(row['question_audio_path']) | |
| location = timing(row, intervals, len(clip)/16000., session_end, last_packet) | |
| if location is None: | |
| continue | |
| if sha256(row['question_audio_path']) != row['question_audio_sha256']: | |
| raise ValueError('External question PCM changed after ASR') | |
| first,last = location | |
| audio = continuous_audio(source) | |
| clip = clip.copy() | |
| gain = min(1., .045/max(float(np.sqrt(np.mean(clip**2))),1e-8)) | |
| edge = min(int(self.policy.question_fade_seconds*16000),len(clip)//4) | |
| if edge: | |
| clip[:edge] *= np.linspace(0.,1.,edge,dtype=np.float32) | |
| clip[-edge:] *= np.linspace(1.,0.,edge,dtype=np.float32) | |
| start = round(first*16000) | |
| if start < 0 or start+len(clip)>len(audio): | |
| raise ValueError('External question exceeds microphone timeline') | |
| safe = safe_question_gain(audio[start:start+len(clip)],clip,gain) | |
| if safe < gain*.1: | |
| continue | |
| audio[start:start+len(clip)] += clip*safe | |
| audio = np.rint(audio*32768).clip(-32768,32767).astype(np.int16).astype(np.float32)/32768 | |
| for step in source['steps']: | |
| if step['update_semantic_cache']: | |
| offset = int(step['semantic_window_index'])*6400 | |
| step['audio_waveform'] = audio[offset:offset+9600] | |
| relation = dict(task=interaction_task(source),mode='qa',voice=voice,asset_id=row['asset_id'], | |
| image_path='',video_path=row['video_path'],camera_kind='dynamic', | |
| camera_duration_seconds=row['duration_seconds'],session_required_seconds=session_end, | |
| visual_features_path=row['visual_features_path'],visual_input_mode='cached', | |
| question_audio_path=row['question_audio_path'],question=row['question'],answer=row['answer'], | |
| question_start=start/16000.,question_end=(start+len(clip))/16000.,question_gain=safe, | |
| answer_not_in_microphone=True,original_motion_unchanged=True,data_domain=row['data_domain'], | |
| external_benchmark=dict(id=row['external_id'],question_time=row['question_time'], | |
| original_question=row['original_question'],original_answer=row['original_answer'], | |
| options=row['options'],correct_option=row['correct_option'], | |
| protocol='spoken open QA; question ends at original query time; full prefix from t=0', | |
| source_video_sha256=row['video_sha256'],review_receipt=row['visual_receipt'], | |
| feature_sha256=row.get('visual_features_sha256'), | |
| review_receipt_sha256=row.get('visual_receipt_sha256'), | |
| question_audio_sha256=row['question_audio_sha256'])) | |
| break | |
| source['multimodal_overlay'] = relation | |
| source['source_modality'] = 'swa_visual_qa' | |
| return source | |