"""Attach reviewed external QA without changing a single motion/event target. The original benchmark's visual timestamp is the END of the spoken question. The visual stream starts at original frame zero. No history crop or time warp. This is a spoken HFD adaptation, not the original multiple-choice protocol. """ import json import math from pathlib import Path import sys import numpy as np from common import ROOT, sha256 sys.path[:0] = [str(ROOT), str(ROOT/'src')] from psi.data.seed.live_multimodal_overlay import ( available_intervals, continuous_audio, interaction_task, question_wave, safe_question_gain, seed_for, ) def timing(row, intervals, question_seconds, session_end, last_packet): end = float(row['question_time']) start = end - question_seconds first_reply = max(0, math.ceil((end-.6-1e-8)/.4)) final_reply = .6 + .4*(first_reply+2) if row['duration_seconds'] < session_end+.2 or final_reply > last_packet+1e-7: return None if not any(a-1e-7 <= start and final_reply <= b+1e-7 for a,b in intervals): return None return start, end def reviewed(row): """Human/machine provenance must say what was actually checked.""" review = json.loads(Path(row['visual_receipt']).read_text()) asr = json.loads(Path(row['asr_receipt']).read_text()) if row.get('direct_inspection_path'): path=Path(row['direct_inspection_path']) if sha256(path)!=row.get('direct_inspection_sha256'):return False inspection=json.loads(path.read_text()) matches=[r for r in inspection['rows'] if r['id']==row['asset_id']] if inspection.get('complete') is not True or len(matches)!=1 or matches[0].get('accepted') is not True: return False return bool(review.get('accepted') is True and review.get('video_sha256') == row['video_sha256'] and review.get('question') == row['original_question'] and review.get('reference') == row['original_answer'] and review.get('question_time') == row['question_time'] and review.get('observed_prefix_only') is True and asr.get('accepted') is True and asr.get('wave_sha256') == row['question_audio_sha256']) class ExternalOverlay: def __init__(self, policy, seed=42): self.policy, self.seed, self.by_voice = policy, seed, {} def compose(self, original, forced_mode='qa'): if forced_mode != 'qa': raise ValueError('External benchmark requires a visual question') source = dict(original) source['steps'] = [dict(s) for s in original['steps']] source['initial_abs_state'] = np.asarray(original['steps'][0]['states'], np.float32).reshape(-1).copy() voice = source['session_voice_name'] intervals = available_intervals(source, self.policy.motion_lead_seconds, self.policy.motion_tail_seconds) semantic = [s for s in source['steps'] if s['update_semantic_cache']] last_packet = .6+.4*int(semantic[-1]['semantic_window_index']) session_end = max(last_packet, (len(source['steps'])+30)/50.) pool = self.by_voice.get(voice, ()) rng = np.random.default_rng(seed_for(source['source_id'], self.seed)) relation = dict(task=interaction_task(source), mode='audio-only', voice=voice, fallback_reason='no_safe_external_question_interval') for i in rng.permutation(len(pool)): row = pool[int(i)] clip = question_wave(row['question_audio_path']) location = timing(row, intervals, len(clip)/16000., session_end, last_packet) if location is None: continue if sha256(row['question_audio_path']) != row['question_audio_sha256']: raise ValueError('External question PCM changed after ASR') first,last = location audio = continuous_audio(source) clip = clip.copy() gain = min(1., .045/max(float(np.sqrt(np.mean(clip**2))),1e-8)) edge = min(int(self.policy.question_fade_seconds*16000),len(clip)//4) if edge: clip[:edge] *= np.linspace(0.,1.,edge,dtype=np.float32) clip[-edge:] *= np.linspace(1.,0.,edge,dtype=np.float32) start = round(first*16000) if start < 0 or start+len(clip)>len(audio): raise ValueError('External question exceeds microphone timeline') safe = safe_question_gain(audio[start:start+len(clip)],clip,gain) if safe < gain*.1: continue audio[start:start+len(clip)] += clip*safe audio = np.rint(audio*32768).clip(-32768,32767).astype(np.int16).astype(np.float32)/32768 for step in source['steps']: if step['update_semantic_cache']: offset = int(step['semantic_window_index'])*6400 step['audio_waveform'] = audio[offset:offset+9600] relation = dict(task=interaction_task(source),mode='qa',voice=voice,asset_id=row['asset_id'], image_path='',video_path=row['video_path'],camera_kind='dynamic', camera_duration_seconds=row['duration_seconds'],session_required_seconds=session_end, visual_features_path=row['visual_features_path'],visual_input_mode='cached', question_audio_path=row['question_audio_path'],question=row['question'],answer=row['answer'], question_start=start/16000.,question_end=(start+len(clip))/16000.,question_gain=safe, answer_not_in_microphone=True,original_motion_unchanged=True,data_domain=row['data_domain'], external_benchmark=dict(id=row['external_id'],question_time=row['question_time'], original_question=row['original_question'],original_answer=row['original_answer'], options=row['options'],correct_option=row['correct_option'], protocol='spoken open QA; question ends at original query time; full prefix from t=0', source_video_sha256=row['video_sha256'],review_receipt=row['visual_receipt'], feature_sha256=row.get('visual_features_sha256'), review_receipt_sha256=row.get('visual_receipt_sha256'), question_audio_sha256=row['question_audio_sha256'])) break source['multimodal_overlay'] = relation source['source_modality'] = 'swa_visual_qa' return source