code-HFD / benchmark /session_overlay.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
fb0074a verified
Raw History Blame Contribute Delete
6.5 kB
"""Attach reviewed external QA without changing a single motion/event target.
The original benchmark's visual timestamp is the END of the spoken question.
The visual stream starts at original frame zero. No history crop or time warp.
This is a spoken HFD adaptation, not the original multiple-choice protocol.
"""
import json
import math
from pathlib import Path
import sys
import numpy as np
from common import ROOT, sha256
sys.path[:0] = [str(ROOT), str(ROOT/'src')]
from psi.data.seed.live_multimodal_overlay import (
available_intervals, continuous_audio, interaction_task, question_wave,
safe_question_gain, seed_for,
)
def timing(row, intervals, question_seconds, session_end, last_packet):
end = float(row['question_time'])
start = end - question_seconds
first_reply = max(0, math.ceil((end-.6-1e-8)/.4))
final_reply = .6 + .4*(first_reply+2)
if row['duration_seconds'] < session_end+.2 or final_reply > last_packet+1e-7:
return None
if not any(a-1e-7 <= start and final_reply <= b+1e-7 for a,b in intervals):
return None
return start, end
def reviewed(row):
"""Human/machine provenance must say what was actually checked."""
review = json.loads(Path(row['visual_receipt']).read_text())
asr = json.loads(Path(row['asr_receipt']).read_text())
if row.get('direct_inspection_path'):
path=Path(row['direct_inspection_path'])
if sha256(path)!=row.get('direct_inspection_sha256'):return False
inspection=json.loads(path.read_text())
matches=[r for r in inspection['rows'] if r['id']==row['asset_id']]
if inspection.get('complete') is not True or len(matches)!=1 or matches[0].get('accepted') is not True:
return False
return bool(review.get('accepted') is True
and review.get('video_sha256') == row['video_sha256']
and review.get('question') == row['original_question']
and review.get('reference') == row['original_answer']
and review.get('question_time') == row['question_time']
and review.get('observed_prefix_only') is True
and asr.get('accepted') is True
and asr.get('wave_sha256') == row['question_audio_sha256'])
class ExternalOverlay:
def __init__(self, policy, seed=42):
self.policy, self.seed, self.by_voice = policy, seed, {}
def compose(self, original, forced_mode='qa'):
if forced_mode != 'qa':
raise ValueError('External benchmark requires a visual question')
source = dict(original)
source['steps'] = [dict(s) for s in original['steps']]
source['initial_abs_state'] = np.asarray(original['steps'][0]['states'], np.float32).reshape(-1).copy()
voice = source['session_voice_name']
intervals = available_intervals(source, self.policy.motion_lead_seconds, self.policy.motion_tail_seconds)
semantic = [s for s in source['steps'] if s['update_semantic_cache']]
last_packet = .6+.4*int(semantic[-1]['semantic_window_index'])
session_end = max(last_packet, (len(source['steps'])+30)/50.)
pool = self.by_voice.get(voice, ())
rng = np.random.default_rng(seed_for(source['source_id'], self.seed))
relation = dict(task=interaction_task(source), mode='audio-only', voice=voice,
fallback_reason='no_safe_external_question_interval')
for i in rng.permutation(len(pool)):
row = pool[int(i)]
clip = question_wave(row['question_audio_path'])
location = timing(row, intervals, len(clip)/16000., session_end, last_packet)
if location is None:
continue
if sha256(row['question_audio_path']) != row['question_audio_sha256']:
raise ValueError('External question PCM changed after ASR')
first,last = location
audio = continuous_audio(source)
clip = clip.copy()
gain = min(1., .045/max(float(np.sqrt(np.mean(clip**2))),1e-8))
edge = min(int(self.policy.question_fade_seconds*16000),len(clip)//4)
if edge:
clip[:edge] *= np.linspace(0.,1.,edge,dtype=np.float32)
clip[-edge:] *= np.linspace(1.,0.,edge,dtype=np.float32)
start = round(first*16000)
if start < 0 or start+len(clip)>len(audio):
raise ValueError('External question exceeds microphone timeline')
safe = safe_question_gain(audio[start:start+len(clip)],clip,gain)
if safe < gain*.1:
continue
audio[start:start+len(clip)] += clip*safe
audio = np.rint(audio*32768).clip(-32768,32767).astype(np.int16).astype(np.float32)/32768
for step in source['steps']:
if step['update_semantic_cache']:
offset = int(step['semantic_window_index'])*6400
step['audio_waveform'] = audio[offset:offset+9600]
relation = dict(task=interaction_task(source),mode='qa',voice=voice,asset_id=row['asset_id'],
image_path='',video_path=row['video_path'],camera_kind='dynamic',
camera_duration_seconds=row['duration_seconds'],session_required_seconds=session_end,
visual_features_path=row['visual_features_path'],visual_input_mode='cached',
question_audio_path=row['question_audio_path'],question=row['question'],answer=row['answer'],
question_start=start/16000.,question_end=(start+len(clip))/16000.,question_gain=safe,
answer_not_in_microphone=True,original_motion_unchanged=True,data_domain=row['data_domain'],
external_benchmark=dict(id=row['external_id'],question_time=row['question_time'],
original_question=row['original_question'],original_answer=row['original_answer'],
options=row['options'],correct_option=row['correct_option'],
protocol='spoken open QA; question ends at original query time; full prefix from t=0',
source_video_sha256=row['video_sha256'],review_receipt=row['visual_receipt'],
feature_sha256=row.get('visual_features_sha256'),
review_receipt_sha256=row.get('visual_receipt_sha256'),
question_audio_sha256=row['question_audio_sha256']))
break
source['multimodal_overlay'] = relation
source['source_modality'] = 'swa_visual_qa'
return source