File size: 6,496 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
"""Attach reviewed external QA without changing a single motion/event target.

The original benchmark's visual timestamp is the END of the spoken question.
The visual stream starts at original frame zero. No history crop or time warp.
This is a spoken HFD adaptation, not the original multiple-choice protocol.
"""
import json
import math
from pathlib import Path
import sys
import numpy as np
from common import ROOT, sha256
sys.path[:0] = [str(ROOT), str(ROOT/'src')]
from psi.data.seed.live_multimodal_overlay import (
    available_intervals, continuous_audio, interaction_task, question_wave,
    safe_question_gain, seed_for,
)


def timing(row, intervals, question_seconds, session_end, last_packet):
    end = float(row['question_time'])
    start = end - question_seconds
    first_reply = max(0, math.ceil((end-.6-1e-8)/.4))
    final_reply = .6 + .4*(first_reply+2)
    if row['duration_seconds'] < session_end+.2 or final_reply > last_packet+1e-7:
        return None
    if not any(a-1e-7 <= start and final_reply <= b+1e-7 for a,b in intervals):
        return None
    return start, end


def reviewed(row):
    """Human/machine provenance must say what was actually checked."""
    review = json.loads(Path(row['visual_receipt']).read_text())
    asr = json.loads(Path(row['asr_receipt']).read_text())
    if row.get('direct_inspection_path'):
        path=Path(row['direct_inspection_path'])
        if sha256(path)!=row.get('direct_inspection_sha256'):return False
        inspection=json.loads(path.read_text())
        matches=[r for r in inspection['rows'] if r['id']==row['asset_id']]
        if inspection.get('complete') is not True or len(matches)!=1 or matches[0].get('accepted') is not True:
            return False
    return bool(review.get('accepted') is True
        and review.get('video_sha256') == row['video_sha256']
        and review.get('question') == row['original_question']
        and review.get('reference') == row['original_answer']
        and review.get('question_time') == row['question_time']
        and review.get('observed_prefix_only') is True
        and asr.get('accepted') is True
        and asr.get('wave_sha256') == row['question_audio_sha256'])


class ExternalOverlay:
    def __init__(self, policy, seed=42):
        self.policy, self.seed, self.by_voice = policy, seed, {}

    def compose(self, original, forced_mode='qa'):
        if forced_mode != 'qa':
            raise ValueError('External benchmark requires a visual question')
        source = dict(original)
        source['steps'] = [dict(s) for s in original['steps']]
        source['initial_abs_state'] = np.asarray(original['steps'][0]['states'], np.float32).reshape(-1).copy()
        voice = source['session_voice_name']
        intervals = available_intervals(source, self.policy.motion_lead_seconds, self.policy.motion_tail_seconds)
        semantic = [s for s in source['steps'] if s['update_semantic_cache']]
        last_packet = .6+.4*int(semantic[-1]['semantic_window_index'])
        session_end = max(last_packet, (len(source['steps'])+30)/50.)
        pool = self.by_voice.get(voice, ())
        rng = np.random.default_rng(seed_for(source['source_id'], self.seed))
        relation = dict(task=interaction_task(source), mode='audio-only', voice=voice,
                        fallback_reason='no_safe_external_question_interval')
        for i in rng.permutation(len(pool)):
            row = pool[int(i)]
            clip = question_wave(row['question_audio_path'])
            location = timing(row, intervals, len(clip)/16000., session_end, last_packet)
            if location is None:
                continue
            if sha256(row['question_audio_path']) != row['question_audio_sha256']:
                raise ValueError('External question PCM changed after ASR')
            first,last = location
            audio = continuous_audio(source)
            clip = clip.copy()
            gain = min(1., .045/max(float(np.sqrt(np.mean(clip**2))),1e-8))
            edge = min(int(self.policy.question_fade_seconds*16000),len(clip)//4)
            if edge:
                clip[:edge] *= np.linspace(0.,1.,edge,dtype=np.float32)
                clip[-edge:] *= np.linspace(1.,0.,edge,dtype=np.float32)
            start = round(first*16000)
            if start < 0 or start+len(clip)>len(audio):
                raise ValueError('External question exceeds microphone timeline')
            safe = safe_question_gain(audio[start:start+len(clip)],clip,gain)
            if safe < gain*.1:
                continue
            audio[start:start+len(clip)] += clip*safe
            audio = np.rint(audio*32768).clip(-32768,32767).astype(np.int16).astype(np.float32)/32768
            for step in source['steps']:
                if step['update_semantic_cache']:
                    offset = int(step['semantic_window_index'])*6400
                    step['audio_waveform'] = audio[offset:offset+9600]
            relation = dict(task=interaction_task(source),mode='qa',voice=voice,asset_id=row['asset_id'],
                image_path='',video_path=row['video_path'],camera_kind='dynamic',
                camera_duration_seconds=row['duration_seconds'],session_required_seconds=session_end,
                visual_features_path=row['visual_features_path'],visual_input_mode='cached',
                question_audio_path=row['question_audio_path'],question=row['question'],answer=row['answer'],
                question_start=start/16000.,question_end=(start+len(clip))/16000.,question_gain=safe,
                answer_not_in_microphone=True,original_motion_unchanged=True,data_domain=row['data_domain'],
                external_benchmark=dict(id=row['external_id'],question_time=row['question_time'],
                    original_question=row['original_question'],original_answer=row['original_answer'],
                    options=row['options'],correct_option=row['correct_option'],
                    protocol='spoken open QA; question ends at original query time; full prefix from t=0',
                    source_video_sha256=row['video_sha256'],review_receipt=row['visual_receipt'],
                    feature_sha256=row.get('visual_features_sha256'),
                    review_receipt_sha256=row.get('visual_receipt_sha256'),
                    question_audio_sha256=row['question_audio_sha256']))
            break
        source['multimodal_overlay'] = relation
        source['source_modality'] = 'swa_visual_qa'
        return source