sources/qualification-snapshot/scripts/validate_caption_run_v2.py

"""Bind every visible caption delta to its producing token before scoring time."""
from probe_native_stream import token_table, token_oracle
from validate_caption_run import validate as validate_accounting, require


def validate(rows, observations, case, tokenizer, *, frozen_segments=None, expected_ids=None):
    table = token_table(tokenizer)
    active = None
    prior_visible = False
    global_count = 0
    for row in rows:
        kind = row['kind']
        if kind == 'caption_segment_start':
            require(active is None, 'overlapping caption segments')
            active = dict(id=row['segment'], base=global_count, ids=[], deltas=[])
        elif kind == 'caption_text_delta':
            require(active is not None and row['segment'] == active['id'], 'caption delta segment identity')
            require(type(row.get('index')) is int and row['index'] == global_count,
                    'caption delta not emitted at its declared token clock')
            require(type(row.get('text')) is str and bool(row['text']), 'empty/invalid caption delta')
            flush = row.get('utf8_final_flush', False)
            separator = row.get('segment_separator', False)
            require(type(flush) is bool and type(separator) is bool and not (flush and separator),
                    'caption delta flags differ')
            active['deltas'].append((row['index'] - active['base'], row['text'], flush, separator))
        elif kind == 'caption_window':
            require(active is not None and row['segment'] == active['id'], 'caption token segment identity')
            require(row['index'] == global_count, 'caption token clock differs')
            active['ids'].append(row['token_id'])
            global_count += 1
        elif kind == 'caption_segment_end':
            require(active is not None and row['segment'] == active['id'], 'caption end segment identity')
            oracle = token_oracle(active['ids'], table)
            expected = [(index, text, flush, False) for index, text, flush in oracle['deltas']]
            if expected:
                if prior_visible:
                    expected.insert(0, (expected[0][0], '\n', False, True))
                prior_visible = True
            require(active['deltas'] == expected, 'caption delta index/text/flush/separator differs from token oracle')
            active = None
    require(active is None, 'unfinished caption delta segment')
    result = validate_accounting(rows, observations, case, tokenizer,
                                frozen_segments=frozen_segments, expected_ids=expected_ids)
    result['independent_delta_token_clock_oracle'] = True
    return result