"""Bind every visible caption delta to its producing token before scoring time.""" from probe_native_stream import token_table, token_oracle from validate_caption_run import validate as validate_accounting, require def validate(rows, observations, case, tokenizer, *, frozen_segments=None, expected_ids=None): table = token_table(tokenizer) active = None prior_visible = False global_count = 0 for row in rows: kind = row['kind'] if kind == 'caption_segment_start': require(active is None, 'overlapping caption segments') active = dict(id=row['segment'], base=global_count, ids=[], deltas=[]) elif kind == 'caption_text_delta': require(active is not None and row['segment'] == active['id'], 'caption delta segment identity') require(type(row.get('index')) is int and row['index'] == global_count, 'caption delta not emitted at its declared token clock') require(type(row.get('text')) is str and bool(row['text']), 'empty/invalid caption delta') flush = row.get('utf8_final_flush', False) separator = row.get('segment_separator', False) require(type(flush) is bool and type(separator) is bool and not (flush and separator), 'caption delta flags differ') active['deltas'].append((row['index'] - active['base'], row['text'], flush, separator)) elif kind == 'caption_window': require(active is not None and row['segment'] == active['id'], 'caption token segment identity') require(row['index'] == global_count, 'caption token clock differs') active['ids'].append(row['token_id']) global_count += 1 elif kind == 'caption_segment_end': require(active is not None and row['segment'] == active['id'], 'caption end segment identity') oracle = token_oracle(active['ids'], table) expected = [(index, text, flush, False) for index, text, flush in oracle['deltas']] if expected: if prior_visible: expected.insert(0, (expected[0][0], '\n', False, True)) prior_visible = True require(active['deltas'] == expected, 'caption delta index/text/flush/separator differs from token oracle') active = None require(active is None, 'unfinished caption delta segment') result = validate_accounting(rows, observations, case, tokenizer, frozen_segments=frozen_segments, expected_ids=expected_ids) result['independent_delta_token_clock_oracle'] = True return result