| """Audit the full paired campaign, preserving errors and timing uncertainty.""" |
| from pathlib import Path |
| import hashlib,json,math,struct,sys |
| from silero_qualification_v2 import load_frozen,sha |
| from caption_prefix_token_oracle_v1 import token_table,token_oracle |
| from probe_torch_stream_reference import score |
| from validate_caption_run_v2 import validate |
| ROOT=Path(__file__).resolve().parents[1];P=ROOT/'reports/q3-final-qualification-v1';QSHA='0c96a730cf50cce413033ff196959264134c63469d95fd1d81c137f66d3a8897' |
| def main(): |
| frozen,cases=load_frozen(P/'qualification.json',QSHA);table=token_table(ROOT/'artifacts/native-tokenizer-v1.a8tok') |
| campaigns={arm:json.loads((P/arm/'campaign.json').read_text()) for arm in ('native','mlx','original')} |
| for arm,c in campaigns.items(): |
| assert c['status']=='complete' and c['qualification_sha256']==QSHA and c['all_eleven_cases_attempted'] |
| assert [x['id'] for x in c['cases']]==[x['id'] for x in cases] and all(x['status']=='complete' for x in c['cases']) |
| totals={arm:{lang:dict(errors=0,reference_units=0,substitutions=0,deletions=0,insertions=0,hits=0) for lang in ('en','zh')} for arm in campaigns} |
| records=[];lags={arm:[] for arm in ('native','mlx')} |
| for index,case in enumerate(cases): |
| segment_path=P/'segments'/case['id']/'segments.json';plan=json.loads(segment_path.read_text());assert plan['qualification_sha256']==QSHA |
| row=dict(id=case['id'],language=case['language'],arms={}) |
| original_path=P/'original'/case['id']/'result.json';original=json.loads(original_path.read_text());entry=campaigns['original']['cases'][index] |
| assert sha(original_path)==entry['report_sha256'] and original['status']=='complete' and original['assets_unchanged'] |
| candidate=P/'original'/case['id']/'candidate.json';assert sha(candidate)==entry['candidate_sha256'] and original['identity']==json.loads(candidate.read_text())['identity'] |
| assert len(original['results'])==1;source=original['results'][0] |
| assert source['id']==case['id'] and source['input_sha256']==case['sha256'] and source['full_input_samples_observed']==case['samples'] |
| assert source['all_expected_windows_emitted'] and source['full_reference_including_vad_omissions'] and len(source['segments'])==len(plan['segments']) |
| ids=[];texts=[] |
| for part,segment in zip(source['segments'],plan['segments'],strict=True): |
| ids.extend(part['generated_token_ids']);texts.append(token_oracle(part['generated_token_ids'],table)['text']) |
| assert texts[-1]==part['text'] and len(part['generated_token_ids'])==segment['windows'] |
| assert ids==source['generated_token_ids'] and ' '.join(texts)==source['text'] |
| assert hashlib.sha256(b''.join(struct.pack('<I',n) for n in ids)).hexdigest()==source['generated_ids_sha256'] |
| original_events=P/'original'/case['id']/'result.jsonl';assert sha(original_events)==original['events']['sha256'] |
| oe=[json.loads(l) for l in original_events.read_text().splitlines()] |
| assert all(e['all_unsuppressed_logits_finite'] for e in oe if e['kind']=='cuda_all_clock_logits') |
| for arm in campaigns: |
| if arm=='original':text=source['text'];details={} |
| else: |
| folder=P/arm/case['id'];r=json.loads((folder/'run.json').read_text());assert sha(folder/'run.json')==campaigns[arm]['cases'][index]['sha256'] |
| raw=folder/'run.jsonl';observed=folder/'run.observed.jsonl';assert sha(raw)==r['logs']['raw']['sha256'] and sha(observed)==r['logs']['observed']['sha256'] |
| rows=[json.loads(l) for l in raw.read_text().splitlines()];obs=[json.loads(l) for l in observed.read_text().splitlines()] |
| v=validate(rows,obs,case,ROOT/'artifacts/native-tokenizer-v1.a8tok',frozen_segments=plan['segments']) |
| assert v['score']==r['score'] and v['independent_delta_token_clock_oracle'] |
| text=v['score']['raw_hypothesis'];lags[arm].extend(v['score']['alignment']) |
| windows=[w for w in rows if w['kind']=='caption_window'] |
| mature=[w for w in windows if not w['eof_drain'] and w.get('batch_first_index',w['index'])-(w['index']-w['segment_index'])>=375] |
| mature_time=sum(w['work_seconds'] for w in mature);budget=len(mature)*.08 |
| details=dict(memory=r['memory'],word_timing=v['score']['aligned_word_timing'],longest_deleted_reference_run=v['score']['longest_contiguous_deleted_reference_units'],max_queue_seconds=v['max_queue_seconds'],mature_windows=len(mature),mature_work_rtf=mature_time/budget if budget else None) |
| metric=score(case['reference'],text,case['language']);errors=sum(metric[k] for k in ('substitutions','deletions','insertions')) |
| if arm!='original':assert errors==v['score']['counts']['errors'] and metric['normalized_reference']==v['score']['normalized_reference'] |
| target=totals[arm][case['language']] |
| for key in ('reference_units','substitutions','deletions','insertions','hits'):target[key]+=metric[key] |
| target['errors']+=errors;row['arms'][arm]=dict(text=text,errors=errors,reference_units=metric['reference_units'],error_rate=metric['error_rate'],**details) |
| records.append(row) |
| for arm in totals: |
| for lang,m in totals[arm].items():m['error_rate']=m['errors']/m['reference_units'] if m['reference_units'] else None |
| timing={} |
| for arm,values in lags.items(): |
| ordered=sorted(v['lag_from_word_end_seconds'] for v in values);unambiguous=[v['lag_from_word_end_seconds'] for v in values if v['alignment_unambiguous']] |
| timing[arm]=dict(matched_units=len(values),ambiguous_matched_units=len(values)-len(unambiguous),p50_seconds=ordered[math.ceil(.5*len(ordered))-1] if ordered else None,p95_seconds=ordered[math.ceil(.95*len(ordered))-1] if ordered else None,maximum_seconds=max(ordered,default=None),matched_units_above_one_second=sum(x>1 for x in ordered),unambiguous_units_above_one_second=sum(x>1 for x in unambiguous),all_reference_units_within_one_second_proven=False) |
| report=dict(status='complete_verified_paired_evaluation',qualification_sha256=QSHA,production_ready=False,totals=totals,aligned_word_timing=timing,cases=records,no_post_result_selection=True,scope='All eleven prospectively frozen cases; complete original references and identical causal boundaries. CPU and MLX are paired with original CUDA BF16. Omissions and alignment uncertainty remain explicit; no automatic production acceptance.') |
| (P/'verification.json').write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n');print(json.dumps(dict(totals=totals,timing=timing))) |
| if __name__=='__main__':main() |
|
|
