File size: 6,725 Bytes
c49eca9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """Audit the full paired campaign, preserving errors and timing uncertainty."""
from pathlib import Path
import hashlib,json,math,struct,sys
from silero_qualification_v2 import load_frozen,sha
from caption_prefix_token_oracle_v1 import token_table,token_oracle
from probe_torch_stream_reference import score
from validate_caption_run_v2 import validate
ROOT=Path(__file__).resolve().parents[1];P=ROOT/'reports/q3-final-qualification-v1';QSHA='0c96a730cf50cce413033ff196959264134c63469d95fd1d81c137f66d3a8897'
def main():
frozen,cases=load_frozen(P/'qualification.json',QSHA);table=token_table(ROOT/'artifacts/native-tokenizer-v1.a8tok')
campaigns={arm:json.loads((P/arm/'campaign.json').read_text()) for arm in ('native','mlx','original')}
for arm,c in campaigns.items():
assert c['status']=='complete' and c['qualification_sha256']==QSHA and c['all_eleven_cases_attempted']
assert [x['id'] for x in c['cases']]==[x['id'] for x in cases] and all(x['status']=='complete' for x in c['cases'])
totals={arm:{lang:dict(errors=0,reference_units=0,substitutions=0,deletions=0,insertions=0,hits=0) for lang in ('en','zh')} for arm in campaigns}
records=[];lags={arm:[] for arm in ('native','mlx')}
for index,case in enumerate(cases):
segment_path=P/'segments'/case['id']/'segments.json';plan=json.loads(segment_path.read_text());assert plan['qualification_sha256']==QSHA
row=dict(id=case['id'],language=case['language'],arms={})
original_path=P/'original'/case['id']/'result.json';original=json.loads(original_path.read_text());entry=campaigns['original']['cases'][index]
assert sha(original_path)==entry['report_sha256'] and original['status']=='complete' and original['assets_unchanged']
candidate=P/'original'/case['id']/'candidate.json';assert sha(candidate)==entry['candidate_sha256'] and original['identity']==json.loads(candidate.read_text())['identity']
assert len(original['results'])==1;source=original['results'][0]
assert source['id']==case['id'] and source['input_sha256']==case['sha256'] and source['full_input_samples_observed']==case['samples']
assert source['all_expected_windows_emitted'] and source['full_reference_including_vad_omissions'] and len(source['segments'])==len(plan['segments'])
ids=[];texts=[]
for part,segment in zip(source['segments'],plan['segments'],strict=True):
ids.extend(part['generated_token_ids']);texts.append(token_oracle(part['generated_token_ids'],table)['text'])
assert texts[-1]==part['text'] and len(part['generated_token_ids'])==segment['windows']
assert ids==source['generated_token_ids'] and ' '.join(texts)==source['text']
assert hashlib.sha256(b''.join(struct.pack('<I',n) for n in ids)).hexdigest()==source['generated_ids_sha256']
original_events=P/'original'/case['id']/'result.jsonl';assert sha(original_events)==original['events']['sha256']
oe=[json.loads(l) for l in original_events.read_text().splitlines()]
assert all(e['all_unsuppressed_logits_finite'] for e in oe if e['kind']=='cuda_all_clock_logits')
for arm in campaigns:
if arm=='original':text=source['text'];details={}
else:
folder=P/arm/case['id'];r=json.loads((folder/'run.json').read_text());assert sha(folder/'run.json')==campaigns[arm]['cases'][index]['sha256']
raw=folder/'run.jsonl';observed=folder/'run.observed.jsonl';assert sha(raw)==r['logs']['raw']['sha256'] and sha(observed)==r['logs']['observed']['sha256']
rows=[json.loads(l) for l in raw.read_text().splitlines()];obs=[json.loads(l) for l in observed.read_text().splitlines()]
v=validate(rows,obs,case,ROOT/'artifacts/native-tokenizer-v1.a8tok',frozen_segments=plan['segments'])
assert v['score']==r['score'] and v['independent_delta_token_clock_oracle']
text=v['score']['raw_hypothesis'];lags[arm].extend(v['score']['alignment'])
windows=[w for w in rows if w['kind']=='caption_window']
mature=[w for w in windows if not w['eof_drain'] and w.get('batch_first_index',w['index'])-(w['index']-w['segment_index'])>=375]
mature_time=sum(w['work_seconds'] for w in mature);budget=len(mature)*.08
details=dict(memory=r['memory'],word_timing=v['score']['aligned_word_timing'],longest_deleted_reference_run=v['score']['longest_contiguous_deleted_reference_units'],max_queue_seconds=v['max_queue_seconds'],mature_windows=len(mature),mature_work_rtf=mature_time/budget if budget else None)
metric=score(case['reference'],text,case['language']);errors=sum(metric[k] for k in ('substitutions','deletions','insertions'))
if arm!='original':assert errors==v['score']['counts']['errors'] and metric['normalized_reference']==v['score']['normalized_reference']
target=totals[arm][case['language']]
for key in ('reference_units','substitutions','deletions','insertions','hits'):target[key]+=metric[key]
target['errors']+=errors;row['arms'][arm]=dict(text=text,errors=errors,reference_units=metric['reference_units'],error_rate=metric['error_rate'],**details)
records.append(row)
for arm in totals:
for lang,m in totals[arm].items():m['error_rate']=m['errors']/m['reference_units'] if m['reference_units'] else None
timing={}
for arm,values in lags.items():
ordered=sorted(v['lag_from_word_end_seconds'] for v in values);unambiguous=[v['lag_from_word_end_seconds'] for v in values if v['alignment_unambiguous']]
timing[arm]=dict(matched_units=len(values),ambiguous_matched_units=len(values)-len(unambiguous),p50_seconds=ordered[math.ceil(.5*len(ordered))-1] if ordered else None,p95_seconds=ordered[math.ceil(.95*len(ordered))-1] if ordered else None,maximum_seconds=max(ordered,default=None),matched_units_above_one_second=sum(x>1 for x in ordered),unambiguous_units_above_one_second=sum(x>1 for x in unambiguous),all_reference_units_within_one_second_proven=False)
report=dict(status='complete_verified_paired_evaluation',qualification_sha256=QSHA,production_ready=False,totals=totals,aligned_word_timing=timing,cases=records,no_post_result_selection=True,scope='All eleven prospectively frozen cases; complete original references and identical causal boundaries. CPU and MLX are paired with original CUDA BF16. Omissions and alignment uncertainty remain explicit; no automatic production acceptance.')
(P/'verification.json').write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n');print(json.dumps(dict(totals=totals,timing=timing)))
if __name__=='__main__':main()
|