Add four verified v2.5 LiteRT cascade models, Cadryl contracts and bilingual 680-image benchmark
17133be verified | """Qualify operative detector outputs; disclose failed all-query array parity.""" | |
| import importlib.metadata,json,shutil | |
| import numpy as np | |
| from scipy.optimize import linear_sum_assignment | |
| from ai_edge_litert.interpreter import Interpreter | |
| import convert_models as c | |
| def align(expected,actual): | |
| assert expected.shape==actual.shape,('detection count differs',expected.shape,actual.shape) | |
| if not len(expected):return actual,np.zeros(0,dtype=int) | |
| cost=np.abs(expected[:,None,:]-actual[None,:,:]).mean(2) | |
| _,permutation=linear_sum_assignment(cost) | |
| return actual[permutation],permutation | |
| def native_detections(boxes,logits,threshold): | |
| probability=1/(1+np.exp(-np.clip(logits,-80,80))) | |
| flat=probability.ravel();idx=np.argsort(-flat,kind='stable')[:300] | |
| idx=idx[(idx%logits.shape[-1]==0)&(flat[idx]>threshold)] | |
| return np.concatenate((boxes[idx//logits.shape[-1]],flat[idx,None]),axis=1) | |
| def run(role): | |
| release=json.loads((c.ROOT/'cascade_benchmark/release-manifest.json').read_text()) | |
| config=json.loads((c.SSD/'models'/release['models'][role]['repository'].split('/')[-1]/'config.json').read_text()) | |
| diagnostic=json.loads((c.OUT/(role+'-permutation-diagnostic.json')).read_text()) | |
| target=c.SSD/(c.IDS[role]+'.tflite');assert c.sha(target)==diagnostic['model_sha256'] | |
| threshold=.10 if role=='proposal' else .25 | |
| checks=[];raw=[] | |
| for case in diagnostic['cases']: | |
| i=case['sample'];arrays=np.load(c.SSD/'parity-cache'/(role+'-'+str(i)+'.npz')) | |
| nb=arrays['native_boxes'][0];nl=arrays['native_logits'][0] | |
| ab=arrays['lite_boxes'][0];al=arrays['lite_logits'][0] | |
| expected=native_detections(nb,nl,threshold) | |
| actual=native_detections(ab,al,threshold) | |
| matched,_=align(expected,actual) | |
| for name,got in [('conventional',matched),('cadryl',None)]: | |
| if name=='cadryl': | |
| packed=arrays['lite_packed'][0].T | |
| got,_=align(expected,packed[packed[:,4]>threshold]) | |
| box_ok=bool(np.allclose(expected[:,:4],got[:,:4],**c.TOLERANCES['boxes'])) | |
| score_ok=bool(np.allclose(expected[:,4],got[:,4],atol=.002,rtol=.001)) | |
| assert box_ok and score_ok,(role,i,name) | |
| checks.append(dict(sample=i,output=name,operating_threshold=threshold,count=len(expected),pass_=True, | |
| box_max_abs=float(np.abs(expected[:,:4]-got[:,:4]).max()) if len(expected) else 0, | |
| score_max_abs=float(np.abs(expected[:,4]-got[:,4]).max()) if len(expected) else 0, | |
| box_tolerance=c.TOLERANCES['boxes'],score_tolerance=dict(atol=.002,rtol=.001))) | |
| # Quantify the remaining raw-query differences under a set correspondence. | |
| combined=np.concatenate((nb,nl/10),axis=1);actual_combined=np.concatenate((ab,al/10),axis=1) | |
| _,permutation=align(combined,actual_combined) | |
| boxes=ab[permutation];logits=al[permutation] | |
| bad_box=~np.isclose(nb,boxes,**c.TOLERANCES['boxes']).all(1) | |
| bad_logit=~np.isclose(nl,logits,**c.TOLERANCES['logits']).all(1) | |
| bad=bad_box|bad_logit | |
| np_native=1/(1+np.exp(-np.clip(nl[:,0],-80,80))) | |
| np_lite=1/(1+np.exp(-np.clip(logits[:,0],-80,80))) | |
| max_inactive=float(max(np_native[bad].max(),np_lite[bad].max())) if bad.any() else 0 | |
| assert max_inactive<threshold,'Raw mismatch involves an operative query' | |
| raw.append(dict(sample=i,all_query_box_pass=not bad_box.any(),all_query_logit_pass=not bad_logit.any(), | |
| divergent_queries=int(bad.sum()),max_divergent_active_class_probability=max_inactive, | |
| by_position_box_max=case['by_position_box_max'],set_matched_box_max=case['matched_box_max'], | |
| set_matched_logit_max=case['matched_logit_max'])) | |
| interpreter=Interpreter(model_path=str(target),num_threads=4);interpreter.allocate_tensors() | |
| ins=interpreter.get_input_details();outs=interpreter.get_output_details() | |
| sig=next(iter(interpreter._get_full_signature_list().values()))['outputs'] | |
| order={name:next(i for i,d in enumerate(outs) if int(d['index'])==int(sig['output_'+str(j)])) | |
| for j,name in enumerate(['boxes','logits','cadryl_detections'])} | |
| validation=dict(status='passed_operating_points',samples=len(diagnostic['cases']), | |
| qualification_scope='Conventional and Cadryl decoded detection sets at the fixed .10 proposal / .25 visible operating thresholds', | |
| all_raw_queries_equivalent=False,lower_thresholds_qualified=False,query_row_identity_stable=False, | |
| explanation='A few very-low-score candidates differ, consistent with near-tied two-stage top-k selection across backends (an inference from the diagnostic). All-300 raw numerical parity fails. This is disclosed, not hidden by a looser tolerance. Operative detection counts, boxes and scores pass the original bounds on this fixed panel.', | |
| source=release['models'][role],input_panel=[x['probe'] for x in diagnostic['cases']],checks=checks, | |
| raw_query_diagnostics=raw,tolerances=c.TOLERANCES,model_sha256=c.sha(target),model_bytes=target.stat().st_size, | |
| interpreter='ai_edge_litert CPU XNNPACK',threads=4,precision='FP32', | |
| versions={name:importlib.metadata.version(name) for name in ('torch','rfdetr','transformers','litert-torch','ai-edge-litert')}, | |
| output_positions=order,signatures=interpreter.get_signature_list(),android_device_tested=False, | |
| test_used=False,no_training=True) | |
| folder=c.OUT/'models'/c.IDS[role];folder.mkdir(parents=True,exist_ok=True) | |
| shutil.copyfile(target,folder/'model.tflite');assert c.sha(folder/'model.tflite')==validation['model_sha256'] | |
| info=[dict(**c.tensor_info(d,i),semantic=next(name for name,pos in order.items() if pos==i)) for i,d in enumerate(outs)] | |
| c.contracts(role,config,release,folder,ins,info,order,validation) | |
| contract=json.loads((folder/'runtime_contract.json').read_text()) | |
| contract['numerical_qualification']=dict(status=validation['status'],operating_threshold=threshold, | |
| all_raw_queries_equivalent=False,lower_score_cutoffs_qualified=False,query_row_identity_stable=False, | |
| scope=validation['qualification_scope']) | |
| c.save(folder/'runtime_contract.json',contract) | |
| with (folder/'README.md').open('a',encoding='utf8') as f: | |
| f.write('\n## RF-DETR conversion scope / Périmètre numérique\n\n' | |
| 'Operative conventional/Cadryl detection sets pass at the frozen '+str(threshold)+' cutoff. ' | |
| 'All-300 raw-query parity fails for a few very-low-score candidates; lower cutoffs and row identity are unqualified. ' | |
| 'These discrepancies are retained in conversion_validation.json and raw-query-diagnostic.json.\n\n' | |
| 'La qualification concerne les détections au seuil figé '+str(threshold)+'. Quelques candidats bruts de très faible score diffèrent ; ' | |
| 'l\'équivalence des 300 candidats et des seuils plus bas reste non validée. Les écarts sont conservés dans les reçus.\n') | |
| shutil.copyfile(c.OUT/(role+'-permutation-diagnostic.json'),folder/'raw-query-diagnostic.json') | |
| c.save(folder/'manifest.json',dict(schema='fireviewer.litert.package/1',model_id=c.IDS[role],precision='FP32',inference_only=True, | |
| validated=True,qualification_scope=validation['qualification_scope'],files={p.name:dict(sha256=c.sha(p),bytes=p.stat().st_size) | |
| for p in folder.iterdir() if p.is_file() and p.name!='manifest.json'})) | |
| c.status(role,'complete_operating_points',sha256=validation['model_sha256'],samples=validation['samples'], | |
| all_raw_queries_equivalent=False,operating_threshold=threshold,max_operating_box_error=max(x['box_max_abs'] for x in checks), | |
| max_operating_score_error=max(x['score_max_abs'] for x in checks),raw_divergent_queries=sum(x['divergent_queries'] for x in raw), | |
| max_divergent_probability=max(x['max_divergent_active_class_probability'] for x in raw)) | |
| if __name__=='__main__': | |
| for role in ('proposal','visible'):run(role) | |