File size: 7,137 Bytes
9d6c005 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | """Current release acceptance; no test count is a production maturity score."""
from pathlib import Path
import argparse,csv,json,subprocess,sys,re,hashlib,ast
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT))
import numpy as np
def load(name):return json.loads((ROOT/name).read_text())
def run(output,require_replay=False):
gates=[]
def gate(name,passed,detail):gates.append({'name':name,'passed':bool(passed),'details':detail})
required=['README.md','MANUSCRIPT.md','AUREOLE_R_v3.0.0_Certified_Innovation_Rendering.pdf','STATUS.json','CLAIMS.json','AI_AGENT_INDEX.json',
'THEOREM_INDEX.md','docs/INNOVATION_THEORY.md','docs/CERTIFICATE_API.md','models/visibility_prior.npz',
'results_v3/innovation_raw.csv','results_v3/queries_raw.csv','RUN_INNOVATION.bat','PUBLISH_HF.bat','CITATION.cff','LICENSE']
missing=[p for p in required if not (ROOT/p).is_file()]
gate('required_files',not missing,{'missing':missing})
total=0
for start,count,name in [('tests',44,'current'),('legacy/code',14,'foundational')]:
p=subprocess.run([sys.executable,'-m','unittest','discover','-s',start,'-p','test_*.py','-v'],cwd=ROOT,capture_output=True,text=True)
match=re.search(r'Ran (\d+) tests?',p.stderr);actual=int(match.group(1)) if match else 0;total+=actual
if output.startswith('results_v3/'):(ROOT/f'results_v3/{name}_tests.log').write_text(p.stdout+p.stderr)
gate(name+'_tests',p.returncode==0 and actual==count,{'passed_count':actual,'expected':count})
r=load('results_v3/innovation_report.json');q=load('results_v3/queries_report.json');status=load('STATUS.json')
old=set(load('results/training.json')['training_scene_ids'])|set(load('results/training.json')['validation_scene_ids'])|set(load('results/training.json')['test_scene_ids'])|set(load('results/followup_report.json')['protocol']['scene_ids'])
a=set(r['protocol']['scene_ids']);b=set(q['protocol']['scene_ids'])
gate('held_out_splits',not(a&b or a&old or b&old),{'new_motion_scenes':len(a),'new_query_scenes':len(b)})
raw=list(csv.DictReader((ROOT/'results_v3/innovation_raw.csv').open()))
errs=[]
for s in r['summary']:
vals=[float(x['expected_mse']) for x in raw if x['phase']==s['phase'] and x['method']==s['method']]
errs.append(abs(np.mean(vals)-s['expected_mse']))
rays=sum(int(x['rays']) for x in raw)
gate('motion_raw_consistency',len(raw)==r['records'] and rays==r['online_segment_queries'] and max(errs)<1e-12,
{'records':len(raw),'queries':rays,'max_summary_difference':float(max(errs))})
supported=[x for x in raw if x['method']!='unsafe_eliminate']
false=sum(int(x['false_certificates']) for x in supported);width=max(float(x['enclosure_violation']) for x in supported)
gate('supported_certificate_audit',false==0 and width<1e-12,{'false_acceptances':false,'max_interval_violation':width})
unsafe=sum(int(x['false_certificates']) for x in raw if x['method']=='unsafe_eliminate')
gate('unsafe_negative_control_retained',unsafe>0,{'false_acceptances_in_deliberately_unsafe_ablation':unsafe})
qr=list(csv.DictReader((ROOT/'results_v3/queries_raw.csv').open()))
fresh=sum(int(x['fresh_queries']) for x in qr);base=sum(int(x['shared_fresh_baseline_queries']) for x in qr)
gate('query_family_accounting',fresh==q['residual_queries'] and base==q['shared_fresh_baseline_queries'] and q['total_queries_including_initialization']==fresh+q['initialization_queries'],
{'initialization':q['initialization_queries'],'residual_queries':fresh,'shared_baseline':base})
gate('independent_readout_audit',q['false_certificates']==0 and q['max_linear_rgb_error']<1e-12,
{'false_certificates':q['false_certificates'],'max_linear_error':q['max_linear_rgb_error']})
hashes=[]
for file,report in [('experiments_innovation.json',r),('experiments_queries.json',q)]:
hashes.append(hashlib.sha256((ROOT/file).read_bytes()).hexdigest()==report['protocol_sha256'])
gate('frozen_protocol_hashes',all(hashes),{'both_match':all(hashes)})
v2=load('results/followup_report.json');retained=load('results/rendering_report.json')
gate('prior_evidence_retained',v2['record_count']==5280 and retained['record_count']==7392,
{'v2_frame_method_records':v2['record_count']+retained['record_count']})
errors=[]
if require_replay:
rr=load('innovation_reproduced/innovation_report.json');qq=load('queries_reproduced/queries_report.json')
for original,again in zip(r['summary'],rr['summary'],strict=True):
for key in ('expected_mse','observed_mse','rays','certified_fraction_before','false_certificates','max_enclosure_violation'):
errors.append(abs(original[key]-again[key]))
for key in ('initialization_queries','residual_queries','false_certificates','max_linear_rgb_error'):
errors.append(abs(q[key]-qq[key]))
gate('full_v3_replay',max(errors)<1e-12,{'metrics':len(errors),'max_difference':float(max(errors)),'timings_excluded':True})
else:
recorded=ROOT/'results_v3/replay_validation.json'
gate('recorded_v3_replay',recorded.exists() and load('results_v3/replay_validation.json')['passed'],{'live_replay':False})
demo=load('results_v3/demo/demo.json')
gate('finite_completion_demo',demo['final_max_error']<1e-12 and demo['physical_queries']==demo['full_domain_terms'] and demo['zero_query_final_frames']>0,demo|{'records':'see demo JSON'})
sources=list((ROOT/'aureole').glob('*.py'))+list((ROOT/'scripts').glob('*.py'))
for p in sources:ast.parse(p.read_text())
gate('source_syntax',True,{'files':len(sources)})
gate('honest_scope',status['verified']['gpu_benchmarks']==0 and status['verified']['full_sr_rr_fg_experiments']==0 and status['overall_maturity_estimate_percent']<100,
{'maturity_is_subjective':True,'no_gpu_or_full_pipeline_claim':True,'executable_tests':total})
report={'version':'3.0.0','passed':all(x['passed'] for x in gates),'gates_passed':sum(x['passed'] for x in gates),'gates_total':len(gates),'tests':total,'gates':gates,
'scope':'Acceptance of the finite CPU reference; not full scientific maturity or production certification.'}
path=ROOT/output;path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(report,indent=2)+'\n')
if require_replay and output.startswith('results_v3/'):
(ROOT/'results_v3/replay_validation.json').write_text(json.dumps({'passed':max(errors)<1e-12,'metrics':len(errors),'max_nontiming_difference':float(max(errors))},indent=2)+'\n')
print(json.dumps({'passed':report['passed'],'gates_passed':report['gates_passed'],'gates_total':report['gates_total'],'tests':total,'failures':[x for x in gates if not x['passed']]},indent=2))
return 0 if report['passed'] else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--output',default='validation_reproduced.json');p.add_argument('--require-replay',action='store_true');args=p.parse_args();raise SystemExit(run(args.output,args.require_replay))
|