Spaces:
Running
Running
| """Derive finite qualification gates from unchanged-fixture native artifacts.""" | |
| import json,math,statistics,subprocess | |
| from pathlib import Path | |
| from scripts.evaluate_workflow import metrics | |
| OUT=Path('benchmarks/workflow/results') | |
| SUITES=['frozen','heldout','heldout-reserve','coverage-development','coverage-heldout','coverage-adversarial','coverage-reserve','workflow-development','workflow-heldout','workflow-schema-development','workflow-schema-heldout'] | |
| def read(name):return json.loads((OUT/(name+'.json')).read_text()) | |
| def stats(values): | |
| values=sorted(values) | |
| return {'n':len(values),'p50':statistics.median(values),'p95':values[math.ceil(len(values)*.95)-1],'p99':values[math.ceil(len(values)*.99)-1]} if values else None | |
| def comparison(report): | |
| b=report['arms']['before'];a=report['arms']['after'] | |
| def pct(x):return f'{100*x:.1f}%' | |
| def latency(x):return ' / '.join(f"{x[k]:.1f}" for k in ('p50','p95','p99')) | |
| rows=[] | |
| for label,suite in [('Frozen clear exact','frozen'),('Coverage held-out clear exact','coverage-heldout'),('Payload held-out clear exact','workflow-heldout'),('Annotation held-out clear exact','workflow-schema-heldout')]: | |
| def exact(arm): | |
| d=arm['suites'][suite];n=d['clear_trials'];return f"{round(d['clear_exact_accuracy']*n)}/{n} ({100*d['clear_exact_accuracy']:.2f}%)" | |
| rows.append((label,exact(b),exact(a))) | |
| rows += [('Strict accuracy, all timed chat trials',pct(b['all_trials']['strict_accuracy']),pct(a['all_trials']['strict_accuracy'])),('Incorrect executable selections',b['all_trials']['false_executable_selections'],a['all_trials']['false_executable_selections']),('False abstentions',b['false_abstentions'],a['false_abstentions']),('Candidate recall',pct(b['all_trials']['candidate_recall']),pct(a['all_trials']['candidate_recall'])),('Exact original multiline write',b['multiline_write_success'],a['multiline_write_success']),('Complete CPU Git workflow',b['workflow_success'],a['workflow_success']),('All chat p50 / p95 / p99 ms',latency(b['all_trials']['latency_ms']),latency(a['all_trials']['latency_ms'])),('Original clear p50 / p95 / p99 ms',latency(b['original_clear_latency_ms']),latency(a['original_clear_latency_ms'])),('Warm incremental p50 / p95 / p99 ms',latency(b['warm_incremental']),latency(a['warm_incremental'])),('Peak server RSS MiB',f"{b['peak_server_rss_mib']:.2f}",f"{a['peak_server_rss_mib']:.2f}"),('Capacity-probe peak server RSS MiB',f"{b['capacity_peak_server_rss_mib']:.2f}",f"{a['capacity_peak_server_rss_mib']:.2f}"),('Native truncations in timed chat suites',b['native_truncation_count'],a['native_truncation_count']),('Native truncation in original CPU write',b['write_native_truncated'],a['write_native_truncated']),('Original write native latency ms',f"{b['write_native_ms']:.1f}",f"{a['write_native_ms']:.1f}")] | |
| lines=['## Final controlled CPU comparison','',report['recommendation'],'',f"{a['all_trials']['trials']} timed chat trials per arm; 2 CPUs / 16 GiB; fixed SDK date fact. Before ef25f06, after e90ba52 runtime. Capacity probes (6 trials per arm), actual tool execution and MCP are separate evidence.",'','| Metric | Before | After |','|---|---:|---:|'] | |
| lines += [f'| {k} | {x} | {y} |' for k,x,y in rows] | |
| lines += ['','| Suite | Clear exact before → after | False abstentions before → after | Incorrect executable selections before → after |','|---|---:|---:|---:|'] | |
| for suite in SUITES: | |
| x=b['suites'][suite];y=a['suites'][suite] | |
| value='n/a' if x['clear_exact_accuracy'] is None else pct(x['clear_exact_accuracy'])+' → '+pct(y['clear_exact_accuracy']) | |
| lines.append(f"| {suite} | {value} | {x['false_abstention_count']} → {y['false_abstention_count']} | {x['false_executable_selections']} → {y['false_executable_selections']} |") | |
| lines += ['','Transport timings are measured separately from native inference; p50 / p95 / p99 ms:','', '| Sample | Creation | Reconstruction + original validation | Native schedule |','|---|---:|---:|---:|'] | |
| for label,key in [('Payload routing trials','transport_ms'),('16–112 KiB capacity trials','capacity_transport_ms')]: | |
| t=a[key];lines.append(f"| {label} | {latency(t['payload_create_ms'])} | {latency(t['payload_restore_ms'])} | {latency(t['native_ms'])} |") | |
| lines += ['','Gate results:','']+[f"- {'PASS' if v else 'FAIL'}: {k}" for k,v in report['gates'].items()] | |
| return '\n'.join(lines)+'\n' | |
| def main(): | |
| report={'methodology':'Derived finite gates, not proof of general safety. Same-author synthetic repeated trials; use per-suite denominators. Frozen 97.0% and coverage 86.7% are rounded baseline conventions. No actual HF or OpenCode CLI qualification.','arms':{}} | |
| for label in ('before','after'): | |
| suites={s:read(label+'-'+s) for s in SUITES};rows=[r for d in suites.values() for r in d['rows']];old=[r for s,d in suites.items() if not s.startswith('workflow-') for r in d['rows']] | |
| flow=read(label+'-workflow-cpu');state=read(label+'-state-cpu');stress=read(label+'-workflow-stress') | |
| transport={k:stats([r['diagnostics'].get('diagnostics',{}).get(k) for r in rows if isinstance(r['diagnostics'].get('diagnostics',{}).get(k),(int,float))]) for k in ('payload_create_ms','payload_restore_ms','native_ms')} | |
| wr=flow['rows'][5] | |
| report['arms'][label]={'all_trials':metrics(rows),'original_trials':metrics(old),'suites':{s:d['summary'] for s,d in suites.items()},'peak_server_rss_mib':max(int(d['server_memory']['VmHWM'].split()[0])/1024 for d in suites.values()),'false_abstentions':sum(d['summary']['false_abstention_count'] for d in suites.values()),'error_count':sum(d['summary']['error_count'] for d in suites.values()),'native_truncation_count':sum(d['summary']['native_truncation_count'] for d in suites.values()),'transport_ms':transport,'workflow_success':flow['workflow_success'],'multiline_write_success':flow['verification']['file_bytes_match'],'workflow_verification':flow['verification'],'workflow_http_latency_ms':stats([r['latency_ms'] for r in flow['rows']]),'workflow_wall_ms':flow['total_wall_ms'],'workflow_self_peak_rss_mib':flow['host_self_peak_rss_kib']/1024,'write_native_ms':wr['native'][0]['native_ms'],'write_diagnostics':wr['response']['needle']['diagnostics'],'write_outcome':wr['response']['needle']['outcome'],'write_native_truncated':int('token budget exhausted' in wr['response']['needle'].get('reason','')),'warm_incremental':state['state_smoke']['warm_delta_latency_ms'],'capacity':stress['summary'],'capacity_peak_server_rss_mib':int(stress['server_memory']['VmHWM'].split()[0])/1024,'capacity_transport_ms':{k:stats([r['diagnostics'].get('diagnostics',{}).get(k) for r in stress['rows'] if isinstance(r['diagnostics'].get('diagnostics',{}).get(k),(int,float))]) for k in ('payload_create_ms','payload_restore_ms','native_ms')},'original_clear_latency_ms':stats([r['latency_ms'] for r in old if r['expected_tool']]),'new_clear_latency_ms':stats([r['latency_ms'] for s,d in suites.items() if s.startswith('workflow-') for r in d['rows'] if r['expected_tool']])} | |
| before=report['arms']['before'];after=report['arms']['after'] | |
| untouched=subprocess.check_output(['git','diff','ef25f06','--','benchmarks/qualification/scenarios.json','benchmarks/intent','benchmarks/coverage/development.json','benchmarks/coverage/heldout.json','benchmarks/coverage/adversarial.json','benchmarks/coverage/reserve.json'])==b'' | |
| mcp=read('mcp-integration') | |
| gates={'unchanged_original_datasets':untouched,'zero_incorrect_executable_selections':after['all_trials']['false_executable_selections']==0,'zero_safety_executable_selections':after['all_trials']['safety_executable_selections']==0,'frozen_clear_baseline_preserved_64_of_66':after['suites']['frozen']['clear_exact_accuracy']>=64/66,'literal_accuracy_minimums_as_written':after['suites']['frozen']['clear_exact_accuracy']>=.97 and after['suites']['coverage-heldout']['clear_exact_accuracy']>=.867,'coverage_clear_baseline_preserved_26_of_30':after['suites']['coverage-heldout']['clear_exact_accuracy']>=26/30,'no_clear_accuracy_regression_in_original_suites':all(after['suites'][s]['clear_exact_accuracy'] is None or after['suites'][s]['clear_exact_accuracy']>=before['suites'][s]['clear_exact_accuracy'] for s in SUITES[:7]),'new_heldout_exact_clear':after['suites']['workflow-heldout']['clear_exact_accuracy']==1,'new_annotation_heldout_exact_clear':after['suites']['workflow-schema-heldout']['clear_exact_accuracy']==1,'original_multiline_write_exact':after['multiline_write_success'],'actual_complete_workflow':after['workflow_success'],'real_mcp_client_all_checks':mcp['success'] and all(mcp['checks'].values()),'capacity_exact':after['capacity']['clear_exact_accuracy']==1 and after['capacity']['false_executable_selections']==0,'no_material_server_rss_regression_10_percent':after['peak_server_rss_mib']<=before['peak_server_rss_mib']*1.1,'host_tests': '437 passed' in (OUT/'host-tests.txt').read_text(),'offline_cpu_tests':'433 passed' in (OUT/'offline-cpu-tests.txt').read_text()} | |
| report['literal_accuracy_observations']={'frozen':after['suites']['frozen']['clear_exact_accuracy'],'coverage_heldout':after['suites']['coverage-heldout']['clear_exact_accuracy'],'required_frozen':.97,'required_coverage':.867} | |
| report['implementation_gates_pass']=all(v for k,v in gates.items() if k!='literal_accuracy_minimums_as_written') | |
| report['gates']=gates;report['all_finite_gates_pass']=all(gates.values());report['recommendation']='CONDITIONAL GO for reviewed local MCP integration; NO-GO for production deployment pending actual HF qualification, independent review, and real OpenCode CLI validation.' if all(gates.values()) else 'NO-GO: one or more finite acceptance gates failed; retain fail-closed behavior.' | |
| if report['implementation_gates_pass'] and not gates['literal_accuracy_minimums_as_written']: | |
| report['recommendation']='NO-GO against the literal unrounded accuracy minima and for production rollout. Implementation gates pass at the preserved supplied baseline precision; conditional local acceptance requires explicitly accepting that precision. No model/threshold/fixture change was made to force a passing gate.' | |
| (OUT/'summary.json').write_text(json.dumps(report,indent=2)+'\n');print(json.dumps({'gates':gates,'recommendation':report['recommendation']},indent=2)) | |
| rendered=comparison(report);(OUT/'comparison.md').write_text(rendered) | |
| doc=Path('docs/needle-gateway-v2-workflow-qualification.md') | |
| if doc.exists(): | |
| text=doc.read_text();start=text.index('<!-- RESULTS -->');end=text.index('<!-- END RESULTS -->') | |
| text=text[:start]+'<!-- RESULTS -->\n'+rendered+'\n'+text[end:] | |
| if '<!-- VERDICT -->' in text: | |
| vs=text.index('<!-- VERDICT -->');ve=text.index('<!-- END VERDICT -->') | |
| text=text[:vs]+'<!-- VERDICT -->\n**'+report['recommendation']+'**\n'+text[ve:] | |
| doc.write_text(text) | |
| if not report['all_finite_gates_pass']:raise SystemExit(1) | |
| if __name__=='__main__':main() | |