Spaces:
Running
Running
| """Fresh native observations for public synthetic false abstentions; no tool execution.""" | |
| import json | |
| import argparse | |
| from pathlib import Path | |
| from fastapi.testclient import TestClient | |
| import gateway | |
| from scripts.evaluate_intent import load_cases | |
| def main(): | |
| parser=argparse.ArgumentParser() | |
| parser.add_argument('--include-evaluation-failures',action='store_true') | |
| args=parser.parse_args() | |
| cases=[] | |
| for suite in ['frozen','heldout','heldout-reserve']: | |
| result=json.loads(Path('benchmarks/intent/results/after-'+suite+'.json').read_text()) | |
| failed={r['id'] for r in result['rows'] if r['expected_tool'] and not r['correct']} | |
| cases.extend({**c,'suite':suite} for c in load_cases(suite) if c['id'] in failed) | |
| dev=json.loads(Path('benchmarks/coverage/development.json').read_text()) | |
| cases.extend({**c,'suite':'development','request':{'model':'needle3','tools':dev['tools'],'decision_context':c['decision_context'],'messages':[{'role':'user','content':c['query']}]}} for c in dev['cases'] if c['expected_tool']) | |
| if args.include_evaluation_failures: | |
| from scripts.evaluate_coverage import load_cases as evaluated_cases | |
| for suite in ['frozen','heldout','heldout-reserve','coverage-development','coverage-heldout']: | |
| failed=set() | |
| for label in ['before','after']: | |
| path=Path('benchmarks/coverage/results/'+label+'-'+suite+'.json') | |
| if path.exists(): | |
| failed.update(r['id'] for r in json.loads(path.read_text())['rows'] if r['expected_tool'] and not r['correct']) | |
| cases.extend({**c,'suite':suite} for c in evaluated_cases(suite) if c['id'] in failed) | |
| cases=list({(c['suite'],c['id']):c for c in cases}.values()) | |
| native=[];original=gateway.scheduler.schedule | |
| async def observe(tools,system,text,priority): | |
| r=await original(tools,system,text,priority) | |
| native.append({'candidates':[t['name'] for t in tools],'input':text,'native_result':r}) | |
| return r | |
| gateway.scheduler.schedule=observe;gateway.API_KEY='';gateway.AUTH_MODE='optional' | |
| rows=[] | |
| try: | |
| with TestClient(gateway.app) as client: | |
| for c in cases: | |
| n=len(native);r=client.post('/v1/chat/completions',json=c['request']);d=r.json() | |
| rows.append({'id':c['id'],'suite':c['suite'],'expected_tool':c['expected_tool'],'expected_arguments':c['expected_arguments'],'http_status':r.status_code,'response':d,'inference':native[n:]}) | |
| finally:gateway.scheduler.schedule=original | |
| print(json.dumps({'methodology':'Fresh actual native CPU observations on public synthetic inputs; no returned tool executes. Host resources, not paired CPU benchmark.','rows':rows},indent=2)) | |
| if __name__=='__main__':main() | |