Spaces:
Running
Running
File size: 2,797 Bytes
25a2c1c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | """Fresh native observations for public synthetic false abstentions; no tool execution."""
import json
import argparse
from pathlib import Path
from fastapi.testclient import TestClient
import gateway
from scripts.evaluate_intent import load_cases
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--include-evaluation-failures',action='store_true')
args=parser.parse_args()
cases=[]
for suite in ['frozen','heldout','heldout-reserve']:
result=json.loads(Path('benchmarks/intent/results/after-'+suite+'.json').read_text())
failed={r['id'] for r in result['rows'] if r['expected_tool'] and not r['correct']}
cases.extend({**c,'suite':suite} for c in load_cases(suite) if c['id'] in failed)
dev=json.loads(Path('benchmarks/coverage/development.json').read_text())
cases.extend({**c,'suite':'development','request':{'model':'needle3','tools':dev['tools'],'decision_context':c['decision_context'],'messages':[{'role':'user','content':c['query']}]}} for c in dev['cases'] if c['expected_tool'])
if args.include_evaluation_failures:
from scripts.evaluate_coverage import load_cases as evaluated_cases
for suite in ['frozen','heldout','heldout-reserve','coverage-development','coverage-heldout']:
failed=set()
for label in ['before','after']:
path=Path('benchmarks/coverage/results/'+label+'-'+suite+'.json')
if path.exists():
failed.update(r['id'] for r in json.loads(path.read_text())['rows'] if r['expected_tool'] and not r['correct'])
cases.extend({**c,'suite':suite} for c in evaluated_cases(suite) if c['id'] in failed)
cases=list({(c['suite'],c['id']):c for c in cases}.values())
native=[];original=gateway.scheduler.schedule
async def observe(tools,system,text,priority):
r=await original(tools,system,text,priority)
native.append({'candidates':[t['name'] for t in tools],'input':text,'native_result':r})
return r
gateway.scheduler.schedule=observe;gateway.API_KEY='';gateway.AUTH_MODE='optional'
rows=[]
try:
with TestClient(gateway.app) as client:
for c in cases:
n=len(native);r=client.post('/v1/chat/completions',json=c['request']);d=r.json()
rows.append({'id':c['id'],'suite':c['suite'],'expected_tool':c['expected_tool'],'expected_arguments':c['expected_arguments'],'http_status':r.status_code,'response':d,'inference':native[n:]})
finally:gateway.scheduler.schedule=original
print(json.dumps({'methodology':'Fresh actual native CPU observations on public synthetic inputs; no returned tool executes. Host resources, not paired CPU benchmark.','rows':rows},indent=2))
if __name__=='__main__':main()
|