Spaces:
Running
Running
| """Supplemental development-only five-candidate native output probes.""" | |
| import copy,json,resource,time | |
| from pathlib import Path | |
| import needle | |
| from context_adapter import DecisionContext,budget_fits | |
| from decision_policy import validate_decision | |
| from gateway import with_current_facts | |
| from scripts.coverage_workflow import TOOLS,FIXED | |
| def main(): | |
| rows=[] | |
| for projected in (False,True): | |
| tools=copy.deepcopy([t for t in TOOLS if t['name'] in {'write_file','read_file','git_status','run_tests','list_files'}]) | |
| args={'path':'/project/app.py'} | |
| if projected: | |
| write=next(t for t in tools if t['name']=='write_file') | |
| write['parameters']['properties'].pop('content');write['parameters']['required'].remove('content') | |
| query='write_file /project/app.py' | |
| else: | |
| args['content']=FIXED;query='write_file /project/app.py content '+json.dumps(FIXED) | |
| policy=DecisionContext(workspace='/project',arguments={'write_file':args}) | |
| system=with_current_facts('Public disposable repository.',tools,policy) | |
| agent=needle.Needle(tools=tools,system=system,generation=3) | |
| start=time.perf_counter();native=agent.complete(query) | |
| rows.append({'projected':projected,'tools':tools,'query':query,'schema_utf8_bytes':sum(len(json.dumps(t['parameters']).encode()) for t in tools),'prompt_proxy_units':budget_fits(query,system,tools,10**9)[1],'public_max_new_tokens':512,'latency_ms':(time.perf_counter()-start)*1000,'native':native,'validated':validate_decision(native,tools,policy,system+'\n'+query),'serialized_envelope_bytes':len(json.dumps(native).encode()),'host_self_peak_rss_kib':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}) | |
| # Eight nested object levels exercise output structure independently of size. | |
| value='leaf';schema={'type':'string'} | |
| for level in range(8): | |
| value={f'level{level}':value};schema={'type':'object','properties':{f'level{level}':schema},'required':[f'level{level}'],'additionalProperties':False} | |
| nested={'name':'store_config','description':'Store the explicitly supplied configuration','parameters':{'type':'object','properties':{'config':schema},'required':['config'],'additionalProperties':False}} | |
| tools=[nested];policy=DecisionContext(arguments={'store_config':{'config':value}}) | |
| query='store_config config '+json.dumps(value);system=with_current_facts('Public disposable configuration.',tools,policy) | |
| agent=needle.Needle(tools=tools,system=system,generation=3);start=time.perf_counter();native=agent.complete(query) | |
| rows.append({'case':'eight_nested_objects','tools':tools,'query':query,'schema_utf8_bytes':len(json.dumps(nested['parameters']).encode()),'prompt_proxy_units':budget_fits(query,system,tools,10**9)[1],'public_max_new_tokens':512,'latency_ms':(time.perf_counter()-start)*1000,'native':native,'validated':validate_decision(native,tools,policy,system+'\n'+query),'serialized_envelope_bytes':len(json.dumps(native).encode()),'host_self_peak_rss_kib':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}) | |
| Path('benchmarks/workflow/results/native-candidate-audit.json').write_text(json.dumps({'methodology':'Supplemental host five-candidate development probes; no executable tools run. Same public default 512 output budget. Proxy units and envelope bytes are not exact native tokens. Not a replacement for gateway filtering or workflow reproduction.','rows':rows},indent=2)+'\n') | |
| if __name__=='__main__':main() | |