Spaces:
Running
Running
| """Count large fixtures with the official *reference*, not proven native tokenizer. | |
| An offline study only. No outputs here may be called exact native token counts. | |
| """ | |
| import argparse | |
| import json | |
| import time | |
| from pathlib import Path | |
| from context_adapter import DecisionContext, adapt, canonical | |
| from decision_policy import eligible_tools, intent_eligible, retrieve | |
| from scripts.token_accounting_study import reference, fast_bpe | |
| from scripts.final_qualification import cases | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--wheel',required=True);p.add_argument('--assets',required=True) | |
| a=p.parse_args();ns=reference(a.wheel) | |
| meta=ns['parse_tokenizer_blob'](ns['read_tokenizer_blob'](str(Path(a.assets)/'needle3.cact'))) | |
| tok=ns['RefTokenizer'](meta);tok._bpe=lambda text:fast_bpe(tok,text) | |
| rows=[] | |
| # One placement per size/family for tokenizer cost. HTTP evaluation covers all three placements. | |
| for case in cases(): | |
| if case['group']!='long-context' or not case['id'].endswith('-middle'):continue | |
| b=case['request'];text=b['messages'][1]['content'];start=time.perf_counter() | |
| count=len(tok.encode(text));ms=(time.perf_counter()-start)*1000 | |
| policy=DecisionContext(**b['decision_context']);reduced=adapt(b['messages'],policy,b['tools']) | |
| compact=None;compact_bytes=None | |
| if reduced.sufficient: | |
| tools,_=eligible_tools(b['tools'],policy) | |
| tools=intent_eligible(reduced.inference_text(),b['tools'],tools) | |
| tools,_=retrieve(reduced.inference_text(),tools) | |
| bindings={t['name']:policy.arguments[t['name']] for t in tools if t['name'] in policy.arguments} | |
| system=reduced.inference_system() | |
| if bindings:system+='\nCurrent exact argument bindings: '+canonical(bindings) | |
| visible=system+reduced.inference_text()+canonical(tools) | |
| compact=len(tok.encode(visible));compact_bytes=len(visible.encode()) | |
| rows.append({'id':case['id'],'tool_output_utf8_bytes':len(text.encode()),'reference_tool_output_tokens':count,'reference_tokenization_ms':ms,'inference_sufficient':reduced.sufficient,'reference_visible_compact_tokens':compact,'visible_compact_utf8_bytes':compact_bytes,'native_hidden_template_tokens':None,'native_query_tokens':None}) | |
| print(json.dumps({'count_kind':'official_newer_reference_not_native_verified','native_exact_accounting':False,'rows':rows,'limitations':['Counts apply to tool-output text and visible compact concatenation; no native hidden template count implied.','Size labels still denote records. Actual reference token counts are reported independently.','Standalone reference encoder timing is not pinned native tokenizer latency.']},indent=2)) | |
| if __name__=='__main__':main() | |