scripts/benchmark_coverage_guard.py

"""CPU-only deterministic admission overhead, no native model or tool execution."""
import argparse
import hashlib
import importlib.util
import json
from pathlib import Path
import statistics
import sys
import time
from context_adapter import DecisionContext
import intent_guard


def main():
    p=argparse.ArgumentParser();p.add_argument('--baseline',required=True);p.add_argument('--repeats',type=int,default=200);a=p.parse_args()
    spec=importlib.util.spec_from_file_location('baseline_intent_guard',a.baseline)
    old=importlib.util.module_from_spec(spec);sys.modules[spec.name]=old;spec.loader.exec_module(old)
    d=json.loads(Path('benchmarks/coverage/development.json').read_text());cases=[(c['query'],DecisionContext(**c['decision_context'])) for c in d['cases']]
    rows={}
    for label,module in [('before',old),('after',intent_guard)]:
        for query,policy in cases:module.assess_intent(query,d['tools'],policy)
        elapsed=[]
        for _ in range(a.repeats):
            for query,policy in cases:
                start=time.perf_counter_ns();module.assess_intent(query,d['tools'],policy);elapsed.append((time.perf_counter_ns()-start)/1e6)
        elapsed.sort();rows[label]={'calls':len(elapsed),'p50_ms':statistics.median(elapsed),'p95_ms':elapsed[__import__('math').ceil(len(elapsed)*.95)-1],'mean_ms':statistics.mean(elapsed)}
    print(json.dumps({'methodology':'Deterministic development-set admission only; no native inference, model loading or selected tool execution. Sequential timings, not confidence intervals.','baseline_sha256':hashlib.sha256(Path(a.baseline).read_bytes()).hexdigest(),'candidate_sha256':hashlib.sha256(Path(intent_guard.__file__).read_bytes()).hexdigest(),'development_sha256':hashlib.sha256(Path('benchmarks/coverage/development.json').read_bytes()).hexdigest(),'timings':rows},indent=2))

if __name__=='__main__':main()