File size: 2,337 Bytes
795f737 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """Sequential isolated-process checkpoint comparison. Never promotes a model."""
import json
from pathlib import Path
import subprocess
import sys
def run(module,args,log):
result = subprocess.run([sys.executable,'-m',module,*args],stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,text=True,encoding='utf-8',errors='replace')
Path(log).write_text(result.stdout,encoding='utf-8')
if result.returncode:
raise RuntimeError(f'{module} failed; inspect {log}')
def main():
Path('reports').mkdir(exist_ok=True)
candidates = [('v000-mean',False),('v001-no-lexical',False),('v002-gru',False),
('v003-transformer',False),('v000-mean',True)]
ranked = []
for name,quantized in candidates:
label = name + ('-int8' if quantized else '-fp32')
output = f'reports/bench-{label}.json'
run('baim.bench_policy',['--checkpoint',f'models/{name}','--output',output]
+ (['--quantized'] if quantized else []),f'reports/bench-{label}.log')
report = json.loads(Path(output).read_text())
evaluation = report['evaluation']
# Diagnostic rank only: recovery and full task data are absent.
score = evaluation['test']['joint_step_accuracy'] * evaluation['novel_wording']['joint_step_accuracy'] / (
max(report['end_to_end_policy_ms']['p95'],.001) * max(report['observed_process_rss_bytes']/1024**3,.01))
ranked.append(dict(label=label,diagnostic_score=score,report=output,
test_joint=evaluation['test']['joint_step_accuracy'],novel_joint=evaluation['novel_wording']['joint_step_accuracy'],
p95_policy_ms=report['end_to_end_policy_ms']['p95'],rss_bytes=report['observed_process_rss_bytes'],
disk_bytes=report['disk_bytes']))
print(json.dumps(ranked[-1]),flush=True)
ranked.sort(key=lambda row:row['diagnostic_score'],reverse=True)
summary = dict(candidates=ranked,production_promoted=False,
formula='test_joint * novel_wording_joint / (p95_policy_ms * observed_python_rss_GiB)',
reason='Diagnostic synthetic ranking only; target-hardware, full task, recovery and security evidence missing.')
Path('reports/model-comparison.json').write_text(json.dumps(summary,indent=2),encoding='utf-8')
if __name__=='__main__':
main()
|