"""Summarize completed paired experiments without conflating loss and success.""" import json,pathlib def main(): root=pathlib.Path(__file__).resolve().parents[1]/'.release_local';results={} for kind in ('lite','reference'): train=root/'runs'/('comparison-'+kind) audit=json.loads((train/'model_audit.json').read_text()) flow=json.loads((root/'evaluations'/('flow-'+kind)/'summary.json').read_text()) rollout=json.loads((root/'evaluations'/('libero-'+kind)/'summary.json').read_text()) results[kind]=dict(parameters=audit['extension_trainable_parameters'],tensors=audit['extension_trainable_tensors'],held_out_flow_loss=flow['mean_loss'],libero_attempts=rollout['attempts'],libero_successes=rollout['successes'],libero_success_rate=rollout['success_rate'],libero_complete=rollout['complete']) summary=dict(kind='new_common_pipeline_paired_comparison',parameter_reduction_fraction=1-results['lite']['parameters']/results['reference']['parameters'],shared_artifacts=json.loads((root/'common-artifacts.json').read_text()),conditions=results,interpretation='Comparison of complete TTT designs, not parameter-count-only causal isolation; not historical OpenTau trainer results.') (root/'comparison-results.json').write_text(json.dumps(summary,indent=2)) try: import numpy as np import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt fig,ax=plt.subplots(figsize=(8,4.5)) for kind,label in [('lite','4-layer lightweight TTT'),('reference','18-layer OpenTau core')]: log=root/'runs'/('comparison-'+kind)/'ledger/events.jsonl' rows=[json.loads(line) for line in log.read_text().splitlines()] observed=[r for r in rows if r['event']=='training_step'] y=np.array([r['loss'] for r in observed]) x=np.array([r['step'] for r in observed]);line,=ax.plot(x,y,alpha=.15) window=100 if len(y)>=window:ax.plot(x[window-1:],np.convolve(y,np.ones(window)/window,mode='valid'),color=line.get_color(),label=label) ax.set(xlabel='Optimizer update',ylabel='Normalized real-7D flow MSE',title='Same official base and saved training inputs') ax.legend();ax.grid(alpha=.2);fig.tight_layout();fig.savefig(root/'comparison-loss.png',dpi=200);plt.close(fig) except ImportError:pass print(json.dumps(summary,indent=2)) if __name__=='__main__':main()