robottt-pi05-base-comparison / code /scripts /summarize_comparison.py
jisue's picture
Experiment backup 2026-11-10T11:05:43.719594+00:00
3f8ab35 verified
Raw History Blame Contribute Delete
2.45 kB
"""Summarize completed paired experiments without conflating loss and success."""
import json,pathlib
def main():
root=pathlib.Path(__file__).resolve().parents[1]/'.release_local';results={}
for kind in ('lite','reference'):
train=root/'runs'/('comparison-'+kind)
audit=json.loads((train/'model_audit.json').read_text())
flow=json.loads((root/'evaluations'/('flow-'+kind)/'summary.json').read_text())
rollout=json.loads((root/'evaluations'/('libero-'+kind)/'summary.json').read_text())
results[kind]=dict(parameters=audit['extension_trainable_parameters'],tensors=audit['extension_trainable_tensors'],held_out_flow_loss=flow['mean_loss'],libero_attempts=rollout['attempts'],libero_successes=rollout['successes'],libero_success_rate=rollout['success_rate'],libero_complete=rollout['complete'])
summary=dict(kind='new_common_pipeline_paired_comparison',parameter_reduction_fraction=1-results['lite']['parameters']/results['reference']['parameters'],shared_artifacts=json.loads((root/'common-artifacts.json').read_text()),conditions=results,interpretation='Comparison of complete TTT designs, not parameter-count-only causal isolation; not historical OpenTau trainer results.')
(root/'comparison-results.json').write_text(json.dumps(summary,indent=2))
try:
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig,ax=plt.subplots(figsize=(8,4.5))
for kind,label in [('lite','4-layer lightweight TTT'),('reference','18-layer OpenTau core')]:
log=root/'runs'/('comparison-'+kind)/'ledger/events.jsonl'
rows=[json.loads(line) for line in log.read_text().splitlines()]
observed=[r for r in rows if r['event']=='training_step']
y=np.array([r['loss'] for r in observed])
x=np.array([r['step'] for r in observed]);line,=ax.plot(x,y,alpha=.15)
window=100
if len(y)>=window:ax.plot(x[window-1:],np.convolve(y,np.ones(window)/window,mode='valid'),color=line.get_color(),label=label)
ax.set(xlabel='Optimizer update',ylabel='Normalized real-7D flow MSE',title='Same official base and saved training inputs')
ax.legend();ax.grid(alpha=.2);fig.tight_layout();fig.savefig(root/'comparison-loss.png',dpi=200);plt.close(fig)
except ImportError:pass
print(json.dumps(summary,indent=2))
if __name__=='__main__':main()