Download code/scripts/summarize_comparison.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 2.45 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/summarize_comparison.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/summarize_comparison.py
-
curl -L -o summarize_comparison.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/summarize_comparison.py
2.45 kB
| """Summarize completed paired experiments without conflating loss and success.""" | |
| import json,pathlib | |
| def main(): | |
| root=pathlib.Path(__file__).resolve().parents[1]/'.release_local';results={} | |
| for kind in ('lite','reference'): | |
| train=root/'runs'/('comparison-'+kind) | |
| audit=json.loads((train/'model_audit.json').read_text()) | |
| flow=json.loads((root/'evaluations'/('flow-'+kind)/'summary.json').read_text()) | |
| rollout=json.loads((root/'evaluations'/('libero-'+kind)/'summary.json').read_text()) | |
| results[kind]=dict(parameters=audit['extension_trainable_parameters'],tensors=audit['extension_trainable_tensors'],held_out_flow_loss=flow['mean_loss'],libero_attempts=rollout['attempts'],libero_successes=rollout['successes'],libero_success_rate=rollout['success_rate'],libero_complete=rollout['complete']) | |
| summary=dict(kind='new_common_pipeline_paired_comparison',parameter_reduction_fraction=1-results['lite']['parameters']/results['reference']['parameters'],shared_artifacts=json.loads((root/'common-artifacts.json').read_text()),conditions=results,interpretation='Comparison of complete TTT designs, not parameter-count-only causal isolation; not historical OpenTau trainer results.') | |
| (root/'comparison-results.json').write_text(json.dumps(summary,indent=2)) | |
| try: | |
| import numpy as np | |
| import matplotlib | |
| matplotlib.use('Agg') | |
| import matplotlib.pyplot as plt | |
| fig,ax=plt.subplots(figsize=(8,4.5)) | |
| for kind,label in [('lite','4-layer lightweight TTT'),('reference','18-layer OpenTau core')]: | |
| log=root/'runs'/('comparison-'+kind)/'ledger/events.jsonl' | |
| rows=[json.loads(line) for line in log.read_text().splitlines()] | |
| observed=[r for r in rows if r['event']=='training_step'] | |
| y=np.array([r['loss'] for r in observed]) | |
| x=np.array([r['step'] for r in observed]);line,=ax.plot(x,y,alpha=.15) | |
| window=100 | |
| if len(y)>=window:ax.plot(x[window-1:],np.convolve(y,np.ones(window)/window,mode='valid'),color=line.get_color(),label=label) | |
| ax.set(xlabel='Optimizer update',ylabel='Normalized real-7D flow MSE',title='Same official base and saved training inputs') | |
| ax.legend();ax.grid(alpha=.2);fig.tight_layout();fig.savefig(root/'comparison-loss.png',dpi=200);plt.close(fig) | |
| except ImportError:pass | |
| print(json.dumps(summary,indent=2)) | |
| if __name__=='__main__':main() | |