File size: 2,453 Bytes
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3f8ab35
 
 
0f5d220
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
"""Summarize completed paired experiments without conflating loss and success."""
import json,pathlib

def main():
    root=pathlib.Path(__file__).resolve().parents[1]/'.release_local';results={}
    for kind in ('lite','reference'):
        train=root/'runs'/('comparison-'+kind)
        audit=json.loads((train/'model_audit.json').read_text())
        flow=json.loads((root/'evaluations'/('flow-'+kind)/'summary.json').read_text())
        rollout=json.loads((root/'evaluations'/('libero-'+kind)/'summary.json').read_text())
        results[kind]=dict(parameters=audit['extension_trainable_parameters'],tensors=audit['extension_trainable_tensors'],held_out_flow_loss=flow['mean_loss'],libero_attempts=rollout['attempts'],libero_successes=rollout['successes'],libero_success_rate=rollout['success_rate'],libero_complete=rollout['complete'])
    summary=dict(kind='new_common_pipeline_paired_comparison',parameter_reduction_fraction=1-results['lite']['parameters']/results['reference']['parameters'],shared_artifacts=json.loads((root/'common-artifacts.json').read_text()),conditions=results,interpretation='Comparison of complete TTT designs, not parameter-count-only causal isolation; not historical OpenTau trainer results.')
    (root/'comparison-results.json').write_text(json.dumps(summary,indent=2))
    try:
        import numpy as np
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        fig,ax=plt.subplots(figsize=(8,4.5))
        for kind,label in [('lite','4-layer lightweight TTT'),('reference','18-layer OpenTau core')]:
            log=root/'runs'/('comparison-'+kind)/'ledger/events.jsonl'
            rows=[json.loads(line) for line in log.read_text().splitlines()]
            observed=[r for r in rows if r['event']=='training_step']
            y=np.array([r['loss'] for r in observed])
            x=np.array([r['step'] for r in observed]);line,=ax.plot(x,y,alpha=.15)
            window=100
            if len(y)>=window:ax.plot(x[window-1:],np.convolve(y,np.ones(window)/window,mode='valid'),color=line.get_color(),label=label)
        ax.set(xlabel='Optimizer update',ylabel='Normalized real-7D flow MSE',title='Same official base and saved training inputs')
        ax.legend();ax.grid(alpha=.2);fig.tight_layout();fig.savefig(root/'comparison-loss.png',dpi=200);plt.close(fig)
    except ImportError:pass
    print(json.dumps(summary,indent=2))

if __name__=='__main__':main()