Download code/scripts/evaluate_flow.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 3.7 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/evaluate_flow.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/evaluate_flow.py
-
curl -L -o evaluate_flow.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/evaluate_flow.py
3.7 kB
| """Paired held-out flow-matching loss using identical saved observations/noise/time.""" | |
| import argparse,json,pathlib,sys,time | |
| import torch | |
| from robottt_pi05.config import ExperimentConfig | |
| from robottt_pi05.modeling import create_adapter | |
| from robottt_pi05.openpi_backend import load_model | |
| from robottt_pi05.assets import FrozenPrefixStore,ProcessedSequenceSource | |
| from robottt_pi05.checkpoint import load_initial,load_weights | |
| from robottt_pi05.engine import train_sequence | |
| from robottt_pi05.eval_io import atomic_json,sha | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--run-config',required=True);p.add_argument('--output',required=True);a=p.parse_args() | |
| rp=pathlib.Path(a.run_config).resolve();r=json.loads(rp.read_text()) | |
| def path(k):return (rp.parent/r['paths'][k]).resolve() | |
| source=path('openpi_source');sys.path[:0]=[str(source/'src'),str(source/'packages/openpi-client/src')] | |
| torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=True | |
| config=ExperimentConfig.load(path('experiment_config'));m=json.loads(path('asset_index').read_text());ids=m['dev_ids'] | |
| if not ids:raise ValueError('Explicit held-out dev allowlist required') | |
| model=load_model(source=source,checkpoint=path('model_checkpoint'),profile=r['model_profile']);adapter,slow=create_adapter(model,config,inference=True,batched_segments=True) | |
| adapter.minimal_diagnostics=True;adapter.use_batched_segments=True | |
| prefixes=FrozenPrefixStore(m['prefixes'],root=path('prefix_root'),allowed_episodes=ids) | |
| if r.get('prefix_mode')=='on_demand': | |
| from robottt_pi05.ondemand_prefix import OnDemandPrefixStore | |
| prefixes=OnDemandPrefixStore(model,m,data_root=path('data_root'),contract_root=path('prefix_contracts'),model_sha256=r['hashes']['model_checkpoint'],allowed_episodes=ids) | |
| data=ProcessedSequenceSource(m['episodes'],root=path('data_root'),allowed_episodes=ids,stride=config.observation_stride) | |
| out=pathlib.Path(a.output);out.mkdir(parents=True,exist_ok=True);results=[] | |
| for phase in ('initial','trained'): | |
| weight=path('initial_extension') if phase=='initial' else path('output')/'checkpoints/latest.pt';weight_sha=sha(weight) | |
| if phase=='initial':load_initial(weight,slow,expected_sha256=weight_sha) | |
| else:load_weights(weight,slow,config,expected_sha256=weight_sha) | |
| torch.manual_seed(2000) | |
| for episode in ids: | |
| prepared=prefixes.sequence(episode,list(range(config.sequence_length))) | |
| actions=data.targets(episode,0,config.sequence_length).cuda() | |
| for noise_id in range(2): | |
| noise=torch.randn((1,32,10,32));times=torch.distributions.Beta(1.5,1.).sample((1,32,1,1))*.999+.001 | |
| begin=time.monotonic();value=train_sequence(adapter,config,prepared,actions,noise.cuda(),times.cuda(),backward=None) | |
| record=dict(phase=phase,episode=episode,noise_id=noise_id,flow_matching_loss=value['loss'],supervised_scalars=value['valid_scalar_count'],weight_sha256=weight_sha,seconds=time.monotonic()-begin) | |
| atomic_json(out/f'{phase}_e{episode:06d}_n{noise_id}.json',record);results.append(record);print(json.dumps(record),flush=True) | |
| adapter.history.clear();adapter.call_contracts.clear();del value | |
| del prepared,actions | |
| means={phase:sum(x['flow_matching_loss'] for x in results if x['phase']==phase)/sum(x['phase']==phase for x in results) for phase in ('initial','trained')} | |
| atomic_json(out/'summary.json',dict(kind='held_out_teacher_forced_flow_matching_not_rollout_success',run_config_sha256=sha(rp),episodes=ids,noise_seed=2000,mean_loss=means,records=results)) | |
| if __name__=='__main__':main() | |