Download code/scripts/prepare_libero.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 8.24 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/prepare_libero.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/prepare_libero.py
-
curl -L -o prepare_libero.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/prepare_libero.py
8.24 kB
| """Prepare a new, explicitly versioned LIBERO comparison dataset. | |
| Matches OpenTau's selected-episode stats, BF16 dataset boundary, MIN_MAX state, | |
| MEAN_STD actions, two cameras and state-bearing language prompt. Full valid | |
| T32 windows are used in the exported shared plan (no implicit boundary repeats). | |
| """ | |
| import argparse,io,json,pathlib,sys,hashlib,zipfile | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from PIL import Image | |
| import pyarrow.parquet as pq | |
| from transformers import GemmaTokenizer | |
| def aggregate(records,key): | |
| values=[r['stats'][key] for r in records] | |
| means=np.array([x['mean'] for x in values],dtype=np.float64) | |
| stds=np.array([x['std'] for x in values],dtype=np.float64) | |
| counts=np.array([x['count'] for x in values],dtype=np.float64).reshape(len(values),-1)[:,0,None] | |
| mean=(counts*means).sum(0)/counts.sum(0) | |
| variance=(counts*(stds**2+(means-mean)**2)).sum(0)/counts.sum(0) | |
| out=dict(mean=mean,std=np.sqrt(variance),min=np.min([x['min'] for x in values],0),max=np.max([x['max'] for x in values],0)) | |
| return {k:np.pad(np.asarray(v),(0,32-len(v))).astype(np.float32) for k,v in out.items()} | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--source',required=True);p.add_argument('--tokenizer',required=True);p.add_argument('--output',required=True);p.add_argument('--dev-count',type=int,default=0) | |
| p.add_argument('--shard-count',type=int,default=1);p.add_argument('--shard-index',type=int,default=0) | |
| args=p.parse_args();root=pathlib.Path(args.source);out=pathlib.Path(args.output);out.mkdir(parents=True,exist_ok=True) | |
| previous=json.loads((out/'asset-index.json').read_text()) if (out/'asset-index.json').exists() else {} | |
| source_manifest=json.loads((root/'download-manifest.json').read_text()) | |
| if not 0<=args.shard_index<args.shard_count:raise ValueError('Invalid shard') | |
| torch.set_num_threads(2) | |
| episodes=[json.loads(x) for x in (root/'meta/episodes.jsonl').read_text().splitlines()];episodes={x['episode_index']:x for x in episodes} | |
| records=[json.loads(x) for x in (root/'meta/episodes_stats.jsonl').read_text().splitlines()];records=[x for x in records if x['episode_index']<60] | |
| if len(records)!=60:raise ValueError('Missing selected-episode stats') | |
| stats={key:aggregate(records,key) for key in ('state','actions')} | |
| if args.shard_index==0:(out/'normalization.json').write_text(json.dumps({k:{s:v.tolist() for s,v in value.items()} for k,value in stats.items()},indent=2)) | |
| tok=GemmaTokenizer(vocab_file=args.tokenizer,add_bos_token=True,add_eos_token=False,padding_side='right') | |
| manifest={'train_ids':list(range(60)),'dev_ids':list(range(60,60+args.dev_count)),'episodes':{},'prefixes':{},'protocol':{'kind':'new_shared_comparison','source':'TensorAuto/libero','source_revision':source_manifest['revision'],'tokenizer_sha256':hashlib.sha256(pathlib.Path(args.tokenizer).read_bytes()).hexdigest(),'episodes':list(range(60)),'fps':20,'sequence_length':32,'stride':1,'chunk':10,'normalization_epsilon':1e-6,'normalization_version':1,'dataset_float_boundary':'bfloat16','prompt_max_length':256,'camera_order':['image','wrist_image'],'boundary_policy':'full_valid_windows_for_both_models','not_exact_legacy_opentau_sampler':True}} | |
| state_stats={k:torch.from_numpy(v) for k,v in stats['state'].items()};action_stats={k:torch.from_numpy(v) for k,v in stats['actions'].items()} | |
| for episode in range(60+args.dev_count): | |
| if episode%args.shard_count!=args.shard_index:continue | |
| if str(episode) in previous.get('episodes',{}): | |
| manifest['episodes'][str(episode)]=previous['episodes'][str(episode)];continue | |
| folder=out/f'episode_{episode:06d}';folder.mkdir(exist_ok=True) | |
| table=pq.read_table(root/f'data/chunk-000/episode_{episode:06d}.parquet');data=table.to_pydict();n=len(data['state']) | |
| raw_state=torch.zeros(n,32);raw_state[:,:8]=torch.tensor(data['state']);raw_state=raw_state.bfloat16().float() | |
| raw_action=torch.zeros(n,32);raw_action[:,:7]=torch.tensor(data['actions']);raw_action=raw_action.bfloat16().float() | |
| ranges=state_stats['max']-state_stats['min'];safe=torch.where(ranges.abs()<1e-6,1.,ranges) | |
| states=((raw_state-state_stats['min']+.5*(safe-ranges))/(safe+1e-6))*2-1 | |
| std=action_stats['std'];std=torch.where(std.abs()<1e-6,1.,std) | |
| actions=(raw_action-action_stats['mean'])/(std+1e-6) | |
| targets=torch.stack([actions[torch.arange(r,r+10).clamp(max=n-1)] for r in range(n)]) | |
| np.save(folder/'state.npy',states.numpy());np.save(folder/'targets.npy',targets.numpy()) | |
| prompt=episodes[episode]['tasks'][0] | |
| bins=((states.clamp(-1,1)+1)*128).long().clamp(0,255) | |
| prompts=[f'Task: {prompt}, State: '+ ' '.join(map(str,row))+';\n' for row in bins.tolist()] | |
| tokenized=tok(prompts,padding='max_length',max_length=256,truncation=True,return_tensors='np') | |
| np.save(folder/'prompt.npy',tokenized['input_ids'].astype(np.int64));np.save(folder/'prompt_mask.npy',tokenized['attention_mask'].astype(bool)) | |
| for key in ('image','wrist_image'): | |
| existing=folder/f'{key}.npz' | |
| if existing.exists(): | |
| with np.load(existing,allow_pickle=False) as archive: | |
| saved=archive['values'] | |
| if saved.shape!=(n,3,224,224) or saved.dtype!=np.float32 or not np.isfinite(saved).all(): | |
| raise ValueError('Existing compressed image is invalid') | |
| del saved | |
| continue | |
| mm=np.empty((n,3,224,224),dtype=np.float32) | |
| for lo in range(0,n,32): | |
| ims=[] | |
| for image in data[key][lo:lo+32]: | |
| pixels=np.array(Image.open(io.BytesIO(image['bytes'])).convert('RGB'),copy=True) | |
| ims.append(torch.from_numpy(pixels).permute(2,0,1).float()/255.) | |
| resized=F.interpolate(torch.stack(ims),size=(224,224),mode='bilinear',align_corners=False) | |
| mm[lo:lo+len(ims)]=(resized.bfloat16()*2-1).float().numpy() | |
| # Lossless compression preserves the exact BF16-rounded F32 inputs. | |
| temporary=folder/f'{key}.tmp.npz' | |
| with zipfile.ZipFile(temporary,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=1) as archive: | |
| with archive.open('values.npy','w',force_zip64=True) as stream: | |
| np.lib.format.write_array(stream,mm,allow_pickle=False) | |
| with np.load(temporary,allow_pickle=False) as restored: | |
| if not np.array_equal(restored['values'],mm): | |
| raise ValueError('Compressed image round-trip differs') | |
| temporary.replace(folder/f'{key}.npz') | |
| del mm | |
| # Last source row with a complete action chunk. The shared sampler also | |
| # requires all 32 observation rows to be inside this set. | |
| rows=list(range(n-9)) | |
| manifest['episodes'][str(episode)]={'folder':folder.name,'manifest':{'rows':rows,'fields':{'state':'state.npy','tokenized_prompt':'prompt.npy','tokenized_prompt_mask':'prompt_mask.npy'}},'length':n,'task':prompt} | |
| print(json.dumps({'episode':episode,'frames':n,'complete_target_rows':len(rows)}),flush=True) | |
| def file_hash(path): | |
| h=hashlib.sha256() | |
| with path.open('rb') as f: | |
| while chunk:=f.read(8*1024*1024):h.update(chunk) | |
| return h.hexdigest() | |
| source_hashes={x['path']:x['sha256'] for x in source_manifest['files']} | |
| for episode,record in manifest['episodes'].items(): | |
| record['file_sha256']={f.name:file_hash(f) for f in sorted((out/record['folder']).iterdir()) if f.suffix in ('.npy','.npz')} | |
| relative=f'data/chunk-000/episode_{int(episode):06d}.parquet' | |
| record['source_parquet_sha256']=source_hashes[relative] | |
| if file_hash(root/relative)!=source_hashes[relative]:raise ValueError('Source parquet changed') | |
| # Arrays are indexed by source row; the rows metadata starts at zero. | |
| name='asset-index.json' if args.shard_count==1 else f'asset-index-shard-{args.shard_index}.json' | |
| temporary=out/(name+'.tmp');temporary.write_text(json.dumps(manifest,indent=2));temporary.replace(out/name) | |
| if __name__=='__main__':main() | |