File size: 8,240 Bytes
0f5d220 af6a9d0 0f5d220 af6a9d0 0f5d220 af6a9d0 0f5d220 af6a9d0 0f5d220 af6a9d0 0f5d220 af6a9d0 c347d96 0f5d220 c347d96 af6a9d0 c347d96 0f5d220 c347d96 0f5d220 af6a9d0 0f5d220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """Prepare a new, explicitly versioned LIBERO comparison dataset.
Matches OpenTau's selected-episode stats, BF16 dataset boundary, MIN_MAX state,
MEAN_STD actions, two cameras and state-bearing language prompt. Full valid
T32 windows are used in the exported shared plan (no implicit boundary repeats).
"""
import argparse,io,json,pathlib,sys,hashlib,zipfile
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
import pyarrow.parquet as pq
from transformers import GemmaTokenizer
def aggregate(records,key):
values=[r['stats'][key] for r in records]
means=np.array([x['mean'] for x in values],dtype=np.float64)
stds=np.array([x['std'] for x in values],dtype=np.float64)
counts=np.array([x['count'] for x in values],dtype=np.float64).reshape(len(values),-1)[:,0,None]
mean=(counts*means).sum(0)/counts.sum(0)
variance=(counts*(stds**2+(means-mean)**2)).sum(0)/counts.sum(0)
out=dict(mean=mean,std=np.sqrt(variance),min=np.min([x['min'] for x in values],0),max=np.max([x['max'] for x in values],0))
return {k:np.pad(np.asarray(v),(0,32-len(v))).astype(np.float32) for k,v in out.items()}
def main():
p=argparse.ArgumentParser();p.add_argument('--source',required=True);p.add_argument('--tokenizer',required=True);p.add_argument('--output',required=True);p.add_argument('--dev-count',type=int,default=0)
p.add_argument('--shard-count',type=int,default=1);p.add_argument('--shard-index',type=int,default=0)
args=p.parse_args();root=pathlib.Path(args.source);out=pathlib.Path(args.output);out.mkdir(parents=True,exist_ok=True)
previous=json.loads((out/'asset-index.json').read_text()) if (out/'asset-index.json').exists() else {}
source_manifest=json.loads((root/'download-manifest.json').read_text())
if not 0<=args.shard_index<args.shard_count:raise ValueError('Invalid shard')
torch.set_num_threads(2)
episodes=[json.loads(x) for x in (root/'meta/episodes.jsonl').read_text().splitlines()];episodes={x['episode_index']:x for x in episodes}
records=[json.loads(x) for x in (root/'meta/episodes_stats.jsonl').read_text().splitlines()];records=[x for x in records if x['episode_index']<60]
if len(records)!=60:raise ValueError('Missing selected-episode stats')
stats={key:aggregate(records,key) for key in ('state','actions')}
if args.shard_index==0:(out/'normalization.json').write_text(json.dumps({k:{s:v.tolist() for s,v in value.items()} for k,value in stats.items()},indent=2))
tok=GemmaTokenizer(vocab_file=args.tokenizer,add_bos_token=True,add_eos_token=False,padding_side='right')
manifest={'train_ids':list(range(60)),'dev_ids':list(range(60,60+args.dev_count)),'episodes':{},'prefixes':{},'protocol':{'kind':'new_shared_comparison','source':'TensorAuto/libero','source_revision':source_manifest['revision'],'tokenizer_sha256':hashlib.sha256(pathlib.Path(args.tokenizer).read_bytes()).hexdigest(),'episodes':list(range(60)),'fps':20,'sequence_length':32,'stride':1,'chunk':10,'normalization_epsilon':1e-6,'normalization_version':1,'dataset_float_boundary':'bfloat16','prompt_max_length':256,'camera_order':['image','wrist_image'],'boundary_policy':'full_valid_windows_for_both_models','not_exact_legacy_opentau_sampler':True}}
state_stats={k:torch.from_numpy(v) for k,v in stats['state'].items()};action_stats={k:torch.from_numpy(v) for k,v in stats['actions'].items()}
for episode in range(60+args.dev_count):
if episode%args.shard_count!=args.shard_index:continue
if str(episode) in previous.get('episodes',{}):
manifest['episodes'][str(episode)]=previous['episodes'][str(episode)];continue
folder=out/f'episode_{episode:06d}';folder.mkdir(exist_ok=True)
table=pq.read_table(root/f'data/chunk-000/episode_{episode:06d}.parquet');data=table.to_pydict();n=len(data['state'])
raw_state=torch.zeros(n,32);raw_state[:,:8]=torch.tensor(data['state']);raw_state=raw_state.bfloat16().float()
raw_action=torch.zeros(n,32);raw_action[:,:7]=torch.tensor(data['actions']);raw_action=raw_action.bfloat16().float()
ranges=state_stats['max']-state_stats['min'];safe=torch.where(ranges.abs()<1e-6,1.,ranges)
states=((raw_state-state_stats['min']+.5*(safe-ranges))/(safe+1e-6))*2-1
std=action_stats['std'];std=torch.where(std.abs()<1e-6,1.,std)
actions=(raw_action-action_stats['mean'])/(std+1e-6)
targets=torch.stack([actions[torch.arange(r,r+10).clamp(max=n-1)] for r in range(n)])
np.save(folder/'state.npy',states.numpy());np.save(folder/'targets.npy',targets.numpy())
prompt=episodes[episode]['tasks'][0]
bins=((states.clamp(-1,1)+1)*128).long().clamp(0,255)
prompts=[f'Task: {prompt}, State: '+ ' '.join(map(str,row))+';\n' for row in bins.tolist()]
tokenized=tok(prompts,padding='max_length',max_length=256,truncation=True,return_tensors='np')
np.save(folder/'prompt.npy',tokenized['input_ids'].astype(np.int64));np.save(folder/'prompt_mask.npy',tokenized['attention_mask'].astype(bool))
for key in ('image','wrist_image'):
existing=folder/f'{key}.npz'
if existing.exists():
with np.load(existing,allow_pickle=False) as archive:
saved=archive['values']
if saved.shape!=(n,3,224,224) or saved.dtype!=np.float32 or not np.isfinite(saved).all():
raise ValueError('Existing compressed image is invalid')
del saved
continue
mm=np.empty((n,3,224,224),dtype=np.float32)
for lo in range(0,n,32):
ims=[]
for image in data[key][lo:lo+32]:
pixels=np.array(Image.open(io.BytesIO(image['bytes'])).convert('RGB'),copy=True)
ims.append(torch.from_numpy(pixels).permute(2,0,1).float()/255.)
resized=F.interpolate(torch.stack(ims),size=(224,224),mode='bilinear',align_corners=False)
mm[lo:lo+len(ims)]=(resized.bfloat16()*2-1).float().numpy()
# Lossless compression preserves the exact BF16-rounded F32 inputs.
temporary=folder/f'{key}.tmp.npz'
with zipfile.ZipFile(temporary,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=1) as archive:
with archive.open('values.npy','w',force_zip64=True) as stream:
np.lib.format.write_array(stream,mm,allow_pickle=False)
with np.load(temporary,allow_pickle=False) as restored:
if not np.array_equal(restored['values'],mm):
raise ValueError('Compressed image round-trip differs')
temporary.replace(folder/f'{key}.npz')
del mm
# Last source row with a complete action chunk. The shared sampler also
# requires all 32 observation rows to be inside this set.
rows=list(range(n-9))
manifest['episodes'][str(episode)]={'folder':folder.name,'manifest':{'rows':rows,'fields':{'state':'state.npy','tokenized_prompt':'prompt.npy','tokenized_prompt_mask':'prompt_mask.npy'}},'length':n,'task':prompt}
print(json.dumps({'episode':episode,'frames':n,'complete_target_rows':len(rows)}),flush=True)
def file_hash(path):
h=hashlib.sha256()
with path.open('rb') as f:
while chunk:=f.read(8*1024*1024):h.update(chunk)
return h.hexdigest()
source_hashes={x['path']:x['sha256'] for x in source_manifest['files']}
for episode,record in manifest['episodes'].items():
record['file_sha256']={f.name:file_hash(f) for f in sorted((out/record['folder']).iterdir()) if f.suffix in ('.npy','.npz')}
relative=f'data/chunk-000/episode_{int(episode):06d}.parquet'
record['source_parquet_sha256']=source_hashes[relative]
if file_hash(root/relative)!=source_hashes[relative]:raise ValueError('Source parquet changed')
# Arrays are indexed by source row; the rows metadata starts at zero.
name='asset-index.json' if args.shard_count==1 else f'asset-index-shard-{args.shard_index}.json'
temporary=out/(name+'.tmp');temporary.write_text(json.dumps(manifest,indent=2));temporary.replace(out/name)
if __name__=='__main__':main()
|