File size: 8,240 Bytes
0f5d220
 
 
 
 
 
af6a9d0
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
af6a9d0
 
0f5d220
 
af6a9d0
 
0f5d220
 
 
 
af6a9d0
0f5d220
 
 
 
af6a9d0
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
af6a9d0
 
 
 
 
 
 
 
c347d96
0f5d220
 
 
 
 
 
 
c347d96
 
af6a9d0
 
 
c347d96
 
 
 
 
0f5d220
 
 
 
 
 
 
 
 
 
 
 
c347d96
0f5d220
 
 
 
af6a9d0
 
0f5d220
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
"""Prepare a new, explicitly versioned LIBERO comparison dataset.

Matches OpenTau's selected-episode stats, BF16 dataset boundary, MIN_MAX state,
MEAN_STD actions, two cameras and state-bearing language prompt. Full valid
T32 windows are used in the exported shared plan (no implicit boundary repeats).
"""
import argparse,io,json,pathlib,sys,hashlib,zipfile
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
import pyarrow.parquet as pq
from transformers import GemmaTokenizer

def aggregate(records,key):
    values=[r['stats'][key] for r in records]
    means=np.array([x['mean'] for x in values],dtype=np.float64)
    stds=np.array([x['std'] for x in values],dtype=np.float64)
    counts=np.array([x['count'] for x in values],dtype=np.float64).reshape(len(values),-1)[:,0,None]
    mean=(counts*means).sum(0)/counts.sum(0)
    variance=(counts*(stds**2+(means-mean)**2)).sum(0)/counts.sum(0)
    out=dict(mean=mean,std=np.sqrt(variance),min=np.min([x['min'] for x in values],0),max=np.max([x['max'] for x in values],0))
    return {k:np.pad(np.asarray(v),(0,32-len(v))).astype(np.float32) for k,v in out.items()}

def main():
    p=argparse.ArgumentParser();p.add_argument('--source',required=True);p.add_argument('--tokenizer',required=True);p.add_argument('--output',required=True);p.add_argument('--dev-count',type=int,default=0)
    p.add_argument('--shard-count',type=int,default=1);p.add_argument('--shard-index',type=int,default=0)
    args=p.parse_args();root=pathlib.Path(args.source);out=pathlib.Path(args.output);out.mkdir(parents=True,exist_ok=True)
    previous=json.loads((out/'asset-index.json').read_text()) if (out/'asset-index.json').exists() else {}
    source_manifest=json.loads((root/'download-manifest.json').read_text())
    if not 0<=args.shard_index<args.shard_count:raise ValueError('Invalid shard')
    torch.set_num_threads(2)
    episodes=[json.loads(x) for x in (root/'meta/episodes.jsonl').read_text().splitlines()];episodes={x['episode_index']:x for x in episodes}
    records=[json.loads(x) for x in (root/'meta/episodes_stats.jsonl').read_text().splitlines()];records=[x for x in records if x['episode_index']<60]
    if len(records)!=60:raise ValueError('Missing selected-episode stats')
    stats={key:aggregate(records,key) for key in ('state','actions')}
    if args.shard_index==0:(out/'normalization.json').write_text(json.dumps({k:{s:v.tolist() for s,v in value.items()} for k,value in stats.items()},indent=2))
    tok=GemmaTokenizer(vocab_file=args.tokenizer,add_bos_token=True,add_eos_token=False,padding_side='right')
    manifest={'train_ids':list(range(60)),'dev_ids':list(range(60,60+args.dev_count)),'episodes':{},'prefixes':{},'protocol':{'kind':'new_shared_comparison','source':'TensorAuto/libero','source_revision':source_manifest['revision'],'tokenizer_sha256':hashlib.sha256(pathlib.Path(args.tokenizer).read_bytes()).hexdigest(),'episodes':list(range(60)),'fps':20,'sequence_length':32,'stride':1,'chunk':10,'normalization_epsilon':1e-6,'normalization_version':1,'dataset_float_boundary':'bfloat16','prompt_max_length':256,'camera_order':['image','wrist_image'],'boundary_policy':'full_valid_windows_for_both_models','not_exact_legacy_opentau_sampler':True}}
    state_stats={k:torch.from_numpy(v) for k,v in stats['state'].items()};action_stats={k:torch.from_numpy(v) for k,v in stats['actions'].items()}
    for episode in range(60+args.dev_count):
        if episode%args.shard_count!=args.shard_index:continue
        if str(episode) in previous.get('episodes',{}):
            manifest['episodes'][str(episode)]=previous['episodes'][str(episode)];continue
        folder=out/f'episode_{episode:06d}';folder.mkdir(exist_ok=True)
        table=pq.read_table(root/f'data/chunk-000/episode_{episode:06d}.parquet');data=table.to_pydict();n=len(data['state'])
        raw_state=torch.zeros(n,32);raw_state[:,:8]=torch.tensor(data['state']);raw_state=raw_state.bfloat16().float()
        raw_action=torch.zeros(n,32);raw_action[:,:7]=torch.tensor(data['actions']);raw_action=raw_action.bfloat16().float()
        ranges=state_stats['max']-state_stats['min'];safe=torch.where(ranges.abs()<1e-6,1.,ranges)
        states=((raw_state-state_stats['min']+.5*(safe-ranges))/(safe+1e-6))*2-1
        std=action_stats['std'];std=torch.where(std.abs()<1e-6,1.,std)
        actions=(raw_action-action_stats['mean'])/(std+1e-6)
        targets=torch.stack([actions[torch.arange(r,r+10).clamp(max=n-1)] for r in range(n)])
        np.save(folder/'state.npy',states.numpy());np.save(folder/'targets.npy',targets.numpy())
        prompt=episodes[episode]['tasks'][0]
        bins=((states.clamp(-1,1)+1)*128).long().clamp(0,255)
        prompts=[f'Task: {prompt}, State: '+ ' '.join(map(str,row))+';\n' for row in bins.tolist()]
        tokenized=tok(prompts,padding='max_length',max_length=256,truncation=True,return_tensors='np')
        np.save(folder/'prompt.npy',tokenized['input_ids'].astype(np.int64));np.save(folder/'prompt_mask.npy',tokenized['attention_mask'].astype(bool))
        for key in ('image','wrist_image'):
            existing=folder/f'{key}.npz'
            if existing.exists():
                with np.load(existing,allow_pickle=False) as archive:
                    saved=archive['values']
                    if saved.shape!=(n,3,224,224) or saved.dtype!=np.float32 or not np.isfinite(saved).all():
                        raise ValueError('Existing compressed image is invalid')
                del saved
                continue
            mm=np.empty((n,3,224,224),dtype=np.float32)
            for lo in range(0,n,32):
                ims=[]
                for image in data[key][lo:lo+32]:
                    pixels=np.array(Image.open(io.BytesIO(image['bytes'])).convert('RGB'),copy=True)
                    ims.append(torch.from_numpy(pixels).permute(2,0,1).float()/255.)
                resized=F.interpolate(torch.stack(ims),size=(224,224),mode='bilinear',align_corners=False)
                mm[lo:lo+len(ims)]=(resized.bfloat16()*2-1).float().numpy()
            # Lossless compression preserves the exact BF16-rounded F32 inputs.
            temporary=folder/f'{key}.tmp.npz'
            with zipfile.ZipFile(temporary,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=1) as archive:
                with archive.open('values.npy','w',force_zip64=True) as stream:
                    np.lib.format.write_array(stream,mm,allow_pickle=False)
            with np.load(temporary,allow_pickle=False) as restored:
                if not np.array_equal(restored['values'],mm):
                    raise ValueError('Compressed image round-trip differs')
            temporary.replace(folder/f'{key}.npz')
            del mm
        # Last source row with a complete action chunk. The shared sampler also
        # requires all 32 observation rows to be inside this set.
        rows=list(range(n-9))
        manifest['episodes'][str(episode)]={'folder':folder.name,'manifest':{'rows':rows,'fields':{'state':'state.npy','tokenized_prompt':'prompt.npy','tokenized_prompt_mask':'prompt_mask.npy'}},'length':n,'task':prompt}
        print(json.dumps({'episode':episode,'frames':n,'complete_target_rows':len(rows)}),flush=True)
    def file_hash(path):
        h=hashlib.sha256()
        with path.open('rb') as f:
            while chunk:=f.read(8*1024*1024):h.update(chunk)
        return h.hexdigest()
    source_hashes={x['path']:x['sha256'] for x in source_manifest['files']}
    for episode,record in manifest['episodes'].items():
        record['file_sha256']={f.name:file_hash(f) for f in sorted((out/record['folder']).iterdir()) if f.suffix in ('.npy','.npz')}
        relative=f'data/chunk-000/episode_{int(episode):06d}.parquet'
        record['source_parquet_sha256']=source_hashes[relative]
        if file_hash(root/relative)!=source_hashes[relative]:raise ValueError('Source parquet changed')
    # Arrays are indexed by source row; the rows metadata starts at zero.
    name='asset-index.json' if args.shard_count==1 else f'asset-index-shard-{args.shard_index}.json'
    temporary=out/(name+'.tmp');temporary.write_text(json.dumps(manifest,indent=2));temporary.replace(out/name)

if __name__=='__main__':main()