"""Prepare a new, explicitly versioned LIBERO comparison dataset. Matches OpenTau's selected-episode stats, BF16 dataset boundary, MIN_MAX state, MEAN_STD actions, two cameras and state-bearing language prompt. Full valid T32 windows are used in the exported shared plan (no implicit boundary repeats). """ import argparse,io,json,pathlib,sys,hashlib,zipfile import numpy as np import torch import torch.nn.functional as F from PIL import Image import pyarrow.parquet as pq from transformers import GemmaTokenizer def aggregate(records,key): values=[r['stats'][key] for r in records] means=np.array([x['mean'] for x in values],dtype=np.float64) stds=np.array([x['std'] for x in values],dtype=np.float64) counts=np.array([x['count'] for x in values],dtype=np.float64).reshape(len(values),-1)[:,0,None] mean=(counts*means).sum(0)/counts.sum(0) variance=(counts*(stds**2+(means-mean)**2)).sum(0)/counts.sum(0) out=dict(mean=mean,std=np.sqrt(variance),min=np.min([x['min'] for x in values],0),max=np.max([x['max'] for x in values],0)) return {k:np.pad(np.asarray(v),(0,32-len(v))).astype(np.float32) for k,v in out.items()} def main(): p=argparse.ArgumentParser();p.add_argument('--source',required=True);p.add_argument('--tokenizer',required=True);p.add_argument('--output',required=True);p.add_argument('--dev-count',type=int,default=0) p.add_argument('--shard-count',type=int,default=1);p.add_argument('--shard-index',type=int,default=0) args=p.parse_args();root=pathlib.Path(args.source);out=pathlib.Path(args.output);out.mkdir(parents=True,exist_ok=True) previous=json.loads((out/'asset-index.json').read_text()) if (out/'asset-index.json').exists() else {} source_manifest=json.loads((root/'download-manifest.json').read_text()) if not 0<=args.shard_index