"""Download immutable LIBERO inputs and checksum-audited official base objects.""" import argparse,base64,concurrent.futures,hashlib,json,pathlib,urllib.parse,urllib.request import google_crc32c DATASET_REVISION='72f75834f0b5546940f8bc26a848d2593f77bd59' def digest(path): h=hashlib.sha256();crc=google_crc32c.Checksum();md5=hashlib.md5() with path.open('rb') as f: while block:=f.read(8*1024*1024):h.update(block);crc.update(block);md5.update(block) return h.hexdigest(),base64.b64encode(crc.digest()).decode(),base64.b64encode(md5.digest()).decode() def download(url,path): path.parent.mkdir(parents=True,exist_ok=True) if path.exists():return temp=path.with_name(path.name+'.partial') with urllib.request.urlopen(url,timeout=120) as src,temp.open('wb') as dst: while block:=src.read(4*1024*1024):dst.write(block) temp.replace(path) def main(): p=argparse.ArgumentParser();p.add_argument('--assets',default='.release_local/assets');p.add_argument('--audit',default='.release_local/audit');p.add_argument('--dataset-revision',default=DATASET_REVISION);p.add_argument('--workers',type=int,default=4);a=p.parse_args() root=pathlib.Path(a.assets).resolve();audit=pathlib.Path(a.audit);audit.mkdir(parents=True,exist_ok=True) listing=json.load(urllib.request.urlopen('https://storage.googleapis.com/storage/v1/b/openpi-assets/o?prefix=checkpoints/pi05_base/&maxResults=1000')) if listing.get('nextPageToken'):raise ValueError('Unexpected paginated checkpoint listing') (audit/'pi05-base-gcs-list.json').write_text(json.dumps(listing,indent=2)) def base_object(item): path=(root/'pi05_base').resolve()/item['name'].removeprefix('checkpoints/pi05_base/') url='https://storage.googleapis.com/openpi-assets/'+urllib.parse.quote(item['name'],safe='/')+'?generation='+item['generation'] download(url,path);sha,crc,md5=digest(path) if path.stat().st_size!=int(item['size']) or crc!=item['crc32c'] or ('md5Hash' in item and md5!=item['md5Hash']):raise ValueError('Official object checksum differs: '+str(path)) print(path,flush=True);return dict(path=str(path),sha256=sha,gcs_generation=item['generation'],source='gs://openpi-assets/'+item['name']) with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:base=list(pool.map(base_object,listing['items'])) (audit/'pi05-base-download.json').write_text(json.dumps(base,indent=2)) info=json.load(urllib.request.urlopen('https://huggingface.co/api/datasets/TensorAuto/libero/revision/'+a.dataset_revision)) if info['sha']!=a.dataset_revision:raise ValueError('Expected an immutable dataset commit') files=[x['rfilename'] for x in info['siblings'] if x['rfilename'].startswith('meta/') or x['rfilename']=='README.md' or any(f'episode_{i:06d}.' in x['rfilename'] for i in range(70))] def data_file(name): path=root/'libero'/name;url=f'https://huggingface.co/datasets/TensorAuto/libero/resolve/{a.dataset_revision}/{name}' download(url,path);sha,_,_=digest(path);print(path,flush=True);return dict(path=name,sha256=sha,size=path.stat().st_size) with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:records=list(pool.map(data_file,files)) manifest=dict(repo='TensorAuto/libero',revision=a.dataset_revision,episodes=list(range(70)),files=records) (root/'libero/download-manifest.json').write_text(json.dumps(manifest,indent=2));(audit/'libero-download.json').write_text(json.dumps(manifest,indent=2)) tokenizer=root/'paligemma_tokenizer.model';download('https://storage.googleapis.com/big_vision/paligemma_tokenizer.model',tokenizer) if digest(tokenizer)[0]!='8986bb4f423f07f8c7f70d0dbe3526fb2316056c17bae71b1ea975e77a168fc6':raise ValueError('Tokenizer differs') if __name__=='__main__':main()