robottt-pi05-base-comparison / code /scripts /download_comparison_assets.py
jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
3.84 kB
"""Download immutable LIBERO inputs and checksum-audited official base objects."""
import argparse,base64,concurrent.futures,hashlib,json,pathlib,urllib.parse,urllib.request
import google_crc32c
DATASET_REVISION='72f75834f0b5546940f8bc26a848d2593f77bd59'
def digest(path):
h=hashlib.sha256();crc=google_crc32c.Checksum();md5=hashlib.md5()
with path.open('rb') as f:
while block:=f.read(8*1024*1024):h.update(block);crc.update(block);md5.update(block)
return h.hexdigest(),base64.b64encode(crc.digest()).decode(),base64.b64encode(md5.digest()).decode()
def download(url,path):
path.parent.mkdir(parents=True,exist_ok=True)
if path.exists():return
temp=path.with_name(path.name+'.partial')
with urllib.request.urlopen(url,timeout=120) as src,temp.open('wb') as dst:
while block:=src.read(4*1024*1024):dst.write(block)
temp.replace(path)
def main():
p=argparse.ArgumentParser();p.add_argument('--assets',default='.release_local/assets');p.add_argument('--audit',default='.release_local/audit');p.add_argument('--dataset-revision',default=DATASET_REVISION);p.add_argument('--workers',type=int,default=4);a=p.parse_args()
root=pathlib.Path(a.assets).resolve();audit=pathlib.Path(a.audit);audit.mkdir(parents=True,exist_ok=True)
listing=json.load(urllib.request.urlopen('https://storage.googleapis.com/storage/v1/b/openpi-assets/o?prefix=checkpoints/pi05_base/&maxResults=1000'))
if listing.get('nextPageToken'):raise ValueError('Unexpected paginated checkpoint listing')
(audit/'pi05-base-gcs-list.json').write_text(json.dumps(listing,indent=2))
def base_object(item):
path=(root/'pi05_base').resolve()/item['name'].removeprefix('checkpoints/pi05_base/')
url='https://storage.googleapis.com/openpi-assets/'+urllib.parse.quote(item['name'],safe='/')+'?generation='+item['generation']
download(url,path);sha,crc,md5=digest(path)
if path.stat().st_size!=int(item['size']) or crc!=item['crc32c'] or ('md5Hash' in item and md5!=item['md5Hash']):raise ValueError('Official object checksum differs: '+str(path))
print(path,flush=True);return dict(path=str(path),sha256=sha,gcs_generation=item['generation'],source='gs://openpi-assets/'+item['name'])
with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:base=list(pool.map(base_object,listing['items']))
(audit/'pi05-base-download.json').write_text(json.dumps(base,indent=2))
info=json.load(urllib.request.urlopen('https://huggingface.co/api/datasets/TensorAuto/libero/revision/'+a.dataset_revision))
if info['sha']!=a.dataset_revision:raise ValueError('Expected an immutable dataset commit')
files=[x['rfilename'] for x in info['siblings'] if x['rfilename'].startswith('meta/') or x['rfilename']=='README.md' or any(f'episode_{i:06d}.' in x['rfilename'] for i in range(70))]
def data_file(name):
path=root/'libero'/name;url=f'https://huggingface.co/datasets/TensorAuto/libero/resolve/{a.dataset_revision}/{name}'
download(url,path);sha,_,_=digest(path);print(path,flush=True);return dict(path=name,sha256=sha,size=path.stat().st_size)
with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:records=list(pool.map(data_file,files))
manifest=dict(repo='TensorAuto/libero',revision=a.dataset_revision,episodes=list(range(70)),files=records)
(root/'libero/download-manifest.json').write_text(json.dumps(manifest,indent=2));(audit/'libero-download.json').write_text(json.dumps(manifest,indent=2))
tokenizer=root/'paligemma_tokenizer.model';download('https://storage.googleapis.com/big_vision/paligemma_tokenizer.model',tokenizer)
if digest(tokenizer)[0]!='8986bb4f423f07f8c7f70d0dbe3526fb2316056c17bae71b1ea975e77a168fc6':raise ValueError('Tokenizer differs')
if __name__=='__main__':main()