File size: 4,974 Bytes
a1c64e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | """Restore the original L4 Train/Dev arrays using its supplied source and hashes."""
import concurrent.futures
import hashlib
import importlib
import json
import os
from pathlib import Path
import shutil
import sys
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'
BUNDLE = LOCAL / 'legacy-l4-handoff/bundle'
DATA = LOCAL / 'legacy-ft-data'
def sha(path):
digest = hashlib.sha256()
with path.open('rb') as stream:
while chunk := stream.read(8 * 1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def download_episode(episode, selection):
target = DATA / 'raw' / f"episode_{episode['episode_index']:06d}.parquet"
if target.exists():
if sha(target) != episode['lfs_sha256']:
raise ValueError('Existing episode checksum differs')
return
partial = target.with_suffix('.parquet.partial')
url = f"https://huggingface.co/datasets/{selection['repo']}/resolve/{selection['revision']}/{episode['path']}?download=true"
for attempt in range(5):
offset = partial.stat().st_size if partial.exists() else 0
if offset == episode['size']:
break
request = urllib.request.Request(url, headers={'Range': f'bytes={offset}-'} if offset else {})
try:
with urllib.request.urlopen(request, timeout=90) as response:
if offset and (response.status != 206 or not response.headers.get('Content-Range', '').startswith(f'bytes {offset}-')):
raise ValueError('Incorrect resumed episode range')
with partial.open('ab' if offset else 'wb') as stream:
while chunk := response.read(1024 * 1024):
stream.write(chunk)
if partial.stat().st_size == episode['size']:
break
except Exception:
if attempt == 4:
raise
time.sleep(2 + attempt)
if partial.stat().st_size != episode['size'] or sha(partial) != episode['lfs_sha256']:
raise ValueError('Downloaded episode size/hash mismatch')
partial.replace(target)
print('Raw episode verified', episode['episode_index'], flush=True)
def main():
for folder in ('protocol', 'raw', 'processed', 'runs'):
(DATA / folder).mkdir(parents=True, exist_ok=True)
selection_source = BUNDLE / 'sequence_dataset_v1/protocol/selection.json'
selection = json.loads(selection_source.read_text())
shutil.copyfile(selection_source, DATA / 'protocol/selection.json')
selected = [e for e in selection['selected'] if e['split'] in ('train', 'dev')]
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
list(pool.map(lambda episode: download_episode(episode, selection), selected))
cache = LOCAL / 'legacy-openpi-cache'
(cache / 'big_vision').mkdir(parents=True, exist_ok=True)
shutil.copyfile(LOCAL / 'assets/paligemma_tokenizer.model', cache / 'big_vision/paligemma_tokenizer.model')
os.environ['OPENPI_DATA_HOME'] = str(cache)
source = BUNDLE / 'sequence_dataset_v1/src'
sys.path.insert(0, str(source))
common = importlib.import_module('common')
common.R = DATA
importlib.import_module('read_selected').main()
fixture = importlib.import_module('fixture_preprocessing')
fixture.ASSETS = LOCAL / 'assets/pi05_libero/assets'
importlib.import_module('preprocess_selected').main()
checked = 0
records = {}
for episode in selected:
eid = episode['episode_index']
original = json.loads((BUNDLE / f'sequence_dataset_v1/processed/episode_{eid:06d}/manifest.json').read_text())
folder = DATA / f'processed/episode_{eid:06d}'
for name, expected in original['files'].items():
if sha(folder / name) != expected:
raise ValueError(f'Original processed file differs: {eid}/{name}')
checked += 1
current = json.loads((folder / 'manifest.json').read_text())
records[str(eid)] = dict(folder=f'processed/episode_{eid:06d}', manifest=current, file_sha256=current['files'])
normalization = LOCAL / 'assets/pi05_libero/assets/physical-intelligence/libero/norm_stats.json'
shutil.copyfile(normalization, DATA / 'normalization.json')
index = dict(train_ids=[e['episode_index'] for e in selected if e['split'] == 'train'],
dev_ids=[e['episode_index'] for e in selected if e['split'] == 'dev'],
episodes=records, prefixes={}, protocol=dict(kind='original_L4_libero_ft', selection_sha256=sha(selection_source)))
(DATA / 'asset-index.json').write_text(json.dumps(index, indent=2) + '\n')
(DATA / 'original-file-verification.json').write_text(json.dumps(dict(files_checked=checked, episodes=len(selected), all_original_hashes_match=True), indent=2))
print('ORIGINAL_L4_PROCESSED_FILES_VERIFIED', checked, flush=True)
if __name__ == '__main__':
main()
|