"""Restore the original L4 Train/Dev arrays using its supplied source and hashes.""" import concurrent.futures import hashlib import importlib import json import os from pathlib import Path import shutil import sys import time import urllib.request ROOT = Path(__file__).resolve().parents[1] LOCAL = ROOT / '.release_local' BUNDLE = LOCAL / 'legacy-l4-handoff/bundle' DATA = LOCAL / 'legacy-ft-data' def sha(path): digest = hashlib.sha256() with path.open('rb') as stream: while chunk := stream.read(8 * 1024 * 1024): digest.update(chunk) return digest.hexdigest() def download_episode(episode, selection): target = DATA / 'raw' / f"episode_{episode['episode_index']:06d}.parquet" if target.exists(): if sha(target) != episode['lfs_sha256']: raise ValueError('Existing episode checksum differs') return partial = target.with_suffix('.parquet.partial') url = f"https://huggingface.co/datasets/{selection['repo']}/resolve/{selection['revision']}/{episode['path']}?download=true" for attempt in range(5): offset = partial.stat().st_size if partial.exists() else 0 if offset == episode['size']: break request = urllib.request.Request(url, headers={'Range': f'bytes={offset}-'} if offset else {}) try: with urllib.request.urlopen(request, timeout=90) as response: if offset and (response.status != 206 or not response.headers.get('Content-Range', '').startswith(f'bytes {offset}-')): raise ValueError('Incorrect resumed episode range') with partial.open('ab' if offset else 'wb') as stream: while chunk := response.read(1024 * 1024): stream.write(chunk) if partial.stat().st_size == episode['size']: break except Exception: if attempt == 4: raise time.sleep(2 + attempt) if partial.stat().st_size != episode['size'] or sha(partial) != episode['lfs_sha256']: raise ValueError('Downloaded episode size/hash mismatch') partial.replace(target) print('Raw episode verified', episode['episode_index'], flush=True) def main(): for folder in ('protocol', 'raw', 'processed', 'runs'): (DATA / folder).mkdir(parents=True, exist_ok=True) selection_source = BUNDLE / 'sequence_dataset_v1/protocol/selection.json' selection = json.loads(selection_source.read_text()) shutil.copyfile(selection_source, DATA / 'protocol/selection.json') selected = [e for e in selection['selected'] if e['split'] in ('train', 'dev')] with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: list(pool.map(lambda episode: download_episode(episode, selection), selected)) cache = LOCAL / 'legacy-openpi-cache' (cache / 'big_vision').mkdir(parents=True, exist_ok=True) shutil.copyfile(LOCAL / 'assets/paligemma_tokenizer.model', cache / 'big_vision/paligemma_tokenizer.model') os.environ['OPENPI_DATA_HOME'] = str(cache) source = BUNDLE / 'sequence_dataset_v1/src' sys.path.insert(0, str(source)) common = importlib.import_module('common') common.R = DATA importlib.import_module('read_selected').main() fixture = importlib.import_module('fixture_preprocessing') fixture.ASSETS = LOCAL / 'assets/pi05_libero/assets' importlib.import_module('preprocess_selected').main() checked = 0 records = {} for episode in selected: eid = episode['episode_index'] original = json.loads((BUNDLE / f'sequence_dataset_v1/processed/episode_{eid:06d}/manifest.json').read_text()) folder = DATA / f'processed/episode_{eid:06d}' for name, expected in original['files'].items(): if sha(folder / name) != expected: raise ValueError(f'Original processed file differs: {eid}/{name}') checked += 1 current = json.loads((folder / 'manifest.json').read_text()) records[str(eid)] = dict(folder=f'processed/episode_{eid:06d}', manifest=current, file_sha256=current['files']) normalization = LOCAL / 'assets/pi05_libero/assets/physical-intelligence/libero/norm_stats.json' shutil.copyfile(normalization, DATA / 'normalization.json') index = dict(train_ids=[e['episode_index'] for e in selected if e['split'] == 'train'], dev_ids=[e['episode_index'] for e in selected if e['split'] == 'dev'], episodes=records, prefixes={}, protocol=dict(kind='original_L4_libero_ft', selection_sha256=sha(selection_source))) (DATA / 'asset-index.json').write_text(json.dumps(index, indent=2) + '\n') (DATA / 'original-file-verification.json').write_text(json.dumps(dict(files_checked=checked, episodes=len(selected), all_original_hashes_match=True), indent=2)) print('ORIGINAL_L4_PROCESSED_FILES_VERIFIED', checked, flush=True) if __name__ == '__main__': main()