File size: 4,974 Bytes
a1c64e0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
"""Restore the original L4 Train/Dev arrays using its supplied source and hashes."""
import concurrent.futures
import hashlib
import importlib
import json
import os
from pathlib import Path
import shutil
import sys
import time
import urllib.request

ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'
BUNDLE = LOCAL / 'legacy-l4-handoff/bundle'
DATA = LOCAL / 'legacy-ft-data'


def sha(path):
    digest = hashlib.sha256()
    with path.open('rb') as stream:
        while chunk := stream.read(8 * 1024 * 1024):
            digest.update(chunk)
    return digest.hexdigest()


def download_episode(episode, selection):
    target = DATA / 'raw' / f"episode_{episode['episode_index']:06d}.parquet"
    if target.exists():
        if sha(target) != episode['lfs_sha256']:
            raise ValueError('Existing episode checksum differs')
        return
    partial = target.with_suffix('.parquet.partial')
    url = f"https://huggingface.co/datasets/{selection['repo']}/resolve/{selection['revision']}/{episode['path']}?download=true"
    for attempt in range(5):
        offset = partial.stat().st_size if partial.exists() else 0
        if offset == episode['size']:
            break
        request = urllib.request.Request(url, headers={'Range': f'bytes={offset}-'} if offset else {})
        try:
            with urllib.request.urlopen(request, timeout=90) as response:
                if offset and (response.status != 206 or not response.headers.get('Content-Range', '').startswith(f'bytes {offset}-')):
                    raise ValueError('Incorrect resumed episode range')
                with partial.open('ab' if offset else 'wb') as stream:
                    while chunk := response.read(1024 * 1024):
                        stream.write(chunk)
            if partial.stat().st_size == episode['size']:
                break
        except Exception:
            if attempt == 4:
                raise
            time.sleep(2 + attempt)
    if partial.stat().st_size != episode['size'] or sha(partial) != episode['lfs_sha256']:
        raise ValueError('Downloaded episode size/hash mismatch')
    partial.replace(target)
    print('Raw episode verified', episode['episode_index'], flush=True)


def main():
    for folder in ('protocol', 'raw', 'processed', 'runs'):
        (DATA / folder).mkdir(parents=True, exist_ok=True)
    selection_source = BUNDLE / 'sequence_dataset_v1/protocol/selection.json'
    selection = json.loads(selection_source.read_text())
    shutil.copyfile(selection_source, DATA / 'protocol/selection.json')
    selected = [e for e in selection['selected'] if e['split'] in ('train', 'dev')]
    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
        list(pool.map(lambda episode: download_episode(episode, selection), selected))

    cache = LOCAL / 'legacy-openpi-cache'
    (cache / 'big_vision').mkdir(parents=True, exist_ok=True)
    shutil.copyfile(LOCAL / 'assets/paligemma_tokenizer.model', cache / 'big_vision/paligemma_tokenizer.model')
    os.environ['OPENPI_DATA_HOME'] = str(cache)
    source = BUNDLE / 'sequence_dataset_v1/src'
    sys.path.insert(0, str(source))
    common = importlib.import_module('common')
    common.R = DATA
    importlib.import_module('read_selected').main()
    fixture = importlib.import_module('fixture_preprocessing')
    fixture.ASSETS = LOCAL / 'assets/pi05_libero/assets'
    importlib.import_module('preprocess_selected').main()

    checked = 0
    records = {}
    for episode in selected:
        eid = episode['episode_index']
        original = json.loads((BUNDLE / f'sequence_dataset_v1/processed/episode_{eid:06d}/manifest.json').read_text())
        folder = DATA / f'processed/episode_{eid:06d}'
        for name, expected in original['files'].items():
            if sha(folder / name) != expected:
                raise ValueError(f'Original processed file differs: {eid}/{name}')
            checked += 1
        current = json.loads((folder / 'manifest.json').read_text())
        records[str(eid)] = dict(folder=f'processed/episode_{eid:06d}', manifest=current, file_sha256=current['files'])
    normalization = LOCAL / 'assets/pi05_libero/assets/physical-intelligence/libero/norm_stats.json'
    shutil.copyfile(normalization, DATA / 'normalization.json')
    index = dict(train_ids=[e['episode_index'] for e in selected if e['split'] == 'train'],
                 dev_ids=[e['episode_index'] for e in selected if e['split'] == 'dev'],
                 episodes=records, prefixes={}, protocol=dict(kind='original_L4_libero_ft', selection_sha256=sha(selection_source)))
    (DATA / 'asset-index.json').write_text(json.dumps(index, indent=2) + '\n')
    (DATA / 'original-file-verification.json').write_text(json.dumps(dict(files_checked=checked, episodes=len(selected), all_original_hashes_match=True), indent=2))
    print('ORIGINAL_L4_PROCESSED_FILES_VERIFIED', checked, flush=True)


if __name__ == '__main__':
    main()