File size: 5,353 Bytes
fffb321
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
"""Rebuild original batch-one LIBERO-FT prefixes from hash-matched inputs."""
import json
from pathlib import Path
import shutil
import sys
import numpy as np
import torch

ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'
sys.path[:0] = [str(ROOT / 'src'), str(ROOT.parent / 'openpi/src'), str(ROOT.parent / 'openpi/packages/openpi-client/src')]
from robottt_pi05.openpi_backend import load_model
from robottt_pi05.eval_io import atomic_json, atomic_tensor, sha, tensor_hash
from openpi.models.model import Observation
from openpi.models_pytorch.pi0_pytorch import make_att_2d_masks


def main():
    torch.set_num_threads(4)
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    data = LOCAL / 'legacy-ft-data'
    out = LOCAL / 'legacy-ft-prefix'
    out.mkdir(exist_ok=True)
    manifest = json.loads((data / 'asset-index.json').read_text())
    checkpoint = LOCAL / 'ft-base-reproduction/converted/pi05_libero_pytorch/model.safetensors'
    model_hash = sha(checkpoint)
    if model_hash != 'caf7fce67043d34faddbb9e37ac6f6ae33e2940f2bac9273464ad509cb9e1e31':
        raise ValueError('Original FT checkpoint hash differs')
    model = load_model(source=ROOT.parent / 'openpi', checkpoint=checkpoint, profile='legacy_pi05_libero')
    cameras = ['base_0_rgb', 'left_wrist_0_rgb', 'right_wrist_0_rgb']
    count = 0
    expected = sum(len(r['manifest']['rows']) for r in manifest['episodes'].values())
    assert expected == 521
    for episode, record in manifest['episodes'].items():
        folder = data / record['folder']
        arrays = {k: np.load(folder / name, mmap_mode='r') for k, name in record['manifest']['fields'].items()}
        for index, row in enumerate(record['manifest']['rows']):
            key = f'e{int(episode):06d}_r{row:06d}'
            destination = out / (key + '.pt')
            metadata = out / (key + '.json')
            if metadata.exists():
                rec = json.loads(metadata.read_text())
                if sha(destination) != rec['sha256'] or rec['provenance']['model_checkpoint_sha256'] != model_hash:
                    raise ValueError('Existing prefix differs')
            else:
                tensors = {k: torch.from_numpy(np.array(v[index:index+1])).cuda() for k, v in arrays.items()}
                provenance = dict(episode=int(episode), source_row=row,
                                  inputs={k: tensor_hash(v.cpu()) for k, v in tensors.items()},
                                  model_checkpoint_sha256=model_hash, model_revision='215abfb217dbac7d5f1273282331b9b1866c0479',
                                  dtype='BF16 prefix', attention='eager', positions='padding cumsum-1',
                                  normal_observation=True, selection_sha256=manifest['protocol']['selection_sha256'])
                observation = Observation(images={c: tensors['image/'+c] for c in cameras},
                                          image_masks={c: tensors['image_mask/'+c] for c in cameras},
                                          state=tensors['state'], tokenized_prompt=tensors['tokenized_prompt'],
                                          tokenized_prompt_mask=tensors['tokenized_prompt_mask'])
                with torch.no_grad():
                    images, masks, tokens, token_mask, state = model._preprocess_observation(observation, train=False)
                    embedding, pm, pa = model.embed_prefix(images, masks, tokens, token_mask)
                    embedding = embedding.to(torch.bfloat16)
                    _, cache = model.paligemma_with_expert(attention_mask=model._prepare_attention_masks_4d(make_att_2d_masks(pm, pa)),
                                                         position_ids=pm.cumsum(1)-1, inputs_embeds=[embedding, None], use_cache=True)
                    legacy = cache.to_legacy_cache() if hasattr(cache, 'to_legacy_cache') else tuple(cache)
                    value = dict(prefix_embedding=embedding.cpu(), padding_mask=pm.cpu(), attention_blocks=pa.cpu(), state=state.cpu(),
                                 KV=tuple((k.cpu(), v.cpu()) for k, v in legacy), provenance=provenance)
                if count == 0:
                    size = sum(t.numel()*t.element_size() for t in [value['prefix_embedding'], value['padding_mask'], value['attention_blocks'], value['state'], *[t for pair in value['KV'] for t in pair]])
                    if shutil.disk_usage(out).free - expected*size < 160*2**30:
                        raise RuntimeError('Insufficient prefix storage headroom')
                    print('PREFIX_SIZE_BYTES', size, 'ESTIMATED_TOTAL', expected*size, flush=True)
                digest = atomic_tensor(destination, value)
                rec = dict(path=destination.name, sha256=digest, provenance=provenance)
                atomic_json(metadata, rec)
                del tensors, observation, images, masks, tokens, token_mask, state, embedding, pm, pa, cache, legacy, value
            manifest['prefixes'][key] = rec
            count += 1
        atomic_json(out / 'progress.json', dict(prefixes=count, total=expected), replace=True)
        print('PREFIX_EPISODE', episode, count, flush=True)
    atomic_json(out / 'asset-index.json', manifest)
    print('LEGACY_PREFIXES_COMPLETE', count, flush=True)


if __name__ == '__main__':
    main()