File size: 9,633 Bytes
0f5d220 add1d5f c62a767 0f5d220 add1d5f 0f5d220 c62a767 0f5d220 c347d96 0f5d220 c62a767 0f5d220 c62a767 0f5d220 c62a767 0f5d220 c62a767 0f5d220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """Cache only frozen VLM K/V for a new official-base experiment."""
import argparse,json,pathlib,time,sys,shutil
from concurrent.futures import ThreadPoolExecutor
import numpy as np
import torch
from robottt_pi05.openpi_backend import load_model
from robottt_pi05.eval_io import atomic_json,atomic_tensor,sha,tensor_hash
def main():
p=argparse.ArgumentParser();p.add_argument('--openpi-source',required=True);p.add_argument('--checkpoint',required=True);p.add_argument('--data',required=True);p.add_argument('--output',required=True);p.add_argument('--batch-size',type=int,default=8);p.add_argument('--max-prefixes',type=int)
args=p.parse_args();source=pathlib.Path(args.openpi_source).resolve();sys.path[:0]=[str(source/'src'),str(source/'packages/openpi-client/src')]
from openpi.models_pytorch.pi0_pytorch import make_att_2d_masks
torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=True;torch.backends.cudnn.allow_tf32=True
data=pathlib.Path(args.data);out=pathlib.Path(args.output);out.mkdir(parents=True,exist_ok=True)
manifest=json.loads((data/'asset-index.json').read_text());weight_sha=sha(args.checkpoint);normalization_sha=sha(data/'normalization.json')
estimated_prefixes=args.max_prefixes or sum(min(32,len(rec['manifest']['rows'])) if int(ep) in manifest.get('dev_ids',[]) else len(rec['manifest']['rows']) for ep,rec in manifest['episodes'].items())
if shutil.disk_usage(out).free-estimated_prefixes*12*2**20<160*2**30:
raise RuntimeError('Disk cache would risk node ephemeral-storage eviction; use launch_comparison.py --continue-after-smoke --on-demand')
manifest['protocol']['model_sha256']=weight_sha
model=load_model(source=source,checkpoint=args.checkpoint,profile='pi05_base_libero_protocol')
if args.max_prefixes is None:
from robottt_pi05.profile_common import benchmark,runtime_signature
proof=pathlib.Path('.release_local/audit/runtime-profile-parity.json')
if not proof.exists() or json.loads(proof.read_text()).get('implementation_sha256')!=runtime_signature():
benchmark(model,root=pathlib.Path(__file__).resolve().parents[1],cache=out,data=data)
count=0;begin=time.monotonic();compaction_checked=False
writers=ThreadPoolExecutor(max_workers=4)
def commit_prefix(path,z,key):
digest=atomic_tensor(path,z)
rec=dict(path=path.name,sha256=digest,provenance=z['provenance'])
atomic_json(out/(key+'.json'),rec)
return key,rec
expected=sum(min(32,len(rec['manifest']['rows'])) if int(ep) in manifest.get('dev_ids',[]) else len(rec['manifest']['rows']) for ep,rec in manifest['episodes'].items())
for episode,record in manifest['episodes'].items():
folder=data/record['folder'];rows=record['manifest']['rows'];
if int(episode) in manifest.get('dev_ids',[]):rows=rows[:32]
fields={}
for n in ('image','wrist_image','state','prompt','prompt_mask'):
compressed=folder/(n+'.npz')
if compressed.exists():
with np.load(compressed,allow_pickle=False) as archive:
fields[n]=archive['values']
else:fields[n]=np.load(folder/(n+'.npy'),mmap_mode='r')
for lo in range(0,len(rows),args.batch_size):
selected=rows[lo:lo+args.batch_size];pending=[]
for row in selected:
key=f'e{int(episode):06d}_r{row:06d}';path=out/(key+'.pt');sidecar=out/(key+'.json')
if path.exists() and not sidecar.exists():
recovered=torch.load(path,map_location='cpu',weights_only=True)
prov=recovered['provenance']
if prov['model_sha256']!=weight_sha or prov['source_row']!=row or prov.get('input_file_sha256')!=record['file_sha256']:raise ValueError('Orphan prefix provenance differs')
atomic_json(sidecar,dict(path=path.name,sha256=sha(path),provenance=prov))
del recovered
if sidecar.exists():
rec=json.loads(sidecar.read_text())
if rec['provenance']['model_sha256']!=weight_sha or rec['provenance'].get('input_file_sha256')!=record['file_sha256'] or not path.exists() or sha(path)!=rec['sha256']:raise ValueError('Existing prefix differs')
manifest['prefixes'][key]=rec
else:pending.append(row)
if not pending:continue
images=[torch.from_numpy(np.array(fields[k][pending])).cuda() for k in ('image','wrist_image')]
tokens=torch.from_numpy(np.array(fields['prompt'][pending])).cuda();masks=torch.from_numpy(np.array(fields['prompt_mask'][pending])).cuda()
with torch.no_grad():
embeddings,padding,blocks=model.embed_prefix(images,[torch.ones(len(pending),dtype=torch.bool,device='cuda')]*2,tokens,masks)
attention=model._prepare_attention_masks_4d(make_att_2d_masks(padding,blocks));positions=padding.cumsum(1)-1
_,cache=model.paligemma_with_expert.forward(attention_mask=attention,position_ids=positions,past_key_values=None,inputs_embeds=[embeddings,None],use_cache=True)
legacy=cache.to_legacy_cache() if hasattr(cache,'to_legacy_cache') else tuple(cache)
if not compaction_checked:
from transformers.cache_utils import DynamicCache
keep=padding[0].nonzero(as_tuple=True)[0]
full=DynamicCache.from_legacy_cache(tuple((k[:1].clone(),v[:1].clone()) for k,v in legacy))
compact=DynamicCache.from_legacy_cache(tuple((k[:1,:,keep,:].clone(),v[:1,:,keep,:].clone()) for k,v in legacy))
noise=torch.randn((1,10,32),generator=torch.Generator().manual_seed(303)).cuda();t=torch.tensor([.5],device='cuda')
state=torch.from_numpy(np.array(fields['state'][pending[0]:pending[0]+1])).cuda()
with torch.no_grad():
vf=model.denoise_step(state,padding[:1],full,noise,t)
vc=model.denoise_step(state,padding[:1,keep],compact,noise,t)
torch.testing.assert_close(vc,vf,rtol=.02,atol=.01)
atomic_json(out/'compaction-check.json',dict(model_sha256=weight_sha,max_abs_difference=float((vc-vf).abs().max()),full_tokens=int(padding.shape[1]),compact_tokens=int(keep.numel()),rtol=.02,atol=.01),replace=True)
compaction_checked=True
del full,compact,vf,vc,noise,t,state
# Copy each complete layer once, then select each row on CPU.
# File verification and fsync run concurrently on independent files.
cpu_legacy=tuple((k.cpu(),v.cpu()) for k,v in legacy)
writes=[]
for b,row in enumerate(pending):
key=f'e{int(episode):06d}_r{row:06d}';path=out/(key+'.pt')
provenance=dict(input_file_sha256=record['file_sha256'],tokenizer_sha256=manifest['protocol']['tokenizer_sha256'],source_revision=manifest['protocol']['source_revision'],model_sha256=weight_sha,normalization_sha256=normalization_sha,episode=int(episode),source_row=row,
backend='pinned_openpi_bf16_fp32_eager',position_hash=tensor_hash(positions[b:b+1]),token_hash=tensor_hash(tokens[b:b+1]),
payload='cached_path_only_compact_valid_tokens_no_embedding',prefix_embedding_sha256=tensor_hash(embeddings[b:b+1]))
keep=padding[b].nonzero(as_tuple=True)[0]
provenance['valid_prefix_tokens']=int(keep.numel())
z=dict(prefix_embedding=torch.empty(1,0,embeddings.shape[-1],dtype=embeddings.dtype),padding_mask=padding[b:b+1,keep].cpu().clone(),
attention_blocks=blocks[b:b+1,keep].cpu().clone(),state=torch.from_numpy(np.array(fields['state'][row:row+1])),
KV=tuple((k[b:b+1,:,keep.cpu(),:].clone(),v[b:b+1,:,keep.cpu(),:].clone()) for k,v in cpu_legacy),provenance=provenance)
writes.append(writers.submit(commit_prefix,path,z,key))
for future in writes:
key,rec=future.result();manifest['prefixes'][key]=rec;count+=1
del cpu_legacy,writes
torch.cuda.synchronize()
if count%64<args.batch_size:
elapsed=time.monotonic()-begin;done=len(manifest['prefixes'])
status=dict(stage='frozen_prefix_cache',completed_prefixes=done,total_prefixes=expected,seconds=elapsed,seconds_per_prefix=elapsed/count,remaining_seconds_estimate=(expected-done)*elapsed/count,peak_gib=torch.cuda.max_memory_allocated()/2**30,updated_unix_time=time.time())
atomic_json(out/'status.json',status,replace=True);print(json.dumps(status),flush=True)
del embeddings,padding,blocks,attention,cache,legacy,images,tokens,masks
if args.max_prefixes and count>=args.max_prefixes:
atomic_json(out/'asset-index.json',manifest,replace=True)
atomic_json(out/'status.json',dict(stage='smoke_prefix_ready',completed_prefixes=len(manifest['prefixes']),total_prefixes=expected,seconds=time.monotonic()-begin,seconds_per_prefix=(time.monotonic()-begin)/count,peak_gib=torch.cuda.max_memory_allocated()/2**30),replace=True);writers.shutdown();return
atomic_json(out/'asset-index.json',manifest,replace=True)
manifest['protocol']['model_sha256']=weight_sha;atomic_json(out/'asset-index.json',manifest,replace=True)
atomic_json(out/'completed.json',dict(prefixes=len(manifest['prefixes']),seconds=time.monotonic()-begin,model_sha256=weight_sha))
writers.shutdown()
if __name__=='__main__':main()
|