Ratstar216's picture
Add files using upload-large-folder tool
0a8d20a verified
Raw History Blame Contribute Delete
7.14 kB
"""Read-only representative endpoint audit; optional bounded HTTP video streaming."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import numpy as np
import torch
from PIL import Image, ImageDraw
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from resolution_adapt.data import DATASET,DATASET_REVISION,POOLS,pool_of,ResolutionBottleneck,resize_rgb
from resolution_adapt.model import LIT_REVISION
from training_support import DatasetGripperToImageWAM
def audit(root: Path, output: Path, *, stream=False, stats_path=None, processed=None):
import av
import pyarrow.parquet as pq
from huggingface_hub import hf_hub_download,hf_hub_url,get_token
output.mkdir(parents=True,exist_ok=True)
def file(name):
return Path(hf_hub_download(DATASET,name,repo_type='dataset',revision=DATASET_REVISION,
local_dir=str(root))) if stream else root/name
provenance=[json.loads(line) for line in file('meta/source_episodes.jsonl').read_text().splitlines()]
info=json.loads(file('meta/info.json').read_text())
ep_rows=pq.read_table(file('meta/episodes/chunk-000/file-000.parquet')).to_pylist()
ep_by_id={e['episode_index']:e for e in ep_rows}
if stats_path is None:
stats_path=hf_hub_download('linjianman/LIT','imagewam/lit_stage2/dataset_stats.json',revision=LIT_REVISION)
stats=json.loads(Path(stats_path).read_text())
def normalize(value,category):
st=stats[category]['default']; low=torch.tensor(st['global_min']); high=torch.tensor(st['global_max'])
span=high-low; constant=span<1e-4; span[constant]=2
scale=2/span; offset=-1-scale*low; offset[constant]=-low[constant]
return (value*scale+offset).clamp(-5,5)
report=[]; selected=[next(e for e in provenance if pool_of(e)==p and e['length']>32) for p in POOLS]
for prov in selected:
pool=pool_of(prov); ep=ep_by_id[prov['episode_index']]
shard=info['data_path'].format(chunk_index=ep['data/chunk_index'],file_index=ep['data/file_index'])
rows=pq.read_table(file(shard),filters=[('episode_index','=',ep['episode_index'])]).to_pylist()
rows.sort(key=lambda r:r['frame_index'])
t=0
states=torch.tensor([r['observation.state'] for r in rows[:17]],dtype=torch.float32)
actions=torch.tensor([r['action'] for r in rows[:16]],dtype=torch.float32)
assert states.shape==(17,8) and actions.shape==(16,7)
assert all(r['episode_index']==prov['episode_index'] for r in rows[:17])
original=actions.clone()
converted=DatasetGripperToImageWAM().forward({'action':{'default':actions}})['action']['default']
assert torch.equal(converted[:,:6],original[:,:6])
assert torch.equal(converted[:,-1],(1-original[:,-1])/2)
cameras=[]; panels=[]
for camera in ('front','wrist'):
key=f'observation.images.{camera}'
video=info['video_path'].format(video_key=key,chunk_index=ep[f'videos/{key}/chunk_index'],
file_index=ep[f'videos/{key}/file_index'])
timestamps=[ep[f'videos/{key}/from_timestamp']+rows[i]['timestamp'] for i in (0,16)]
options={'rw_timeout':'30000000'}
location=str(root/video)
if stream:
location=hf_hub_url(DATASET,video,repo_type='dataset',revision=DATASET_REVISION)
token=get_token()
if token: options['headers']='Authorization: Bearer '+token+'\r\n'
with av.open(location,options=options) as container:
video_stream=container.streams.video[0]
container.seek(int(timestamps[0]/video_stream.time_base),stream=video_stream,backward=True)
decoded={}
for frame in container.decode(video_stream):
timestamp=float(frame.pts*video_stream.time_base)
for i,target in enumerate(timestamps):
if abs(timestamp-target)<1e-4: decoded[i]=frame.to_ndarray(format='rgb24')
if len(decoded)==2: break
if timestamp > timestamps[-1]+.1: raise ValueError('Missing endpoint video frame')
raw=torch.from_numpy(np.stack([decoded[i] for i in (0,1)])).permute(0,3,1,2)
assert raw.shape==(2,3,256,256) and raw.dtype==torch.uint8
source=raw.float()/255; degraded=resize_rgb(source,128); final=ResolutionBottleneck()(source)
assert degraded.shape==(2,3,128,128) and final.shape==(2,3,224,224)
cameras.append(final)
panels.append((camera,source[0],degraded[0],final[0]))
combined=torch.cat(cameras,dim=-1)*2-1
assert combined.shape==(2,3,224,448) and -1<=combined.min()<=combined.max()<=1
normalized_action=normalize(converted,'action'); pose=normalize(states[16],'state')
if processed is not None:
augment=processed.video_augmentation; processed.video_augmentation=None
try: sample=processed[prov['dataset_from_index']+t]
finally: processed.video_augmentation=augment
torch.testing.assert_close(sample['goal_pose'],pose)
torch.testing.assert_close(sample['action'],normalized_action)
torch.testing.assert_close(sample['video'],combined.permute(1,0,2,3),atol=.02,rtol=0)
assert not sample['goal_pose_is_pad'].any()
canvas=Image.new('RGB',(720,560),'white'); draw=ImageDraw.Draw(canvas)
for row,(camera,*images) in enumerate(panels):
x=0
for label,img in zip(('source 256','bottleneck 128','model 224'),images,strict=True):
array=(img.permute(1,2,0).numpy()*255).round().clip(0,255).astype(np.uint8)
canvas.paste(Image.fromarray(array),(x,row*280+22))
draw.text((x,row*280),f'{pool} {camera}: {label}',fill='black'); x+=img.shape[-1]+20
canvas.save(output/f'{pool}.png')
report.append({'pool':pool,'episode':prov['episode_index'],'anchor':t,'pose_frame':t+16,
'raw':[3,256,256],'degraded':[3,128,128],'model':[3,224,224],'combined':[3,224,448],
'raw_rgb_range':[0,255],'float_range':[0,1],'normalized_image_range':[float(combined.min()),float(combined.max())],
'state_shape':[8],'action_shape':list(converted.shape),'pose_shape':list(pose.shape),
'action_normalized_range':[float(normalized_action.min()),float(normalized_action.max())],
'gripper_exactly_once':True,'stored_rotation':False,'processed_loader_compared':processed is not None})
(output/'dataset_preflight.json').write_text(json.dumps(report,indent=2)+'\n')
return report
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--dataset-root',type=Path,required=True)
p.add_argument('--output',type=Path,required=True);p.add_argument('--stream-from-hub',action='store_true')
p.add_argument('--stats',type=Path)
a=p.parse_args(); print(json.dumps(audit(a.dataset_root,a.output,stream=a.stream_from_hub,stats_path=a.stats),indent=2))