Download code/src/robottt_pi05/register_layout.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 3.68 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/register_layout.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/src/robottt_pi05/register_layout.py
-
curl -L -o register_layout.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/register_layout.py
3.68 kB
| """π₀.₅-specific register layout; no model construction or CUDA import effects.""" | |
| from dataclasses import dataclass, asdict | |
| import torch | |
| from torch import nn | |
| class RegisterConfig: | |
| count: int | |
| width: int | |
| horizon: int = 10 | |
| init_seed: int = 271016 | |
| init_std: float = 0.02 | |
| position_rule: str = 'legacy_shift_actions' | |
| def __post_init__(self): | |
| if self.count not in (0,16) or self.width <= 0 or self.horizon != 10: | |
| raise ValueError('Scoped implementation supports M0/M16 and H10') | |
| class TokenLayout: | |
| registers: tuple[int,int] | |
| actions: tuple[int,int] | |
| state: None = None | |
| def metadata(self): | |
| return {'register_span':list(self.registers),'state_span':None, | |
| 'action_span':list(self.actions),'suffix_length':self.actions[1], | |
| 'prefix_stored_separately':True,'position_rule':'valid prefix count + suffix ordinal', | |
| 'attention_group':'one bidirectional suffix block'} | |
| class RegisterTokens(nn.Module): | |
| def __init__(self, config: RegisterConfig): | |
| super().__init__();self.config=config | |
| # Local CPU generator, no global RNG consumption. M0 has no new parameter. | |
| if config.count: | |
| g=torch.Generator(device='cpu').manual_seed(config.init_seed) | |
| self.initial=nn.Parameter(torch.randn(config.count,config.width,generator=g,dtype=torch.float32)*config.init_std) | |
| else:self.register_parameter('initial',None) | |
| def prepend(self,suffix,padding,blocks,*,register_probe=None): | |
| c=self.config | |
| assert suffix.shape[1:]==(c.horizon,c.width) | |
| assert padding.dtype==torch.bool and padding.shape==blocks.shape==suffix.shape[:2] | |
| assert bool(padding.all()) and bool((blocks[:,0]==1).all()) and not bool(blocks[:,1:].any()) | |
| layout=TokenLayout((0,c.count),(c.count,c.count+c.horizon)) | |
| if not c.count: | |
| assert register_probe is None | |
| return suffix,padding,blocks,layout | |
| reg=self.initial.unsqueeze(0).expand(suffix.shape[0],-1,-1) | |
| if register_probe is not None: | |
| assert register_probe.shape==reg.shape | |
| reg=reg+register_probe | |
| # The register input is freshly expanded each expert call, never a state. | |
| reg=reg.to(suffix.dtype) | |
| s=torch.cat([reg,suffix],1) | |
| sm=torch.cat([torch.ones((s.shape[0],c.count),dtype=torch.bool,device=s.device),padding],1) | |
| sa=torch.zeros_like(sm,dtype=blocks.dtype);sa[:,0]=1 | |
| return s,sm,sa,layout | |
| def metadata(self):return asdict(self.config) | |
| def attention_contract(pm,pa,sm,sa,*,register_count=0,position_rule='legacy_shift_actions'): | |
| """Return boolean joint/cached masks and RoPE positions from one contract.""" | |
| from openpi.models_pytorch.pi0_pytorch import make_att_2d_masks | |
| pad=torch.cat([pm,sm],1);block=torch.cat([pa,sa],1) | |
| joint=make_att_2d_masks(pad,block) | |
| cached=torch.cat([pm[:,None,:].expand(-1,sm.shape[1],-1),make_att_2d_masks(sm,sa)],2) | |
| jp=pad.cumsum(1)-1;cp=pm.sum(-1)[:,None]+sm.cumsum(1)-1 | |
| assert torch.equal(joint[:,-sm.shape[1]:],cached) | |
| assert torch.equal(jp[:,-sm.shape[1]:],cp) | |
| if position_rule=='preserve_action_positions' and register_count: | |
| register_pad=sm[:,:register_count];action_pad=sm[:,register_count:] | |
| positions=torch.cat([action_pad.sum(1,keepdim=True)+register_pad.cumsum(1)-1,action_pad.cumsum(1)-1],1) | |
| cp=pm.sum(-1)[:,None]+positions | |
| jp=torch.cat([jp[:,:pm.shape[1]],cp],1) | |
| elif position_rule!='legacy_shift_actions':raise ValueError('Unknown register position rule') | |
| return joint,jp,cached,cp | |