"""π₀.₅-specific register layout; no model construction or CUDA import effects.""" from dataclasses import dataclass, asdict import torch from torch import nn @dataclass(frozen=True) class RegisterConfig: count: int width: int horizon: int = 10 init_seed: int = 271016 init_std: float = 0.02 position_rule: str = 'legacy_shift_actions' def __post_init__(self): if self.count not in (0,16) or self.width <= 0 or self.horizon != 10: raise ValueError('Scoped implementation supports M0/M16 and H10') @dataclass(frozen=True) class TokenLayout: registers: tuple[int,int] actions: tuple[int,int] state: None = None def metadata(self): return {'register_span':list(self.registers),'state_span':None, 'action_span':list(self.actions),'suffix_length':self.actions[1], 'prefix_stored_separately':True,'position_rule':'valid prefix count + suffix ordinal', 'attention_group':'one bidirectional suffix block'} class RegisterTokens(nn.Module): def __init__(self, config: RegisterConfig): super().__init__();self.config=config # Local CPU generator, no global RNG consumption. M0 has no new parameter. if config.count: g=torch.Generator(device='cpu').manual_seed(config.init_seed) self.initial=nn.Parameter(torch.randn(config.count,config.width,generator=g,dtype=torch.float32)*config.init_std) else:self.register_parameter('initial',None) def prepend(self,suffix,padding,blocks,*,register_probe=None): c=self.config assert suffix.shape[1:]==(c.horizon,c.width) assert padding.dtype==torch.bool and padding.shape==blocks.shape==suffix.shape[:2] assert bool(padding.all()) and bool((blocks[:,0]==1).all()) and not bool(blocks[:,1:].any()) layout=TokenLayout((0,c.count),(c.count,c.count+c.horizon)) if not c.count: assert register_probe is None return suffix,padding,blocks,layout reg=self.initial.unsqueeze(0).expand(suffix.shape[0],-1,-1) if register_probe is not None: assert register_probe.shape==reg.shape reg=reg+register_probe # The register input is freshly expanded each expert call, never a state. reg=reg.to(suffix.dtype) s=torch.cat([reg,suffix],1) sm=torch.cat([torch.ones((s.shape[0],c.count),dtype=torch.bool,device=s.device),padding],1) sa=torch.zeros_like(sm,dtype=blocks.dtype);sa[:,0]=1 return s,sm,sa,layout def metadata(self):return asdict(self.config) def attention_contract(pm,pa,sm,sa,*,register_count=0,position_rule='legacy_shift_actions'): """Return boolean joint/cached masks and RoPE positions from one contract.""" from openpi.models_pytorch.pi0_pytorch import make_att_2d_masks pad=torch.cat([pm,sm],1);block=torch.cat([pa,sa],1) joint=make_att_2d_masks(pad,block) cached=torch.cat([pm[:,None,:].expand(-1,sm.shape[1],-1),make_att_2d_masks(sm,sa)],2) jp=pad.cumsum(1)-1;cp=pm.sum(-1)[:,None]+sm.cumsum(1)-1 assert torch.equal(joint[:,-sm.shape[1]:],cached) assert torch.equal(jp[:,-sm.shape[1]:],cp) if position_rule=='preserve_action_positions' and register_count: register_pad=sm[:,:register_count];action_pad=sm[:,register_count:] positions=torch.cat([action_pad.sum(1,keepdim=True)+register_pad.cumsum(1)-1,action_pad.cumsum(1)-1],1) cp=pm.sum(-1)[:,None]+positions jp=torch.cat([jp[:,:pm.shape[1]],cp],1) elif position_rule!='legacy_shift_actions':raise ValueError('Unknown register position rule') return joint,jp,cached,cp