jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
3.68 kB
"""π₀.₅-specific register layout; no model construction or CUDA import effects."""
from dataclasses import dataclass, asdict
import torch
from torch import nn
@dataclass(frozen=True)
class RegisterConfig:
count: int
width: int
horizon: int = 10
init_seed: int = 271016
init_std: float = 0.02
position_rule: str = 'legacy_shift_actions'
def __post_init__(self):
if self.count not in (0,16) or self.width <= 0 or self.horizon != 10:
raise ValueError('Scoped implementation supports M0/M16 and H10')
@dataclass(frozen=True)
class TokenLayout:
registers: tuple[int,int]
actions: tuple[int,int]
state: None = None
def metadata(self):
return {'register_span':list(self.registers),'state_span':None,
'action_span':list(self.actions),'suffix_length':self.actions[1],
'prefix_stored_separately':True,'position_rule':'valid prefix count + suffix ordinal',
'attention_group':'one bidirectional suffix block'}
class RegisterTokens(nn.Module):
def __init__(self, config: RegisterConfig):
super().__init__();self.config=config
# Local CPU generator, no global RNG consumption. M0 has no new parameter.
if config.count:
g=torch.Generator(device='cpu').manual_seed(config.init_seed)
self.initial=nn.Parameter(torch.randn(config.count,config.width,generator=g,dtype=torch.float32)*config.init_std)
else:self.register_parameter('initial',None)
def prepend(self,suffix,padding,blocks,*,register_probe=None):
c=self.config
assert suffix.shape[1:]==(c.horizon,c.width)
assert padding.dtype==torch.bool and padding.shape==blocks.shape==suffix.shape[:2]
assert bool(padding.all()) and bool((blocks[:,0]==1).all()) and not bool(blocks[:,1:].any())
layout=TokenLayout((0,c.count),(c.count,c.count+c.horizon))
if not c.count:
assert register_probe is None
return suffix,padding,blocks,layout
reg=self.initial.unsqueeze(0).expand(suffix.shape[0],-1,-1)
if register_probe is not None:
assert register_probe.shape==reg.shape
reg=reg+register_probe
# The register input is freshly expanded each expert call, never a state.
reg=reg.to(suffix.dtype)
s=torch.cat([reg,suffix],1)
sm=torch.cat([torch.ones((s.shape[0],c.count),dtype=torch.bool,device=s.device),padding],1)
sa=torch.zeros_like(sm,dtype=blocks.dtype);sa[:,0]=1
return s,sm,sa,layout
def metadata(self):return asdict(self.config)
def attention_contract(pm,pa,sm,sa,*,register_count=0,position_rule='legacy_shift_actions'):
"""Return boolean joint/cached masks and RoPE positions from one contract."""
from openpi.models_pytorch.pi0_pytorch import make_att_2d_masks
pad=torch.cat([pm,sm],1);block=torch.cat([pa,sa],1)
joint=make_att_2d_masks(pad,block)
cached=torch.cat([pm[:,None,:].expand(-1,sm.shape[1],-1),make_att_2d_masks(sm,sa)],2)
jp=pad.cumsum(1)-1;cp=pm.sum(-1)[:,None]+sm.cumsum(1)-1
assert torch.equal(joint[:,-sm.shape[1]:],cached)
assert torch.equal(jp[:,-sm.shape[1]:],cp)
if position_rule=='preserve_action_positions' and register_count:
register_pad=sm[:,:register_count];action_pad=sm[:,register_count:]
positions=torch.cat([action_pad.sum(1,keepdim=True)+register_pad.cumsum(1)-1,action_pad.cumsum(1)-1],1)
cp=pm.sum(-1)[:,None]+positions
jp=torch.cat([jp[:,:pm.shape[1]],cp],1)
elif position_rule!='legacy_shift_actions':raise ValueError('Unknown register position rule')
return joint,jp,cached,cp