jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
3.25 kB
"""Construct only extension parameters; OpenPI model is supplied explicitly."""
from dataclasses import replace
import torch
from torch import nn
from .core import TTTLayer, TTTConfig
from .register_layout import RegisterConfig, RegisterTokens
from .layer_state import parameter_storage_unique
class SlowParameters(nn.Module):
def __init__(self,ttt,registers):
super().__init__();self.ttt=ttt;self.registers=registers
def extension(config):
"""Shape-compatible CPU container. Exact research initialization must be loaded."""
if config.method=='opentau_reference':
from .reference_core_adapter import ReferenceLayer
modules=nn.ModuleDict({str(i):ReferenceLayer(config.width) for i in config.active_layers})
else:
modules=nn.ModuleDict({str(i):TTTLayer(TTTConfig(dim=config.width,
memory_dim=config.memory_dim,hidden_dim=config.hidden_dim,base_lr=config.inner_base_lr))
for i in config.active_layers})
slow=SlowParameters(modules,RegisterTokens(RegisterConfig(config.register_count,config.width,init_std=config.register_init_std,position_rule=config.register_position_rule)))
configure_trainables(slow,config)
return slow
def configure_trainables(slow,config,*,inference=False):
if list(slow.ttt)!=[str(i) for i in config.active_layers]:raise ValueError('Layer/order mismatch')
slow.requires_grad_(not inference)
for layer in slow.ttt.values():
if config.method=='lightweight':
layer.config=replace(layer.config,base_lr=config.inner_base_lr)
layer.raw_lr.requires_grad_(config.learn_inner_lr and not inference)
if not parameter_storage_unique(slow.ttt):raise ValueError('Aliased slow parameters')
names=[n for n,p in slow.named_parameters() if p.requires_grad]
params=[p for p in slow.parameters() if p.requires_grad]
if not inference and (sum(p.numel() for p in params),len(params))!=config.expected_trainables:
raise ValueError('Unexpected optimizer parameter count')
if len({id(p) for p in params})!=len(params):raise ValueError('Duplicate optimizer mapping')
return names,params
def create_adapter(model,config,*,observer=None,guard=None,layer_observer=None,inference=False,batched_segments=False):
from .multilayer_adapter import MultilayerExpert
model.requires_grad_(False).eval()
adapter_type=MultilayerExpert
if config.method=='opentau_reference':
from .reference_adapter import ReferenceExpert
adapter_type=ReferenceExpert
if batched_segments:
from .batched_adapter import BatchedLightExpert,BatchedReferenceExpert
adapter_type=BatchedReferenceExpert if config.method=='opentau_reference' else BatchedLightExpert
adapter=adapter_type(model,list(config.active_layers),
RegisterConfig(config.register_count,config.width,init_std=config.register_init_std,position_rule=config.register_position_rule),observer=observer,guard=guard,
layer_observer=layer_observer)
adapter.ttt_insertion_point=config.ttt_insertion_point
adapter.GPU_full_valid_only=True
slow=SlowParameters(adapter.ttt,adapter.registers)
configure_trainables(slow,config,inference=inference)
adapter.eval()
return adapter,slow