"""Construct only extension parameters; OpenPI model is supplied explicitly.""" from dataclasses import replace import torch from torch import nn from .core import TTTLayer, TTTConfig from .register_layout import RegisterConfig, RegisterTokens from .layer_state import parameter_storage_unique class SlowParameters(nn.Module): def __init__(self,ttt,registers): super().__init__();self.ttt=ttt;self.registers=registers def extension(config): """Shape-compatible CPU container. Exact research initialization must be loaded.""" if config.method=='opentau_reference': from .reference_core_adapter import ReferenceLayer modules=nn.ModuleDict({str(i):ReferenceLayer(config.width) for i in config.active_layers}) else: modules=nn.ModuleDict({str(i):TTTLayer(TTTConfig(dim=config.width, memory_dim=config.memory_dim,hidden_dim=config.hidden_dim,base_lr=config.inner_base_lr)) for i in config.active_layers}) slow=SlowParameters(modules,RegisterTokens(RegisterConfig(config.register_count,config.width,init_std=config.register_init_std,position_rule=config.register_position_rule))) configure_trainables(slow,config) return slow def configure_trainables(slow,config,*,inference=False): if list(slow.ttt)!=[str(i) for i in config.active_layers]:raise ValueError('Layer/order mismatch') slow.requires_grad_(not inference) for layer in slow.ttt.values(): if config.method=='lightweight': layer.config=replace(layer.config,base_lr=config.inner_base_lr) layer.raw_lr.requires_grad_(config.learn_inner_lr and not inference) if not parameter_storage_unique(slow.ttt):raise ValueError('Aliased slow parameters') names=[n for n,p in slow.named_parameters() if p.requires_grad] params=[p for p in slow.parameters() if p.requires_grad] if not inference and (sum(p.numel() for p in params),len(params))!=config.expected_trainables: raise ValueError('Unexpected optimizer parameter count') if len({id(p) for p in params})!=len(params):raise ValueError('Duplicate optimizer mapping') return names,params def create_adapter(model,config,*,observer=None,guard=None,layer_observer=None,inference=False,batched_segments=False): from .multilayer_adapter import MultilayerExpert model.requires_grad_(False).eval() adapter_type=MultilayerExpert if config.method=='opentau_reference': from .reference_adapter import ReferenceExpert adapter_type=ReferenceExpert if batched_segments: from .batched_adapter import BatchedLightExpert,BatchedReferenceExpert adapter_type=BatchedReferenceExpert if config.method=='opentau_reference' else BatchedLightExpert adapter=adapter_type(model,list(config.active_layers), RegisterConfig(config.register_count,config.width,init_std=config.register_init_std,position_rule=config.register_position_rule),observer=observer,guard=guard, layer_observer=layer_observer) adapter.ttt_insertion_point=config.ttt_insertion_point adapter.GPU_full_valid_only=True slow=SlowParameters(adapter.ttt,adapter.registers) configure_trainables(slow,config,inference=inference) adapter.eval() return adapter,slow