code-HFD / src /psi /models /relative_instruction_memory.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
3.04 kB
"""Relative-turn queries, without a fixed latest/previous/first slot table."""
import torch
from torch import nn
class RelativeLagEmbedding(nn.Module):
def __init__(self, width: int):
super().__init__()
self.net = nn.Sequential(nn.Linear(2, 128), nn.SiLU(), nn.Linear(128, width))
nn.init.zeros_(self.net[-1].weight)
nn.init.zeros_(self.net[-1].bias)
def forward(self, lag: torch.Tensor) -> torch.Tensor:
if bool((lag < 1).any()):
raise ValueError("Relative instruction lag starts at one")
value = lag.to(dtype=self.net[0].weight.dtype)
return self.net(torch.stack((value.log1p(), value.reciprocal()), -1))
def relative_instruction_query(lag: int) -> str:
if lag < 1:
raise ValueError("Relative instruction lag starts at one")
return (
"\nTranscribe only the spoken robot action instruction that is number "
f"{lag} counting backwards from the most recent completed instruction, "
"exactly. Number 1 means the most recent completed instruction. "
"Return only its words.\n<|im_end|>\n<|im_start|>assistant\n"
)
def architecture_contract(model_cfg) -> dict:
"""Empty for legacy configs: old contracts remain byte-for-byte compatible."""
values = {
"control_head_type": getattr(model_cfg, "control_head_type", "mlp"),
"instruction_memory_mode": getattr(model_cfg, "instruction_memory_mode", "legacy"),
"qwen_memory_window_seconds": float(getattr(model_cfg, "qwen_memory_window_seconds", 0)),
"qwen_trainable_master_fp32": bool(getattr(model_cfg, "qwen_trainable_master_fp32", False)),
}
legacy = values == dict(control_head_type="mlp", instruction_memory_mode="legacy",
qwen_memory_window_seconds=0., qwen_trainable_master_fp32=False)
contract = {} if legacy else {"staged_architecture_v1": values}
if getattr(model_cfg, "action_representation", "textop39") == "root40_v1":
contract["motion_representation_v1"] = {
"id": "root40_v1", "anchor": "observed_state_heading",
"rtc": "decode_then_reencode_in_next_observed_heading",
"rotation": "absolute_anchor_rotation6d", "height_index": 37,
"normalization": "root40_bounds_v1", "contact": "legacy_joint_delta_v1"}
if getattr(model_cfg, "action_mask_aware_positions", False):
contract["action_observation_positions_v1"] = "valid_tokens_only"
if getattr(model_cfg, "action_input_projection_fp32", False):
contract["action_input_projection_precision_v1"] = "fp32_input_mlp"
if getattr(model_cfg, "swa_visual_qa_enabled", False) and getattr(model_cfg, "swa_qa_mode", "shared_legacy") == "independent_online_lora":
contract["swa_qa_online_v1"] = dict(
mode="independent_online_lora", initialization="original_pretrained_qwen_zero_lora",
layers=8, rank=16, alpha=16,
private_online_kv=True, body_receives_qa_gradient=False,
)
return contract