Download task 57/policy_code/src/b1k/models/pi_behavior.py from Luke-Xing/task57b1k: direct link, hf CLI and curl.
- Browser
- Download file 53.4 kB
-
https://huggingface.co/Luke-Xing/task57b1k/resolve/main/task%2057/policy_code/src/b1k/models/pi_behavior.py
- Command line
-
hf download 'hf://Luke-Xing/task57b1k/task 57/policy_code/src/b1k/models/pi_behavior.py'
-
curl -L -o pi_behavior.py https://huggingface.co/Luke-Xing/task57b1k/resolve/main/task%2057/policy_code/src/b1k/models/pi_behavior.py
53.4 kB
"""The main model for BEHAVIOR-1K challenge.
Based on Pi0.5 implementation from PhysicalIntelligence/openpi
"""
import logging
import pathlib
import einops
import flax.nnx as nnx
import flax.nnx.bridge as nnx_bridge
import jax
import jax.numpy as jnp
from typing_extensions import override
from openpi.models import model as _model
from openpi.models import gemma as _gemma
from openpi.models import siglip as _siglip
from openpi.models.pi0 import make_attn_mask, posemb_sincos
from openpi.shared import array_typing as at
# Import from our custom modules
from b1k.models import pi_behavior_config
from b1k.models.observation import Observation, preprocess_observation
from b1k.models.pi_behavior_config import (
TASK_NUM_STAGES,
MAX_NUM_STAGES,
TOTAL_TASK_STAGE_EMBEDDINGS,
TASK_STAGE_OFFSETS
)
logger = logging.getLogger("b1k")
class KVCacheTransform(nnx.Module):
"""Transforms prefix KV cache by mixing across layers.
Each destination layer's K and V become learnable linear combinations
of all source layers' K and V, plus a bias term. This allows the action
expert to attend to learned combinations of VLM layers rather than being
forced to attend layer-by-layer.
Initialized as identity transform (k_coeffs = I, bias = 0) so the model
starts with the same behavior as without transformation.
"""
def __init__(self, num_layers: int, head_dim: int, num_kv_heads: int, rngs: nnx.Rngs):
# K transformation: [dest_layer, src_layer]
# Initialize as identity so transformation is initially a no-op
self.k_coeffs = nnx.Param(jnp.eye(num_layers, dtype=jnp.float32))
# K bias: [layer, num_kv_heads, head_dim]
# Initialize as zeros
self.k_bias = nnx.Param(jnp.zeros((num_layers, num_kv_heads, head_dim), dtype=jnp.float32))
# V transformation (independent from K)
self.v_coeffs = nnx.Param(jnp.eye(num_layers, dtype=jnp.float32))
self.v_bias = nnx.Param(jnp.zeros((num_layers, num_kv_heads, head_dim), dtype=jnp.float32))
def __call__(self, kv_cache: tuple[jnp.ndarray, jnp.ndarray]) -> tuple[jnp.ndarray, jnp.ndarray]:
"""Transform KV cache by mixing across layers.
Args:
kv_cache: Tuple of (cache_k, cache_v) where each has shape
[num_layers, batch, seq_len, num_kv_heads, head_dim]
Returns:
Transformed (k_new, v_new) with same shape and dtype as input
"""
cache_k, cache_v = kv_cache
# Shape: [layers, batch, seq_len, num_kv_heads, head_dim]
# Preserve original dtype (important for bfloat16 training)
original_dtype = cache_k.dtype
# Transform K: each destination layer is a weighted combination of all source layers
# k_new[dest] = sum_src(k_coeffs[dest, src] * cache_k[src]) + k_bias[dest]
# Einsum: [dest, src] @ [src, batch, seq, heads, dim] -> [dest, batch, seq, heads, dim]
k_new = jnp.einsum('ds,sbtkh->dbtkh', self.k_coeffs.value, cache_k)
k_new = k_new + self.k_bias.value[:, None, None, :, :] # Add bias
# Transform V (same operation, independent parameters)
v_new = jnp.einsum('ds,sbtkh->dbtkh', self.v_coeffs.value, cache_v)
v_new = v_new + self.v_bias.value[:, None, None, :, :]
# Cast back to original dtype
k_new = k_new.astype(original_dtype)
v_new = v_new.astype(original_dtype)
return (k_new, v_new)
class PiBehavior(_model.BaseModel):
def __init__(self, config: pi_behavior_config.PiBehaviorConfig, rngs: nnx.Rngs):
super().__init__(config.action_dim, config.action_horizon, config.max_token_len)
# Store config for later use
self.config = config
paligemma_config = _gemma.get_config(config.paligemma_variant)
action_expert_config = _gemma.get_config(config.action_expert_variant)
# Initialize Gemma models with AdaRMS (Pi05 style)
llm = nnx_bridge.ToNNX(
_gemma.Module(
configs=[paligemma_config, action_expert_config],
embed_dtype=config.dtype,
adarms=True,
)
)
llm.lazy_init(rngs=rngs, method="init", use_adarms=[False, True])
# Initialize vision model
img = nnx_bridge.ToNNX(
_siglip.Module(
num_classes=paligemma_config.width,
variant="So400m/14",
pool_type="none",
scan=True,
dtype_mm=config.dtype,
)
)
img.lazy_init(next(iter(config.fake_obs().images.values())), train=False, rngs=rngs)
self.PaliGemma = nnx.Dict(llm=llm, img=img)
# KV cache transformation for cross-layer attention
# Allows action expert to attend to learned combinations of VLM layers
if config.use_kv_transform:
self.kv_transform = KVCacheTransform(
num_layers=paligemma_config.depth,
head_dim=paligemma_config.head_dim,
num_kv_heads=paligemma_config.num_kv_heads,
rngs=rngs
)
else:
self.kv_transform = None
# Task embeddings table - trainable embeddings for each task
self.task_embeddings = nnx.Embed(
num_embeddings=config.num_tasks,
features=config.task_embedding_dim,
rngs=rngs,
)
# Stage predictor - predicts stage from VLM output of base task token
# Outputs MAX_NUM_STAGES logits, but invalid stages are masked per task
self.stage_pred_from_vlm = nnx.Linear(paligemma_config.width, MAX_NUM_STAGES, rngs=rngs)
# Task + subtask fusion layers
# Combines task embedding + cos/sin encoded subtask state
self.subtask_encoding_dim = config.task_embedding_dim // 2 # Half of task embedding dim (1024)
# Task-specific stage embeddings (one per stage per task)
# Total embeddings = sum of stages across all tasks (596 for 5-15 stages per task)
self.task_stage_embeddings = nnx.Embed(
num_embeddings=TOTAL_TASK_STAGE_EMBEDDINGS,
features=self.subtask_encoding_dim,
rngs=rngs,
)
# Gated fusion layers
# Input: task_embedding + sincos + task_stage_emb = task_dim + 2*subtask_dim
fusion_input_dim = config.task_embedding_dim + 2 * self.subtask_encoding_dim
# Gate networks to learn how to combine different signals
self.gate_sincos = nnx.Linear(fusion_input_dim, self.subtask_encoding_dim, rngs=rngs)
self.gate_task_stage = nnx.Linear(fusion_input_dim, self.subtask_encoding_dim, rngs=rngs)
self.gate_task = nnx.Linear(fusion_input_dim, config.task_embedding_dim, rngs=rngs)
# Fusion networks to create multiple conditioned vectors
self.fusion_layer1 = nnx.Linear(fusion_input_dim, config.task_embedding_dim * 2, rngs=rngs)
self.fusion_layer2 = nnx.Linear(config.task_embedding_dim * 2, config.task_embedding_dim, rngs=rngs)
# Additional projection for stage-dominant representation (2 signals now)
self.stage_projection = nnx.Linear(2 * self.subtask_encoding_dim, config.task_embedding_dim, rngs=rngs)
# Pi05 style layers
self.action_in_proj = nnx.Linear(config.action_dim, action_expert_config.width, rngs=rngs)
self.time_mlp_in = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs)
self.time_mlp_out = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs)
self.action_out_proj = nnx.Linear(action_expert_config.width, config.action_dim, rngs=rngs)
# Correlated noise generation
# Initialize as NNX Intermediate (excluded from checkpoints, loaded from norm_stats)
# Full correlation matrix with beta shrinkage for robustness
flat_dim = config.action_horizon * config.action_dim
self.action_correlation_cholesky = nnx.Intermediate(
jnp.eye(flat_dim), # Identity matrix as placeholder
)
self.correlation_loaded = False # Track if correlation matrix has been loaded
self.use_correlated_noise = config.use_correlated_noise
self.correlation_beta = config.correlation_beta # Shrinkage parameter for regularization
# Inpainting cache: stores precomputed matrices for simple correlation-based inpainting
# Key: num_inpainted_steps (length of inpainted sequence)
# Value: dict with {O_indices, U_indices, Sigma_UO_SOOinv}
self.inpainting_cache = {}
# FAST auxiliary training components
if config.use_fast_auxiliary:
# FAST embedding layer (vocab_size → paligemma_width)
# Use paligemma width (2048) to match other prefix tokens
self.fast_token_embedding = nnx.Embed(
num_embeddings=config.fast_vocab_size,
features=paligemma_config.width,
rngs=rngs
)
# FAST projection head (paligemma_width → vocab_size)
self.fast_token_proj = nnx.Linear(
paligemma_config.width,
config.fast_vocab_size,
rngs=rngs
)
logger.info(f"FAST auxiliary enabled, vocab_size={config.fast_vocab_size}")
# This attribute gets automatically set by model.train() and model.eval().
self.deterministic = True
def encode_subtask_state(
self,
subtask_state: at.Int[at.Array, " b"],
task_ids: at.Int[at.Array, " b"]
) -> at.Float[at.Array, "b {self.subtask_encoding_dim}"]:
"""Encode subtask state using cos/sin positional encoding, scaled per task.
Args:
subtask_state: Current stage for each sample [B]
task_ids: Task ID for each sample [B]
Returns:
Positional encodings scaled to [0, 1] range based on task-specific stage count [B, 1024]
"""
# Get number of stages for each task in batch using JAX array indexing
# Convert tuple to JAX array inside function to avoid import-time device allocation
task_num_stages_array = jnp.array(TASK_NUM_STAGES, dtype=jnp.int32)
task_num_stages = task_num_stages_array[task_ids] # [B] - JAX array indexing
# Normalize: stage 0 → 0.0, last stage → 1.0 (per-task scaling)
# Add maximum to avoid division by zero for edge cases
normalized_state = subtask_state.astype(jnp.float32) / jnp.maximum(task_num_stages.astype(jnp.float32) - 1.0, 1.0)
# Use cos/sin encoding similar to timestep encoding
return posemb_sincos(
normalized_state,
self.subtask_encoding_dim,
min_period=1e-3,
max_period=1.0
)
def load_correlation_matrix(self, norm_stats: dict):
"""Load full correlation matrix from normalization statistics and apply shrinkage.
This should be called after model initialization when norm_stats are available.
Applies shrinkage regularization: S_reg = beta * S + (1-beta) * I for robustness.
Args:
norm_stats: Dictionary containing normalization statistics (from normalize.load()),
with 'actions' key containing NormStats with action_correlation_cholesky field.
Raises:
ValueError: If use_correlated_noise=True but correlation matrix is missing.
TypeError: If norm_stats structure is incorrect.
"""
if not self.use_correlated_noise:
logger.info("Correlated noise disabled in config, skipping correlation matrix loading")
return
# Validate norm_stats is a dict
if not isinstance(norm_stats, dict):
raise TypeError(
f"norm_stats must be a dict, got {type(norm_stats).__name__}. "
"Ensure norm_stats are loaded using openpi.shared.normalize.load()."
)
# Check 'actions' key exists
if 'actions' not in norm_stats:
raise ValueError(
"use_correlated_noise=True but 'actions' key not found in norm_stats. "
f"Found keys: {list(norm_stats.keys())}. "
"Run compute_norm_stats.py with --correlation flag to generate correlation matrix."
)
actions_stats = norm_stats['actions']
# Extract correlation matrix (support both dict and attribute access for flexibility)
if isinstance(actions_stats, dict):
chol_matrix = actions_stats.get('action_correlation_cholesky')
access_method = "dict"
elif hasattr(actions_stats, 'action_correlation_cholesky'):
chol_matrix = actions_stats.action_correlation_cholesky
access_method = "attribute"
else:
raise TypeError(
f"norm_stats['actions'] has unexpected type {type(actions_stats).__name__} "
f"and cannot access 'action_correlation_cholesky'. "
"Ensure norm_stats are loaded using openpi.shared.normalize.load()."
)
# Strict validation: correlation matrix must exist and be non-None
if chol_matrix is None:
raise ValueError(
"use_correlated_noise=True but 'action_correlation_cholesky' is None in norm_stats['actions']. "
"This means the correlation matrix was not computed during norm_stats generation. "
"Run compute_norm_stats.py with --correlation flag to generate correlation matrix."
)
logger.info(f"Successfully accessed correlation matrix via {access_method} access")
# Validate correlation matrix shape
expected_dim = self.action_horizon * self.action_dim
try:
L = jnp.array(chol_matrix)
except Exception as e:
raise ValueError(
f"Failed to convert action_correlation_cholesky to array: {e}. "
"The correlation matrix may be corrupted or in an invalid format."
)
if L.ndim != 2 or L.shape[0] != L.shape[1]:
raise ValueError(
f"action_correlation_cholesky must be a square 2D matrix, got shape {L.shape}. "
f"Expected shape: ({expected_dim}, {expected_dim})"
)
if L.shape[0] != expected_dim:
raise ValueError(
f"action_correlation_cholesky has wrong dimensions: {L.shape[0]}x{L.shape[0]}. "
f"Expected {expected_dim}x{expected_dim} (action_horizon={self.action_horizon} * action_dim={self.action_dim}). "
"This indicates the correlation matrix was computed for a different action space configuration."
)
# Reconstruct covariance matrix from Cholesky
Sigma = L @ L.T
# Apply shrinkage regularization: Σ_reg = beta * Σ + (1-beta) * I
beta = self.correlation_beta
logger.info(f"Applying shrinkage regularization with beta={beta:.2f}")
Sigma_reg = beta * Sigma + (1 - beta) * jnp.eye(Sigma.shape[0])
# Compute Cholesky decomposition of regularized covariance
try:
L_reg = jnp.linalg.cholesky(Sigma_reg)
except Exception as e:
raise RuntimeError(
f"Cholesky decomposition failed on regularized covariance: {e}. "
"This indicates the regularized correlation matrix is not positive definite. "
f"Current beta={beta:.2f}. Try decreasing correlation_beta closer to 0.0 for more shrinkage/regularization."
)
# Update the Intermediate value
self.action_correlation_cholesky.value = L_reg
self.correlation_loaded = True
logger.info(
f"✓ Loaded correlation matrix with shape {L_reg.shape} "
f"(beta={beta:.2f} shrinkage applied)"
)
logger.info(
f" Memory usage: {L_reg.nbytes / 1024 / 1024:.2f} MB"
)
def generate_correlated_noise(
self,
rng: at.KeyArrayLike,
batch_size: int,
) -> at.Float[at.Array, "b {self.action_horizon} {self.action_dim}"]:
"""Generate correlated noise matching action covariance structure.
Uses full correlation matrix with optional beta shrinkage for robustness.
Args:
rng: Random key for noise generation
batch_size: Number of noise samples to generate
Returns:
Correlated noise with shape [batch_size, action_horizon, action_dim]
Raises:
RuntimeError: If use_correlated_noise=True but correlation matrix not loaded.
"""
if not self.use_correlated_noise:
# Independent Gaussian noise when correlated noise is disabled
return jax.random.normal(rng, (batch_size, self.action_horizon, self.action_dim))
if not self.correlation_loaded:
raise RuntimeError(
"use_correlated_noise=True but correlation matrix is not loaded. "
"Ensure load_correlation_matrix() was called during model initialization. "
"Run compute_norm_stats.py with --correlation flag to generate correlation matrix."
)
# Generate standard correlated noise using Cholesky decomposition
flat_dim = self.action_horizon * self.action_dim
standard_normal = jax.random.normal(rng, (batch_size, flat_dim))
correlated_flat = standard_normal @ self.action_correlation_cholesky.value.T
correlated_noise = correlated_flat.reshape(batch_size, self.action_horizon, self.action_dim)
return correlated_noise
def _precompute_correction_matrix(
self,
O_indices: at.Int[at.Array, " nO"],
U_indices: at.Int[at.Array, " nU"],
) -> dict:
"""Precompute matrix for correlation-aware inpainting correction.
Computes Σ_{UO}Σ_{OO}^{-1} which propagates corrections from O to U
while preserving correlation structure.
Args:
O_indices: Flat indices of inpainted dimensions [|O|]
U_indices: Flat indices of free dimensions [|U|]
Returns:
Dictionary with {O_indices, U_indices, correction_matrix}
Raises:
RuntimeError: If correlation matrix is not loaded
"""
if not self.correlation_loaded:
raise RuntimeError(
"Cannot precompute correction matrix: correlation matrix not loaded. "
"Call load_correlation_matrix() first."
)
L = self.action_correlation_cholesky.value
Sigma = L @ L.T # Full covariance matrix [hd, hd]
# Extract submatrices
Sigma_OO = Sigma[jnp.ix_(O_indices, O_indices)] # [|O|, |O|]
Sigma_UO = Sigma[jnp.ix_(U_indices, O_indices)] # [|U|, |O|]
# Compute correction matrix: Σ_{UO} @ Σ_{OO}^{-1}
# This propagates corrections from O to U
eps_OO = 1e-6 * jnp.maximum(jnp.mean(jnp.diag(Sigma_OO)), 1.0)
Sigma_OO_reg = Sigma_OO + eps_OO * jnp.eye(Sigma_OO.shape[0])
# Solve Σ_{OO}_reg @ X = Σ_{UO}.T for X, then transpose
correction_matrix = jax.scipy.linalg.solve(
Sigma_OO_reg, Sigma_UO.T, assume_a='pos'
).T # [|U|, |O|]
return {
'O_indices': O_indices,
'U_indices': U_indices,
'correction_matrix': correction_matrix, # Σ_{UO}Σ_{OO}^{-1}
}
def fuse_task_and_subtask(
self, task_embedding: at.Float[at.Array, "b d"], task_ids: at.Int[at.Array, " b"], subtask_state: at.Int[at.Array, " b"]
) -> at.Float[at.Array, "b n d"]:
"""Fuse task embedding with subtask state encoding using multiple representations.
Returns multiple vectors that are differently conditioned by the subtask state:
1. Task-gated representation (task embedding modulated by subtask)
2. Balanced fusion (task + subtask combined)
3. Stage-dominant representation (subtask features projected to task space)
4. Pure stage representation (concatenated learned embeddings)
All output representations have dimension 2048 (task_embedding_dim).
Args:
task_embedding: Base task embedding [b, 2048]
task_ids: Task IDs for task-specific stage embeddings [b]
subtask_state: Subtask state indices [b]
Returns:
Multiple fused embeddings [b, 4, 2048]
"""
# Get subtask representations
sincos_encoding = self.encode_subtask_state(subtask_state, task_ids) # [b, 1024]
# Task-specific stage embedding with corrected indexing
# Use vectorized lookup: offset + stage for each task
# Convert tuple to JAX array inside function to avoid import-time device allocation
task_stage_offsets_array = jnp.array(TASK_STAGE_OFFSETS, dtype=jnp.int32)
task_stage_offsets = task_stage_offsets_array[task_ids] # [b] - JAX array indexing
task_stage_idx = task_stage_offsets + subtask_state # [b]
task_stage_embedding = self.task_stage_embeddings(task_stage_idx) # [b, 1024]
# Concatenate inputs for gating: task (2048) + sincos (1024) + task_stage (1024) = 4096
all_inputs = jnp.concatenate([
task_embedding, # [b, 2048]
sincos_encoding, # [b, 1024]
task_stage_embedding # [b, 1024]
], axis=-1) # [b, 4096]
# Learn gates for each component (sigmoid to get 0-1 scaling)
gate_sincos = nnx.sigmoid(self.gate_sincos(all_inputs)) # [b, 1024]
gate_task_stage = nnx.sigmoid(self.gate_task_stage(all_inputs)) # [b, 1024]
gate_task = nnx.sigmoid(self.gate_task(all_inputs)) # [b, 2048]
# 1. Task-gated representation: task embedding modulated by subtask info [b, 2048]
task_gated = task_embedding * gate_task
# 2. Balanced fusion: combine all signals through fusion network [b, 2048]
x = self.fusion_layer1(all_inputs) # [b, 4096]
x = nnx.relu(x)
balanced_fusion = self.fusion_layer2(x) # [b, 2048]
# 3. Stage-dominant: weighted combination of stage signals, then project [b, 2048]
gated_stage_features = jnp.concatenate([
sincos_encoding * gate_sincos, # [b, 1024]
task_stage_embedding * gate_task_stage # [b, 1024]
], axis=-1) # [b, 2048]
stage_dominant = self.stage_projection(gated_stage_features) # [b, 2048]
# 4. Pure stage: concatenate the embeddings (already 2048) [b, 2048]
pure_stage = jnp.concatenate([sincos_encoding, task_stage_embedding], axis=-1)
# Stack all four representations [b, 4, 2048]
fused_embeddings = jnp.stack([task_gated, balanced_fusion, stage_dominant, pure_stage], axis=1)
return fused_embeddings
@at.typecheck
def embed_prefix(
self,
obs: Observation
) -> tuple[
at.Float[at.Array, "b s emb"],
at.Bool[at.Array, "b s"],
at.Bool[at.Array, " s"]
]:
"""
Embed prefix: images + task + state + FAST_tokens (if provided).
Args:
obs: Observation (may include fast_tokens and fast_token_mask)
Returns:
tokens, input_mask, ar_mask
"""
input_mask = []
ar_mask = []
tokens = []
# Embed images
image_token_list = []
# Respect freeze_vision_backbone config: if frozen, always use train=False
# If not frozen, use the model's training state (self.deterministic)
vision_train_mode = (not self.deterministic) and (not self.config.freeze_vision_backbone)
for name in obs.images:
image_tokens, _ = self.PaliGemma.img(obs.images[name], train=vision_train_mode)
image_token_list.append(image_tokens) # Store for subtask prediction
tokens.append(image_tokens)
input_mask.append(
einops.repeat(
obs.image_masks[name],
"b -> b s",
s=image_tokens.shape[1],
)
)
# Image tokens attend to each other
ar_mask += [False] * image_tokens.shape[1]
# Add task embeddings with subtask state fusion
if obs.tokenized_prompt is not None:
# obs.tokenized_prompt now contains task_ids (shape: [batch_size, 2])
task_ids = obs.tokenized_prompt[:, 0] # Extract task_id: [batch_size]
base_task_embedding = self.task_embeddings(task_ids) # shape: [batch_size, embed_dim]
# ALWAYS use the input subtask state - never use predicted state inside model
if obs.tokenized_prompt.shape[1] > 1: # If we have [task_id, subtask_state]
subtask_state = obs.tokenized_prompt[:, 1] # Use input subtask state
else:
raise ValueError("subtask_state must be provided in tokenized_prompt for PI_BEHAVIOR model")
# Fuse task embedding with subtask state - returns [b, 4, d] with multiple representations
fused_task_embeddings = self.fuse_task_and_subtask(base_task_embedding, task_ids, subtask_state)
# Create task token sequence: [base_task, task_gated, balanced_fusion, stage_dominant, pure_stage]
task_sequence = jnp.concatenate([
base_task_embedding[:, None, :], # [b, 1, d] - base task token
fused_task_embeddings # [b, 4, d] - stage-conditioned tokens
], axis=1) # [b, 5, d]
tokens.append(task_sequence)
# All task tokens are valid
task_mask = jnp.ones((obs.tokenized_prompt.shape[0], 5), dtype=jnp.bool_)
input_mask.append(task_mask)
# Hierarchical attention: base task (False) then stage tokens (True, False, False, False)
# Base task attends to images bidirectionally
# Stage tokens attend to images+task but not vice versa
ar_mask += [False] + [True, False, False, False]
# Add state as discrete tokens (Pi05 style)
# Discretize state into bins
discretized_state = jnp.digitize(obs.state, bins=jnp.linspace(-1, 1, 256 + 1)[:-1]) - 1
discretized_state = jnp.clip(discretized_state, 0, 255) # Ensure valid range
# Embed each dimension of the discretized state
state_tokens = []
for i in range(obs.state.shape[-1]):
state_dim_tokens = self.PaliGemma.llm(discretized_state[:, i:i+1], method="embed")
state_tokens.append(state_dim_tokens)
if state_tokens:
state_tokens = jnp.concatenate(state_tokens, axis=1) # shape: [batch_size, state_dim, embed_dim]
tokens.append(state_tokens)
input_mask.append(jnp.ones((obs.state.shape[0], obs.state.shape[-1]), dtype=jnp.bool_))
# State tokens have full bidirectional attention with all prefix tokens
# (images, task, stages, and other state tokens)
ar_mask += [False] * state_tokens.shape[1]
# FAST tokens (from observation if provided)
if self.config.use_fast_auxiliary and obs.fast_tokens is not None:
fast_tokens = obs.fast_tokens # [B, T]
fast_token_mask = obs.fast_token_mask # [B, T]
# Teacher forcing: shift right [BOS, tok0, tok1, ..., tok_{T-1}]
bos_token = jnp.zeros((fast_tokens.shape[0], 1), dtype=jnp.int32)
shifted_tokens = jnp.concatenate([bos_token, fast_tokens[:, :-1]], axis=1)
# Shift mask too: [True, mask_0, mask_1, ..., mask_{T-1}]
bos_mask = jnp.ones((fast_tokens.shape[0], 1), dtype=jnp.bool_)
shifted_mask = jnp.concatenate([bos_mask, fast_token_mask[:, :-1]], axis=1)
# Embed using FAST embedding layer (NOT Paligemma!)
fast_token_emb = self.fast_token_embedding(shifted_tokens) # [B, T, D]
tokens.append(fast_token_emb)
input_mask.append(shifted_mask) # Use the actual token mask
# Causal for FAST: ALL tokens are causal (pure autoregressive)
ar_mask += [True] * shifted_tokens.shape[1]
tokens = jnp.concatenate(tokens, axis=1)
input_mask = jnp.concatenate(input_mask, axis=1)
ar_mask = jnp.array(ar_mask)
return tokens, input_mask, ar_mask
@at.typecheck
def embed_suffix(
self, obs: Observation, noisy_actions: _model.Actions, timestep: at.Float[at.Array, " b"]
) -> tuple[
at.Float[at.Array, "b s emb"],
at.Bool[at.Array, "b s"],
at.Bool[at.Array, " s"],
at.Float[at.Array, "b emb"],
]:
input_mask = []
ar_mask = []
tokens = []
# Pi05 style: no explicit state token in suffix (it's in prefix as discrete tokens)
action_tokens = self.action_in_proj(noisy_actions)
# Embed timestep using sine-cosine positional encoding
time_emb = posemb_sincos(timestep, self.action_in_proj.out_features, min_period=4e-3, max_period=4.0)
# Pi05 style: time MLP for adaRMS
time_emb = self.time_mlp_in(time_emb)
time_emb = nnx.swish(time_emb)
time_emb = self.time_mlp_out(time_emb)
time_emb = nnx.swish(time_emb)
action_expert_tokens = action_tokens
adarms_cond = time_emb
tokens.append(action_expert_tokens)
input_mask.append(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_))
# image/task/state inputs do not attend to action tokens
ar_mask += [True] + ([False] * (self.action_horizon - 1))
tokens = jnp.concatenate(tokens, axis=1)
input_mask = jnp.concatenate(input_mask, axis=1)
ar_mask = jnp.array(ar_mask)
return tokens, input_mask, ar_mask, adarms_cond
@override
def compute_loss(
self, rng: at.KeyArrayLike, observation: Observation, actions: _model.Actions, *, train: bool = False
) -> at.Float[at.Array, "*b ah"]:
"""Not used - we only use compute_detailed_loss() for training."""
raise NotImplementedError("Use compute_detailed_loss() instead")
@override
def compute_detailed_loss(
self, rng: at.KeyArrayLike, observation: Observation, actions: _model.Actions, *, train: bool = False, num_flow_samples: int = 1
) -> dict[str, at.Float[at.Array, "*b"]]:
"""
Compute detailed loss with multiple flow matching samples.
Simplified approach using KV cache:
- Compute prefix KV cache once (with FAST tokens)
- Remove FAST tokens from cache (action expert doesn't attend to FAST)
- Process N flow samples independently, each reusing the same cached prefix
- Each sample has different noise and different time
- Average losses across samples
"""
losses = {}
preprocess_rng, rng = jax.random.split(rng)
observation = preprocess_observation(preprocess_rng, observation, train=train)
batch_size = actions.shape[0]
# 1. Embed prefix once (includes FAST tokens if provided in observation)
prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation)
# 2. Compute prefix KV cache
prefix_attn_mask = make_attn_mask(prefix_mask, prefix_ar_mask)
positions_prefix = jnp.cumsum(prefix_mask, axis=1) - 1
(prefix_out, _), kv_cache_full = self.PaliGemma.llm(
[prefix_tokens, None],
mask=prefix_attn_mask,
positions=positions_prefix
)
# 3. Predict stage from VLM output of base task token
# Base task token is the first token after all image tokens
# Image tokens all have ar_mask=False, task starts with ar_mask=False (base) then True (stage tokens)
# Structure: [images (all False)] [base_task (False)] [stages (True, False, False, False)]
# Find first True (first stage token), base task is at that index - 1
first_stage_token_idx = jnp.argmax(prefix_ar_mask) # Returns index of first True
base_task_token_idx = first_stage_token_idx - 1
base_task_output = prefix_out[:, base_task_token_idx, :]
subtask_logits = self.stage_pred_from_vlm(base_task_output) # [B, MAX_NUM_STAGES]
# Mask out invalid stages for each task (vectorized JAX operations)
task_ids = observation.tokenized_prompt[:, 0] # [B]
task_num_stages_array = jnp.array(TASK_NUM_STAGES, dtype=jnp.int32)
task_num_stages = task_num_stages_array[task_ids] # [B] - JAX array indexing
stage_range = jnp.arange(MAX_NUM_STAGES) # [15]
valid_mask = stage_range[None, :] < task_num_stages[:, None] # [B, 15]
subtask_logits = jnp.where(valid_mask, subtask_logits, -jnp.inf) # Mask invalid stages
# 4. Extract FAST loss from prefix output (before removing from cache)
fast_loss_value = 0.0
fast_len = 0
fast_targets = observation.fast_tokens
fast_token_mask = observation.fast_token_mask
if self.config.use_fast_auxiliary and fast_targets is not None:
fast_len = fast_targets.shape[1]
fast_start_idx = prefix_tokens.shape[1] - fast_len
fast_outputs = prefix_out[:, fast_start_idx:, :] # [B, T, D]
# Project to FAST vocab
fast_logits = self.fast_token_proj(fast_outputs) # [B, T, vocab_size]
# Cross-entropy loss with teacher forcing
pred_logits = fast_logits # [B, T, vocab]
target_tokens = fast_targets # [B, T]
loss_mask = fast_token_mask # [B, T]
log_probs = jax.nn.log_softmax(pred_logits, axis=-1)
target_log_probs = jnp.take_along_axis(
log_probs,
target_tokens[:, :, None],
axis=-1
).squeeze(-1) # [B, T]
fast_token_loss = -target_log_probs # [B, T]
# Apply mask and normalize by number of valid tokens
masked_loss = fast_token_loss * loss_mask # [B, T]
num_valid_tokens = jnp.maximum(jnp.sum(loss_mask, axis=-1), 1) # [B]
losses["fast_loss"] = jnp.sum(masked_loss, axis=-1) / num_valid_tokens # [B]
# Accuracy (only on valid tokens)
pred_tokens = jnp.argmax(pred_logits, axis=-1)
correct = (pred_tokens == target_tokens) * loss_mask
losses["fast_accuracy"] = jnp.sum(correct, axis=-1) / num_valid_tokens
fast_loss_value = self.config.fast_loss_weight * jnp.mean(losses["fast_loss"])
elif fast_targets is not None:
# FAST auxiliary is disabled but data contains FAST tokens
raise ValueError(
"use_fast_auxiliary=False but observation contains fast_tokens. "
"Either enable use_fast_auxiliary in config or ensure data doesn't contain fast_tokens."
)
# 5. Remove FAST tokens from KV cache (action expert doesn't attend to FAST)
# KV cache shape: [layers, batch, seq_len, num_kv_heads, head_dim]
if fast_len > 0:
cache_k, cache_v = kv_cache_full
# Remove last fast_len tokens from sequence dimension
cache_k = cache_k[:, :, :-fast_len, :, :]
cache_v = cache_v[:, :, :-fast_len, :, :]
kv_cache_for_actions = (cache_k, cache_v)
prefix_len_for_actions = prefix_tokens.shape[1] - fast_len
# Truncate prefix mask and ar_mask for action expert
prefix_mask_for_actions = prefix_mask[:, :-fast_len]
prefix_ar_mask_for_actions = prefix_ar_mask[:-fast_len]
else:
kv_cache_for_actions = kv_cache_full
prefix_len_for_actions = prefix_tokens.shape[1]
prefix_mask_for_actions = prefix_mask
prefix_ar_mask_for_actions = prefix_ar_mask
# 6. Knowledge insulation: stop gradients from action expert to VLM
# This must happen BEFORE kv_transform so transform still receives gradients
if self.config.use_knowledge_insulation:
kv_cache_for_actions = jax.tree.map(jax.lax.stop_gradient, kv_cache_for_actions)
# 7. Transform KV cache (after stop_gradient, so it receives action expert gradients)
if self.kv_transform is not None:
kv_cache_for_actions = self.kv_transform(kv_cache_for_actions)
# 8. Define single flow sample processing
def process_one_flow_sample(sample_rng):
"""Process one flow sample using the original cached prefix."""
noise_rng, time_rng = jax.random.split(sample_rng)
# Generate different noise and time for this sample
noise = self.generate_correlated_noise(noise_rng, batch_size)
time = jax.random.beta(time_rng, 1.5, 1, (batch_size,)) * 0.999 + 0.001
# Compute noisy actions and target velocity
time_expanded = time[:, None, None]
x_t = time_expanded * noise + (1 - time_expanded) * actions
u_t = noise - actions
# Embed suffix for this sample
suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix(
observation, x_t, time
)
# Build attention mask: suffix attends to prefix (without FAST) + itself
# When using KV cache, mask shape should be [batch, suffix_len, prefix_len + suffix_len]
suffix_attn_mask = make_attn_mask(suffix_mask, suffix_ar_mask)
prefix_attn_mask = einops.repeat(
prefix_mask_for_actions, "b p -> b s p", s=suffix_tokens.shape[1]
)
full_attn_mask = jnp.concatenate([prefix_attn_mask, suffix_attn_mask], axis=-1)
# Positions for suffix start after cached prefix
suffix_positions = prefix_len_for_actions + jnp.cumsum(suffix_mask, axis=-1) - 1
# Forward pass with cached prefix (discard returned cache - don't modify original!)
(_, suffix_out), _ = self.PaliGemma.llm(
[None, suffix_tokens],
mask=full_attn_mask,
positions=suffix_positions,
kv_cache=kv_cache_for_actions, # Original cache, reused for all samples
adarms_cond=[None, adarms_cond]
)
# Compute velocity and loss
v_t = self.action_out_proj(suffix_out[:, -self.action_horizon:])
action_loss = jnp.square(v_t - u_t) # [B, H, D]
return action_loss
# 9. Vectorize over N flow samples
# Disable type checking inside vmap (jaxtyping doesn't handle traced values well)
flow_rngs = jax.random.split(rng, num_flow_samples)
with at.disable_typechecking():
all_action_losses = jax.vmap(process_one_flow_sample)(flow_rngs) # [N, B, H, D]
# 10. Average over flow samples
action_loss = jnp.mean(all_action_losses, axis=0) # [B, H, D]
# 11. Build per-dimension action losses
# Base velocity (x,y,z)
losses["action_loss_base_vel_x"] = jnp.mean(action_loss[..., 0], axis=-1)
losses["action_loss_base_vel_y"] = jnp.mean(action_loss[..., 1], axis=-1)
losses["action_loss_base_vel_z"] = jnp.mean(action_loss[..., 2], axis=-1)
# Trunk joints (4)
for i in range(4):
losses[f"action_loss_trunk_{i}"] = jnp.mean(action_loss[..., 3+i], axis=-1)
# Left arm joints (7)
for i in range(7):
losses[f"action_loss_left_arm_{i}"] = jnp.mean(action_loss[..., 7+i], axis=-1)
# Left gripper
losses["action_loss_left_gripper"] = jnp.mean(action_loss[..., 14], axis=-1)
# Right arm joints (7)
for i in range(7):
losses[f"action_loss_right_arm_{i}"] = jnp.mean(action_loss[..., 15+i], axis=-1)
# Right gripper
losses["action_loss_right_gripper"] = jnp.mean(action_loss[..., 22], axis=-1)
# Total action loss: mean over horizon (H) and action dims (D) -> [B]
losses["action_loss"] = jnp.mean(action_loss, axis=(-2, -1))
# 12. Add subtask loss during training
subtask_loss_value = 0.0
if train and observation.tokenized_prompt.shape[1] > 1:
ground_truth_subtask = observation.tokenized_prompt[:, 1]
subtask_loss = -jax.nn.log_softmax(subtask_logits)[
jnp.arange(ground_truth_subtask.shape[0]), ground_truth_subtask
]
losses["subtask_loss"] = jnp.mean(subtask_loss)
losses["subtask_accuracy"] = jnp.mean(
jnp.argmax(subtask_logits, axis=-1) == ground_truth_subtask
)
subtask_loss_value = self.config.subtask_loss_weight * jnp.mean(subtask_loss)
# 13. Total loss
losses["total_loss"] = losses["action_loss"] + subtask_loss_value + fast_loss_value
return losses
@override
def sample_actions(
self,
rng: at.KeyArrayLike,
observation: Observation,
*,
num_steps: int | at.Int[at.Array, ""] = 20,
noise: at.Float[at.Array, "b ah ad"] | None = None,
initial_actions: at.Float[at.Array, "b n ad"] | None = None,
) -> _model.Actions:
observation = preprocess_observation(None, observation, train=False)
# Note that we use the convention more common in diffusion literature, where t=1 is noise and t=0 is the target
# distribution. yes, this is the opposite of the pi0 paper, and I'm sorry.
dt = -1.0 / num_steps
batch_size = observation.state.shape[0]
# Generate or constrain noise based on inpainting requirements
if initial_actions is not None:
# INPAINTING PATH: Construct constrained noise z that satisfies initial_actions
num_initial_actions = initial_actions.shape[1]
input_action_dim = initial_actions.shape[2]
# Pad initial_actions to full model dimensions (32D) and action_horizon (30)
if input_action_dim < self.action_dim:
action_padding = jnp.zeros((batch_size, num_initial_actions, self.action_dim - input_action_dim))
initial_actions_full_dim = jnp.concatenate([initial_actions, action_padding], axis=2)
else:
initial_actions_full_dim = initial_actions[:, :, :self.action_dim]
if num_initial_actions < self.action_horizon:
seq_padding = jnp.zeros((batch_size, self.action_horizon - num_initial_actions, self.action_dim))
initial_actions_padded = jnp.concatenate([initial_actions_full_dim, seq_padding], axis=1)
else:
initial_actions_padded = initial_actions_full_dim[:, :self.action_horizon]
# Compute O and U indices for inpainting (JIT-safe: static list comprehensions)
flat_dim = self.action_horizon * self.action_dim
# Build O_indices: first num_initial_actions timesteps, first input_action_dim dimensions
O_indices = jnp.array([
t * self.action_dim + d
for t in range(num_initial_actions)
for d in range(input_action_dim)
], dtype=jnp.int32)
# Build U_indices: all other indices (JIT-safe: static list comprehension)
# Python set operations happen at trace time (before JIT), so this is safe
O_set = {t * self.action_dim + d for t in range(num_initial_actions) for d in range(input_action_dim)}
U_indices = jnp.array([
i for i in range(flat_dim) if i not in O_set
], dtype=jnp.int32)
# Generate noise
rng, noise_rng = jax.random.split(rng)
if self.correlation_loaded:
# CORRELATED NOISE: Sample with correlation matrix
noise = self.generate_correlated_noise(noise_rng, batch_size)
else:
# FALLBACK: Independent noise
noise = jax.random.normal(noise_rng, (batch_size, self.action_horizon, self.action_dim))
# Extract fixed z_O and x0_O for constraint enforcement
noise_flat = noise.reshape(batch_size, flat_dim)
fixed_z_O = noise_flat[:, O_indices] # [b, |O|] - fixed noise for inpainting
x0_O = initial_actions_padded.reshape(batch_size, flat_dim)[:, O_indices] # [b, |O|] - target actions
# Precompute correction matrix for correlation-aware inpainting
inpainting_cache = None
if self.correlation_loaded:
cache_key = (num_initial_actions, input_action_dim)
if cache_key not in self.inpainting_cache:
logger.info(f"Computing correction matrix for {num_initial_actions} steps, {input_action_dim} dims...")
self.inpainting_cache[cache_key] = self._precompute_correction_matrix(O_indices, U_indices)
inpainting_cache = self.inpainting_cache[cache_key]
else:
# NO INPAINTING: Standard noise generation
if noise is None:
rng, noise_rng = jax.random.split(rng)
noise = self.generate_correlated_noise(noise_rng, batch_size)
fixed_z_O = None
x0_O = None
O_indices = None
inpainting_cache = None
# Split RNG for step loop
rng, step_rng = jax.random.split(rng)
# Ensure FAST tokens are never used during inference
if observation.fast_tokens is not None:
raise ValueError(
"FAST tokens must not be provided during inference (sample_actions). "
"FAST tokens are only used during training for auxiliary loss. "
"Set observation.fast_tokens=None before calling sample_actions."
)
# First fill KV cache with a forward pass of the prefix (no FAST tokens during inference)
prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation)
prefix_attn_mask = make_attn_mask(prefix_mask, prefix_ar_mask)
positions = jnp.cumsum(prefix_mask, axis=1) - 1
(prefix_out, _), kv_cache = self.PaliGemma.llm([prefix_tokens, None], mask=prefix_attn_mask, positions=positions)
# Predict stage from VLM output of base task token
# Find base task token position (same logic as in compute_detailed_loss)
first_stage_token_idx = jnp.argmax(prefix_ar_mask) # Returns index of first True
base_task_token_idx = first_stage_token_idx - 1
base_task_output = prefix_out[:, base_task_token_idx, :]
subtask_logits = self.stage_pred_from_vlm(base_task_output) # [B, MAX_NUM_STAGES]
# Mask out invalid stages for each task (vectorized JAX operations)
task_ids = observation.tokenized_prompt[:, 0] # [B]
task_num_stages_array = jnp.array(TASK_NUM_STAGES, dtype=jnp.int32)
task_num_stages = task_num_stages_array[task_ids] # [B] - JAX array indexing
stage_range = jnp.arange(MAX_NUM_STAGES) # [15]
valid_mask = stage_range[None, :] < task_num_stages[:, None] # [B, 15]
subtask_logits = jnp.where(valid_mask, subtask_logits, -jnp.inf)
# Transform KV cache for cross-layer attention
if self.kv_transform is not None:
kv_cache = self.kv_transform(kv_cache)
def step(carry):
x_t, time, step_rng = carry
# Use config value for time threshold
TIME_THRESHOLD_INPAINT = self.config.time_threshold_inpaint
# Model forward pass
suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix(
observation, x_t, jnp.broadcast_to(time, batch_size)
)
suffix_attn_mask = make_attn_mask(suffix_mask, suffix_ar_mask)
prefix_attn_mask = einops.repeat(prefix_mask, "b p -> b s p", s=suffix_tokens.shape[1])
full_attn_mask = jnp.concatenate([prefix_attn_mask, suffix_attn_mask], axis=-1)
assert full_attn_mask.shape == (
batch_size,
suffix_tokens.shape[1],
prefix_tokens.shape[1] + suffix_tokens.shape[1],
)
positions = jnp.sum(prefix_mask, axis=-1)[:, None] + jnp.cumsum(suffix_mask, axis=-1) - 1
(prefix_out, suffix_out), _ = self.PaliGemma.llm(
[None, suffix_tokens],
mask=full_attn_mask,
positions=positions,
kv_cache=kv_cache,
adarms_cond=[None, adarms_cond],
)
assert prefix_out is None
v_t = self.action_out_proj(suffix_out[:, -self.action_horizon :])
# Euler step: x_{t+dt} = x_t + dt * v_t
x_t_new = x_t + dt * v_t
# Apply correlation-aware inpainting correction
# Only enforce when time > TIME_THRESHOLD_INPAINT (let model be free in final steps)
if fixed_z_O is not None:
time_new = time + dt
def apply_correlated_correction(x):
x_flat = x.reshape(batch_size, -1)
# Compute desired state at O: x_t[O] = (1-t)*x0[O] + t*z_O
x_desired_O = (1.0 - time_new) * x0_O + time_new * fixed_z_O # [b, |O|]
# Compute correction at O
delta_O = x_desired_O - x_flat[:, O_indices] # [b, |O|]
# Apply hard constraint at O
x_flat = x_flat.at[:, O_indices].set(x_desired_O)
# If correlation matrix available, propagate correction to U
if inpainting_cache is not None:
correction_matrix = inpainting_cache['correction_matrix'] # [|U|, |O|]
U_indices_cached = inpainting_cache['U_indices']
# Compute correlated correction: δ_U = Σ_{UO}Σ_{OO}^{-1} @ δ_O
delta_U = delta_O @ correction_matrix.T # [b, |U|]
# Skip if correction too large (indicates instability)
max_correction = jnp.max(jnp.abs(delta_U))
x_flat = jax.lax.cond(
# Prevents exploding corrections in case of noisy out of distribution initial actions
max_correction <= 1.0,
lambda x: x.at[:, U_indices_cached].add(delta_U),
lambda x: x,
x_flat
)
# Sanity check: if Σ = I, correction_matrix = 0, so delta_U = 0 that is correct
# If the correlation is 1 everywhere we will go to the flat prediction that is correct
return x_flat.reshape(batch_size, self.action_horizon, self.action_dim)
# Only apply correction when NEW time > threshold
x_t_new = jax.lax.cond(
time_new > TIME_THRESHOLD_INPAINT,
apply_correlated_correction,
lambda x: x,
x_t_new
)
return x_t_new, time + dt, step_rng
def cond(carry):
x_t, time, step_rng = carry
# Robust to floating-point error
return time >= -dt / 2
x_0, _, _ = jax.lax.while_loop(cond, step, (noise, 1.0, step_rng))
return x_0, subtask_logits
"""The main model for BEHAVIOR-1K challenge.
Based on Pi0.5 implementation from PhysicalIntelligence/openpi
"""
import logging
import pathlib
import einops
import flax.nnx as nnx
import flax.nnx.bridge as nnx_bridge
import jax
import jax.numpy as jnp
from typing_extensions import override
from openpi.models import model as _model
from openpi.models import gemma as _gemma
from openpi.models import siglip as _siglip
from openpi.models.pi0 import make_attn_mask, posemb_sincos
from openpi.shared import array_typing as at
# Import from our custom modules
from b1k.models import pi_behavior_config
from b1k.models.observation import Observation, preprocess_observation
from b1k.models.pi_behavior_config import (
TASK_NUM_STAGES,
MAX_NUM_STAGES,
TOTAL_TASK_STAGE_EMBEDDINGS,
TASK_STAGE_OFFSETS
)
logger = logging.getLogger("b1k")
class KVCacheTransform(nnx.Module):
"""Transforms prefix KV cache by mixing across layers.
Each destination layer's K and V become learnable linear combinations
of all source layers' K and V, plus a bias term. This allows the action
expert to attend to learned combinations of VLM layers rather than being
forced to attend layer-by-layer.
Initialized as identity transform (k_coeffs = I, bias = 0) so the model
starts with the same behavior as without transformation.
"""
def __init__(self, num_layers: int, head_dim: int, num_kv_heads: int, rngs: nnx.Rngs):
# K transformation: [dest_layer, src_layer]
# Initialize as identity so transformation is initially a no-op
self.k_coeffs = nnx.Param(jnp.eye(num_layers, dtype=jnp.float32))
# K bias: [layer, num_kv_heads, head_dim]
# Initialize as zeros
self.k_bias = nnx.Param(jnp.zeros((num_layers, num_kv_heads, head_dim), dtype=jnp.float32))
# V transformation (independent from K)
self.v_coeffs = nnx.Param(jnp.eye(num_layers, dtype=jnp.float32))
self.v_bias = nnx.Param(jnp.zeros((num_layers, num_kv_heads, head_dim), dtype=jnp.float32))
def __call__(self, kv_cache: tuple[jnp.ndarray, jnp.ndarray]) -> tuple[jnp.ndarray, jnp.ndarray]:
"""Transform KV cache by mixing across layers.
Args:
kv_cache: Tuple of (cache_k, cache_v) where each has shape
[num_layers, batch, seq_len, num_kv_heads, head_dim]
Returns:
Transformed (k_new, v_new) with same shape and dtype as input
"""
cache_k, cache_v = kv_cache
# Shape: [layers, batch, seq_len, num_kv_heads, head_dim]
# Preserve original dtype (important for bfloat16 training)
original_dtype = cache_k.dtype
# Transform K: each destination layer is a weighted combination of all source layers
# k_new[dest] = sum_src(k_coeffs[dest, src] * cache_k[src]) + k_bias[dest]
# Einsum: [dest, src] @ [src, batch, seq, heads, dim] -> [dest, batch, seq, heads, dim]
k_new = jnp.einsum('ds,sbtkh->dbtkh', self.k_coeffs.value, cache_k)
k_new = k_new + self.k_bias.value[:, None, None, :, :] # Add bias
# Transform V (same operation, independent parameters)
v_new = jnp.einsum('ds,sbtkh->dbtkh', self.v_coeffs.value, cache_v)
v_new = v_new + self.v_bias.value[:, None, None, :, :]
# Cast back to original dtype
k_new = k_new.astype(original_dtype)
v_new = v_new.astype(original_dtype)
return (k_new, v_new)
class PiBehavior(_model.BaseModel):
def __init__(self, config: pi_behavior_config.PiBehaviorConfig, rngs: nnx.Rngs):
super().__init__(config.action_dim, config.action_horizon, config.max_token_len)
# Store config for later use
self.config = config
paligemma_config = _gemma.get_config(config.paligemma_variant)