import dataclasses import logging import einops import flax.nnx as nnx import flax.nnx.bridge as nnx_bridge import jax import jax.numpy as jnp from typing_extensions import override from openpi.models import model as _model from openpi.models import pi0_config import openpi.models.gemma as _gemma import openpi.models.siglip as _siglip from openpi.shared import array_typing as at logger = logging.getLogger("openpi") def make_attn_mask(input_mask, mask_ar): """Adapted from big_vision. Tokens can attend to valid inputs tokens which have a cumulative mask_ar smaller or equal to theirs. This way `mask_ar` bool[?B, N] can be used to setup several types of attention, for example: [[1 1 1 1 1 1]]: pure causal attention. [[0 0 0 1 1 1]]: prefix-lm attention. The first 3 tokens can attend between themselves and the last 3 tokens have a causal attention. The first entry could also be a 1 without changing behaviour. [[1 0 1 0 1 0 0 1 0 0]]: causal attention between 4 blocks. Tokens of a block can attend all previous blocks and all tokens on the same block. Args: input_mask: bool[B, N] true if its part of the input, false if padding. mask_ar: bool[?B, N] mask that's true where previous tokens cannot depend on it and false where it shares the same attention mask as the previous token. """ mask_ar = jnp.broadcast_to(mask_ar, input_mask.shape) cumsum = jnp.cumsum(mask_ar, axis=1) attn_mask = cumsum[:, None, :] <= cumsum[:, :, None] valid_mask = input_mask[:, None, :] * input_mask[:, :, None] return jnp.logical_and(attn_mask, valid_mask) @at.typecheck def posemb_sincos( pos: at.Real[at.Array, " b"], embedding_dim: int, min_period: float, max_period: float ) -> at.Float[at.Array, "b {embedding_dim}"]: """Computes sine-cosine positional embedding vectors for scalar positions.""" if embedding_dim % 2 != 0: raise ValueError(f"embedding_dim ({embedding_dim}) must be divisible by 2") fraction = jnp.linspace(0.0, 1.0, embedding_dim // 2) period = min_period * (max_period / min_period) ** fraction sinusoid_input = jnp.einsum( "i,j->ij", pos, 1.0 / period * 2 * jnp.pi, precision=jax.lax.Precision.HIGHEST, ) return jnp.concatenate([jnp.sin(sinusoid_input), jnp.cos(sinusoid_input)], axis=-1) class Pi0(_model.BaseModel): def __init__(self, config: pi0_config.Pi0Config, rngs: nnx.Rngs): super().__init__(config.action_dim, config.action_horizon, config.max_token_len) self.pi05 = config.pi05 # Per-variant ranks override lora_rank; None falls back to each variant's built-in default. paligemma_lora_rank = config.paligemma_lora_rank if config.paligemma_lora_rank is not None else config.lora_rank action_expert_lora_rank = ( config.action_expert_lora_rank if config.action_expert_lora_rank is not None else config.lora_rank ) paligemma_config = _gemma.get_config(config.paligemma_variant, lora_rank=paligemma_lora_rank) action_expert_config = _gemma.get_config(config.action_expert_variant, lora_rank=action_expert_lora_rank) action_expert_overrides = {} if config.action_expert_num_moe_experts is not None: action_expert_overrides["num_local_experts"] = config.action_expert_num_moe_experts if config.action_expert_moe_layers is not None: action_expert_overrides["moe_layers"] = config.action_expert_moe_layers if config.action_expert_moe_layer_groups is not None: action_expert_overrides["moe_layer_groups"] = config.action_expert_moe_layer_groups if config.action_expert_num_experts_per_tok is not None: action_expert_overrides["num_experts_per_tok"] = config.action_expert_num_experts_per_tok if config.action_expert_router_aux_loss_coef is not None: action_expert_overrides["router_aux_loss_coef"] = config.action_expert_router_aux_loss_coef if config.action_expert_router_entropy_loss_coef is not None: action_expert_overrides["router_entropy_loss_coef"] = config.action_expert_router_entropy_loss_coef if config.action_expert_router_temperature is not None: action_expert_overrides["router_temperature"] = config.action_expert_router_temperature if config.action_expert_router_noise_std is not None: action_expert_overrides["router_noise_std"] = config.action_expert_router_noise_std if config.action_expert_moe_type is not None: action_expert_overrides["moe_type"] = config.action_expert_moe_type if config.action_expert_router_input is not None: action_expert_overrides["router_input"] = config.action_expert_router_input if config.action_expert_moe_gamma is not None: action_expert_overrides["moe_gamma"] = config.action_expert_moe_gamma if config.action_expert_router_type is not None: action_expert_overrides["router_type"] = config.action_expert_router_type if config.action_expert_router_prototype_dim is not None: action_expert_overrides["router_prototype_dim"] = config.action_expert_router_prototype_dim if config.action_expert_router_action_grad_coef is not None: action_expert_overrides["router_action_grad_coef"] = config.action_expert_router_action_grad_coef if config.action_expert_routing_contrastive_temperature is not None: action_expert_overrides["routing_contrastive_temperature"] = ( config.action_expert_routing_contrastive_temperature ) if config.action_expert_routing_contrastive_loss_coef is not None: action_expert_overrides["routing_contrastive_loss_coef"] = ( config.action_expert_routing_contrastive_loss_coef ) if config.action_expert_dead_expert_loss_coef is not None: action_expert_overrides["dead_expert_loss_coef"] = config.action_expert_dead_expert_loss_coef if config.action_expert_dead_expert_min_usage is not None: action_expert_overrides["dead_expert_min_usage"] = config.action_expert_dead_expert_min_usage if action_expert_overrides: action_expert_config = dataclasses.replace(action_expert_config, **action_expert_overrides) self.action_expert_num_moe_experts = action_expert_config.num_local_experts self.action_expert_router_aux_loss_coef = action_expert_config.router_aux_loss_coef self.action_expert_router_entropy_loss_coef = action_expert_config.router_entropy_loss_coef self.action_expert_routing_contrastive_loss_coef = action_expert_config.routing_contrastive_loss_coef self.action_expert_dead_expert_loss_coef = action_expert_config.dead_expert_loss_coef self.action_expert_router_action_grad_coef = action_expert_config.router_action_grad_coef self.action_expert_router_supervised_loss_coef = config.action_expert_router_supervised_loss_coef self.action_expert_router_action_loss_weight_coef = config.action_expert_router_action_loss_weight_coef self.action_expert_moe_layers = action_expert_config.moe_layers self.action_expert_router_input = action_expert_config.router_input self.action_expert_uses_router_context = ( self.action_expert_num_moe_experts > 0 and (self.action_expert_router_input != "token" or action_expert_config.router_type == "prototype") ) logger.info( "Pi0 effective config: pi05=%s, paligemma_variant=%s, action_expert_variant=%s, " "moe_experts=%s, top_k=%s, aux_coef=%s, entropy_coef=%s, router_temp=%s, " "router_noise_std=%s, moe_type=%s, router_input=%s, moe_gamma=%s, " "router_type=%s, router_prototype_dim=%s, router_action_grad_coef=%s, " "contrast_coef=%s, dead_coef=%s, " "router_supervised_coef=%s, action_class_weight_coef=%s, moe_layers=%s, " "moe_layer_groups=%s, " "global_router_context=%s", config.pi05, config.paligemma_variant, config.action_expert_variant, action_expert_config.num_local_experts, action_expert_config.num_experts_per_tok, action_expert_config.router_aux_loss_coef, action_expert_config.router_entropy_loss_coef, action_expert_config.router_temperature, action_expert_config.router_noise_std, action_expert_config.moe_type, action_expert_config.router_input, action_expert_config.moe_gamma, action_expert_config.router_type, action_expert_config.router_prototype_dim, action_expert_config.router_action_grad_coef, action_expert_config.routing_contrastive_loss_coef, action_expert_config.dead_expert_loss_coef, config.action_expert_router_supervised_loss_coef, config.action_expert_router_action_loss_weight_coef, action_expert_config.moe_layers, action_expert_config.moe_layer_groups, self.action_expert_uses_router_context, ) # TODO: rewrite gemma in NNX. For now, use bridge. llm = nnx_bridge.ToNNX( _gemma.Module( configs=[paligemma_config, action_expert_config], embed_dtype=config.dtype, adarms=config.pi05, ) ) llm.lazy_init(rngs=rngs, method="init", use_adarms=[False, True] if config.pi05 else [False, False]) img = nnx_bridge.ToNNX( _siglip.Module( num_classes=paligemma_config.width, variant="So400m/14", pool_type="none", scan=True, dtype_mm=config.dtype, ) ) img.lazy_init(next(iter(config.fake_obs().images.values())), train=False, rngs=rngs) self.PaliGemma = nnx.Dict(llm=llm, img=img) self.action_in_proj = nnx.Linear(config.action_dim, action_expert_config.width, rngs=rngs) if config.pi05: self.time_mlp_in = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs) self.time_mlp_out = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs) else: self.state_proj = nnx.Linear(config.action_dim, action_expert_config.width, rngs=rngs) self.action_time_mlp_in = nnx.Linear(2 * action_expert_config.width, action_expert_config.width, rngs=rngs) self.action_time_mlp_out = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs) self.action_out_proj = nnx.Linear(action_expert_config.width, config.action_dim, rngs=rngs) # This attribute gets automatically set by model.train() and model.eval(). self.deterministic = True @at.typecheck def embed_prefix( self, obs: _model.Observation ) -> tuple[at.Float[at.Array, "b s emb"], at.Bool[at.Array, "b s"], at.Bool[at.Array, " s"]]: input_mask = [] ar_mask = [] tokens = [] # embed images for name in obs.images: image_tokens, _ = self.PaliGemma.img(obs.images[name], train=False) tokens.append(image_tokens) input_mask.append( einops.repeat( obs.image_masks[name], "b -> b s", s=image_tokens.shape[1], ) ) # image tokens attend to each other ar_mask += [False] * image_tokens.shape[1] # add language (aka tokenized inputs) if obs.tokenized_prompt is not None: tokenized_inputs = self.PaliGemma.llm(obs.tokenized_prompt, method="embed") tokens.append(tokenized_inputs) input_mask.append(obs.tokenized_prompt_mask) # full attention between image and language inputs ar_mask += [False] * tokenized_inputs.shape[1] tokens = jnp.concatenate(tokens, axis=1) input_mask = jnp.concatenate(input_mask, axis=1) ar_mask = jnp.array(ar_mask) return tokens, input_mask, ar_mask @staticmethod def _masked_mean_pool(tokens: jnp.ndarray, mask: jnp.ndarray) -> jnp.ndarray: weights = mask.astype(tokens.dtype)[..., None] denom = jnp.clip(jnp.sum(weights, axis=1), min=1.0) return jnp.sum(tokens * weights, axis=1) / denom @staticmethod def _router_supervised_loss( router_metrics: dict[str, at.Array], router_label: at.Array | None, router_class_weight: at.Array | None, moe_layers: tuple[int, ...] | None = None, ) -> tuple[jnp.ndarray, jnp.ndarray]: if router_label is None or "router_route_logits" not in router_metrics: zero = jnp.array(0.0, dtype=jnp.float32) return zero, zero logits = router_metrics["router_route_logits"].astype(jnp.float32) raw_labels = jnp.asarray(router_label) weights = None if router_class_weight is None else jnp.asarray(router_class_weight, dtype=jnp.float32) if raw_labels.ndim == 2 and raw_labels.shape[-1] == logits.shape[-1]: if logits.ndim == 4: logits = jnp.mean(logits[:, :, 0, :], axis=0) elif logits.ndim == 3: logits = jnp.mean(logits[:, :, :], axis=0) if logits.ndim != 2: raise ValueError(f"Unsupported router logits shape for soft supervised loss: {logits.shape}") targets = raw_labels.astype(jnp.float32) target_mass = jnp.sum(targets, axis=-1) valid = target_mass > 0 targets = targets / jnp.clip(target_mass[:, None], a_min=1e-6) per_example = -jnp.sum(targets * jax.nn.log_softmax(logits, axis=-1), axis=-1) if weights is None: weights = jnp.ones_like(per_example, dtype=jnp.float32) weights = jnp.where(valid, weights, 0.0) loss = jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) accuracy = ( jnp.sum((jnp.argmax(logits, axis=-1) == jnp.argmax(targets, axis=-1)) * valid) / jnp.clip(jnp.sum(valid), a_min=1) ) return loss.astype(jnp.float32), accuracy.astype(jnp.float32) labels = raw_labels.astype(jnp.int32) if labels.ndim == 2: if logits.ndim != 4: raise ValueError(f"Layer router labels require per-layer logits, got {logits.shape}") if moe_layers is None: raise ValueError("Layer router labels require action_expert_moe_layers.") selected_logits = [] for layer in moe_layers: selected_logits.append(logits[layer, :, 0, :]) logits = jnp.stack(selected_logits, axis=1) if logits.shape[1] != labels.shape[1]: raise ValueError(f"Got {labels.shape[1]} label columns for {logits.shape[1]} MoE layers.") valid = labels >= 0 clipped = jnp.clip(labels, 0, logits.shape[-1] - 1) per_example = -jax.nn.log_softmax(logits, axis=-1)[ jnp.arange(labels.shape[0])[:, None], jnp.arange(labels.shape[1])[None, :], clipped, ] if weights is None: weights = jnp.ones_like(per_example, dtype=jnp.float32) weights = jnp.where(valid, weights, 0.0) loss = jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) accuracy = ( jnp.sum((jnp.argmax(logits, axis=-1) == labels) * valid) / jnp.clip(jnp.sum(valid), a_min=1) ) return loss.astype(jnp.float32), accuracy.astype(jnp.float32) if logits.ndim == 4: # Dense router stores [layers, batch, 1, experts]; prototype is the same shape. logits = jnp.mean(logits[:, :, 0, :], axis=0) elif logits.ndim == 3: logits = jnp.mean(logits[:, :, :], axis=0) if logits.ndim != 2: raise ValueError(f"Unsupported router logits shape for supervised loss: {logits.shape}") valid = labels >= 0 labels = jnp.clip(labels, 0, logits.shape[-1] - 1) per_example = -jax.nn.log_softmax(logits, axis=-1)[jnp.arange(labels.shape[0]), labels] if weights is None: weights = jnp.ones_like(per_example, dtype=jnp.float32) weights = jnp.where(valid, weights, 0.0) loss = jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) accuracy = jnp.sum((jnp.argmax(logits, axis=-1) == labels) * valid) / jnp.clip(jnp.sum(valid), a_min=1) return loss.astype(jnp.float32), accuracy.astype(jnp.float32) @staticmethod def _layer_router_supervised_metrics( router_metrics: dict[str, at.Array], router_label: at.Array | None, router_class_weight: at.Array | None, moe_layers: tuple[int, ...] | None, ) -> dict[str, jnp.ndarray]: if ( router_label is None or moe_layers is None or "router_route_logits" not in router_metrics ): return {} raw_labels = jnp.asarray(router_label) if raw_labels.ndim != 2: return {} logits_all = router_metrics["router_route_logits"].astype(jnp.float32) if raw_labels.shape[-1] == logits_all.shape[-1]: return {} labels = raw_labels.astype(jnp.int32) top_indices_all = router_metrics.get("router_route_top_indices") if logits_all.ndim != 4: return {} weights_all = ( jnp.ones_like(labels, dtype=jnp.float32) if router_class_weight is None else jnp.asarray(router_class_weight, dtype=jnp.float32) ) metrics = {} for col, layer in enumerate(moe_layers): logits = logits_all[layer, :, 0, :] layer_labels = labels[:, col] valid = layer_labels >= 0 clipped = jnp.clip(layer_labels, 0, logits.shape[-1] - 1) per_example = -jax.nn.log_softmax(logits, axis=-1)[jnp.arange(layer_labels.shape[0]), clipped] weights = jnp.where(valid, weights_all[:, col], 0.0) metrics[f"router_l{layer}_supervised_loss"] = ( jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) ).astype(jnp.float32) metrics[f"router_l{layer}_supervised_accuracy"] = ( jnp.sum((jnp.argmax(logits, axis=-1) == layer_labels) * valid) / jnp.clip(jnp.sum(valid), a_min=1) ).astype(jnp.float32) if top_indices_all is not None: top = top_indices_all[layer, :, 0, :] usage = jnp.mean(jax.nn.one_hot(top, logits.shape[-1], dtype=jnp.float32), axis=(0, 1)) for expert_idx in range(usage.shape[0]): metrics[f"router_l{layer}_usage_{expert_idx}"] = usage[expert_idx] return metrics @at.typecheck def embed_suffix( self, obs: _model.Observation, noisy_actions: _model.Actions, timestep: at.Float[at.Array, " b"] ) -> tuple[ at.Float[at.Array, "b s emb"], at.Bool[at.Array, "b s"], at.Bool[at.Array, " s"], at.Float[at.Array, "b emb"] | None, ]: input_mask = [] ar_mask = [] tokens = [] if not self.pi05: # add a single state token state_token = self.state_proj(obs.state)[:, None, :] tokens.append(state_token) input_mask.append(jnp.ones((obs.state.shape[0], 1), dtype=jnp.bool_)) # image/language inputs do not attend to state or actions ar_mask += [True] action_tokens = self.action_in_proj(noisy_actions) # embed timestep using sine-cosine positional encoding with sensitivity in the range [0, 1] time_emb = posemb_sincos(timestep, self.action_in_proj.out_features, min_period=4e-3, max_period=4.0) if self.pi05: # time MLP (for adaRMS) time_emb = self.time_mlp_in(time_emb) time_emb = nnx.swish(time_emb) time_emb = self.time_mlp_out(time_emb) time_emb = nnx.swish(time_emb) action_expert_tokens = action_tokens adarms_cond = time_emb else: # mix timestep + action information using an MLP (no adaRMS) time_tokens = einops.repeat(time_emb, "b emb -> b s emb", s=self.action_horizon) action_time_tokens = jnp.concatenate([action_tokens, time_tokens], axis=-1) action_time_tokens = self.action_time_mlp_in(action_time_tokens) action_time_tokens = nnx.swish(action_time_tokens) action_time_tokens = self.action_time_mlp_out(action_time_tokens) action_expert_tokens = action_time_tokens adarms_cond = None tokens.append(action_expert_tokens) input_mask.append(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_)) # image/language/state inputs do not attend to action tokens ar_mask += [True] + ([False] * (self.action_horizon - 1)) tokens = jnp.concatenate(tokens, axis=1) input_mask = jnp.concatenate(input_mask, axis=1) ar_mask = jnp.array(ar_mask) return tokens, input_mask, ar_mask, adarms_cond def _prefix_router_contexts(self, prefix_out: jnp.ndarray, prefix_mask: jnp.ndarray) -> jnp.ndarray | None: if not self.action_expert_uses_router_context: return None pooled = self._masked_mean_pool(prefix_out, prefix_mask) # Old per-layer router implementation expanded this to # (depth, batch, width) so each MoE layer could consume its own context. # return jnp.broadcast_to(pooled, (self.PaliGemma.llm.module.configs[0].depth,) + pooled.shape) # The global router now consumes one shared prefix context for all layers. return pooled @override def compute_loss( self, rng: at.KeyArrayLike, observation: _model.Observation, actions: _model.Actions, *, train: bool = False ) -> at.Float[at.Array, "*b ah"]: loss, _ = self.compute_loss_with_info(rng, observation, actions, train=train) return loss def compute_loss_with_info( self, rng: at.KeyArrayLike, observation: _model.Observation, actions: _model.Actions, *, train: bool = False, train_step: at.Array | int | None = None, ) -> tuple[at.Float[at.Array, "*b ah"], dict[str, at.Array]]: preprocess_rng, noise_rng, time_rng, router_noise_rng = jax.random.split(rng, 4) observation = _model.preprocess_observation(preprocess_rng, observation, train=train) batch_shape = actions.shape[:-2] noise = jax.random.normal(noise_rng, actions.shape) time = jax.random.beta(time_rng, 1.5, 1, batch_shape) * 0.999 + 0.001 time_expanded = time[..., None, None] x_t = time_expanded * noise + (1 - time_expanded) * actions u_t = noise - actions prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation) prefix_attn_mask = make_attn_mask(prefix_mask, prefix_ar_mask) prefix_positions = jnp.cumsum(prefix_mask, axis=1) - 1 (prefix_out, _), _, _, _, _, _ = self.PaliGemma.llm( [prefix_tokens, None], mask=prefix_attn_mask, positions=prefix_positions, ) router_contexts = self._prefix_router_contexts(jax.lax.stop_gradient(prefix_out), prefix_mask) router_action_grad_coef = jnp.asarray(self.action_expert_router_action_grad_coef, dtype=jnp.float32) router_supervised_loss_coef = jnp.asarray(self.action_expert_router_supervised_loss_coef, dtype=jnp.float32) suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix(observation, x_t, time) input_mask = jnp.concatenate([prefix_mask, suffix_mask], axis=1) ar_mask = jnp.concatenate([prefix_ar_mask, suffix_ar_mask], axis=0) attn_mask = make_attn_mask(input_mask, ar_mask) positions = jnp.cumsum(input_mask, axis=1) - 1 (_, suffix_out), _, router_aux_loss, _, _, router_metrics = self.PaliGemma.llm( [prefix_tokens, suffix_tokens], mask=attn_mask, positions=positions, adarms_cond=[None, adarms_cond], router_contexts=router_contexts, router_noise_rng=router_noise_rng, router_action_grad_coef=router_action_grad_coef, deterministic=not train, ) v_t = self.action_out_proj(suffix_out[:, -self.action_horizon :]) action_loss = jnp.mean(jnp.square(v_t - u_t), axis=-1) if ( observation.router_class_weight is not None and self.action_expert_router_action_loss_weight_coef > 0.0 ): class_weight = jnp.asarray(observation.router_class_weight, dtype=action_loss.dtype) action_weight = 1.0 + self.action_expert_router_action_loss_weight_coef * (class_weight - 1.0) action_loss = action_loss * action_weight[:, None] weighted_router_aux_loss = self.action_expert_router_aux_loss_coef * router_aux_loss router_entropy_loss = router_metrics["router_entropy_loss"] weighted_router_entropy_loss = self.action_expert_router_entropy_loss_coef * router_entropy_loss routing_contrastive_loss = router_metrics["routing_contrastive_loss"] dead_expert_loss = router_metrics["dead_expert_loss"] router_supervised_loss, router_supervised_accuracy = self._router_supervised_loss( router_metrics, observation.router_label, observation.router_class_weight, self.action_expert_moe_layers, ) weighted_router_supervised_loss = router_supervised_loss_coef * router_supervised_loss weighted_routing_contrastive_loss = ( self.action_expert_routing_contrastive_loss_coef * routing_contrastive_loss ) weighted_dead_expert_loss = self.action_expert_dead_expert_loss_coef * dead_expert_loss total_loss = ( action_loss + weighted_router_aux_loss + weighted_router_entropy_loss + weighted_routing_contrastive_loss + weighted_dead_expert_loss + weighted_router_supervised_loss ) # Keep training logs focused on the losses that are active for supervised router training. # Uncomment these diagnostics when debugging router confidence / usage behavior. # scalar_router_metrics = { # k: v # for k, v in router_metrics.items() # if getattr(v, "ndim", 0) == 0 # } loss_info = { "action_loss": jnp.mean(action_loss), # "router_aux_loss": router_aux_loss, # "router_aux_loss_weighted": weighted_router_aux_loss, # "router_entropy_loss_weighted": weighted_router_entropy_loss, "router_supervised_loss": router_supervised_loss, "router_supervised_loss_weighted": weighted_router_supervised_loss, "router_supervised_coef": router_supervised_loss_coef, "router_supervised_accuracy": router_supervised_accuracy, "router_action_grad_coef": router_action_grad_coef, # "routing_contrastive_loss": routing_contrastive_loss, # "routing_contrastive_loss_weighted": weighted_routing_contrastive_loss, # "dead_expert_loss": dead_expert_loss, # "dead_expert_loss_weighted": weighted_dead_expert_loss, # **scalar_router_metrics, } loss_info.update( self._layer_router_supervised_metrics( router_metrics, observation.router_label, observation.router_class_weight, self.action_expert_moe_layers, ) ) if "router_usage" in router_metrics: for expert_idx in range(router_metrics["router_usage"].shape[0]): loss_info[f"router_usage_{expert_idx}"] = router_metrics["router_usage"][expert_idx] return total_loss, loss_info @override def sample_actions( self, rng: at.KeyArrayLike, observation: _model.Observation, *, num_steps: int | at.Int[at.Array, ""] = 10, noise: at.Float[at.Array, "b ah ad"] | None = None, ) -> _model.Actions: observation = _model.preprocess_observation(None, observation, train=False) x_0, _ = self._sample_actions_core(rng, observation, num_steps=num_steps, noise=noise, return_router_info=False) return x_0 def sample_actions_with_router_info( self, rng: at.KeyArrayLike, observation: _model.Observation, *, num_steps: int | at.Int[at.Array, ""] = 10, noise: at.Float[at.Array, "b ah ad"] | None = None, ffn_activation_topk: int = 0, ) -> tuple[_model.Actions, dict[str, at.Array]]: observation = _model.preprocess_observation(None, observation, train=False) return self._sample_actions_core( rng, observation, num_steps=num_steps, noise=noise, return_router_info=True, ffn_activation_topk=ffn_activation_topk, ) def _sample_actions_core( self, rng: at.KeyArrayLike, observation: _model.Observation, *, num_steps: int | at.Int[at.Array, ""] = 10, noise: at.Float[at.Array, "b ah ad"] | None = None, return_router_info: bool = False, ffn_activation_topk: int = 0, ) -> tuple[_model.Actions, dict[str, at.Array] | None]: # note that we use the convention more common in diffusion literature, where t=1 is noise and t=0 is the target # distribution. yes, this is the opposite of the pi0 paper, and I'm sorry. dt = -1.0 / num_steps batch_size = observation.state.shape[0] if noise is None: noise = jax.random.normal(rng, (batch_size, self.action_horizon, self.action_dim)) # first fill KV cache with a forward pass of the prefix prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation) prefix_attn_mask = make_attn_mask(prefix_mask, prefix_ar_mask) positions = jnp.cumsum(prefix_mask, axis=1) - 1 (prefix_out, _), kv_cache, _, _, _, _ = self.PaliGemma.llm( [prefix_tokens, None], mask=prefix_attn_mask, positions=positions, ) router_contexts = self._prefix_router_contexts(prefix_out, prefix_mask) def denoise_step(carry): x_t, time = carry suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix( observation, x_t, jnp.broadcast_to(time, batch_size) ) # `suffix_attn_mask` is shape (b, suffix_len, suffix_len) indicating how the suffix tokens can attend to each # other suffix_attn_mask = make_attn_mask(suffix_mask, suffix_ar_mask) # `prefix_attn_mask` is shape (b, suffix_len, prefix_len) indicating how the suffix tokens can attend to the # prefix tokens prefix_attn_mask = einops.repeat(prefix_mask, "b p -> b s p", s=suffix_tokens.shape[1]) # `combined_mask` is shape (b, suffix_len, prefix_len + suffix_len) indicating how the suffix tokens (which # generate the queries) can attend to the full prefix + suffix sequence (which generates the keys and values) full_attn_mask = jnp.concatenate([prefix_attn_mask, suffix_attn_mask], axis=-1) assert full_attn_mask.shape == ( batch_size, suffix_tokens.shape[1], prefix_tokens.shape[1] + suffix_tokens.shape[1], ) # `positions` is shape (b, suffix_len) indicating the positions of the suffix tokens positions = jnp.sum(prefix_mask, axis=-1)[:, None] + jnp.cumsum(suffix_mask, axis=-1) - 1 (prefix_out, suffix_out), _, _, router_info, _, _ = self.PaliGemma.llm( [None, suffix_tokens], mask=full_attn_mask, positions=positions, kv_cache=kv_cache, adarms_cond=[None, adarms_cond], router_contexts=router_contexts if self.action_expert_uses_router_context else None, return_router_info=return_router_info, ffn_activation_topk=ffn_activation_topk, ) if router_info is not None: router_info = { **router_info, "router_timestep": jnp.broadcast_to(time, (batch_size,)), } assert prefix_out is None v_t = self.action_out_proj(suffix_out[:, -self.action_horizon :]) return (x_t + dt * v_t, time + dt), router_info if return_router_info: def scan_step(carry, _): return denoise_step(carry) (x_0, _), router_info = jax.lax.scan(scan_step, (noise, 1.0), xs=None, length=num_steps) return x_0, router_info def while_step(carry): carry, _ = denoise_step(carry) return carry def cond(carry): x_t, time = carry # robust to floating-point error return time >= -dt / 2 x_0, _ = jax.lax.while_loop(cond, while_step, (noise, 1.0)) return x_0, None