Download code/openpi/models/pi0.py from beataasd/pi05-clean50-trained: direct link, hf CLI and curl.
- Browser
- Download file 33.9 kB
-
https://huggingface.co/beataasd/pi05-clean50-trained/resolve/main/code/openpi/models/pi0.py
- Command line
-
hf download hf://beataasd/pi05-clean50-trained/code/openpi/models/pi0.py
-
curl -L -o pi0.py https://huggingface.co/beataasd/pi05-clean50-trained/resolve/main/code/openpi/models/pi0.py
33.9 kB
| import dataclasses | |
| import logging | |
| import einops | |
| import flax.nnx as nnx | |
| import flax.nnx.bridge as nnx_bridge | |
| import jax | |
| import jax.numpy as jnp | |
| from typing_extensions import override | |
| from openpi.models import model as _model | |
| from openpi.models import pi0_config | |
| import openpi.models.gemma as _gemma | |
| import openpi.models.siglip as _siglip | |
| from openpi.shared import array_typing as at | |
| logger = logging.getLogger("openpi") | |
| def make_attn_mask(input_mask, mask_ar): | |
| """Adapted from big_vision. | |
| Tokens can attend to valid inputs tokens which have a cumulative mask_ar | |
| smaller or equal to theirs. This way `mask_ar` bool[?B, N] can be used to | |
| setup several types of attention, for example: | |
| [[1 1 1 1 1 1]]: pure causal attention. | |
| [[0 0 0 1 1 1]]: prefix-lm attention. The first 3 tokens can attend between | |
| themselves and the last 3 tokens have a causal attention. The first | |
| entry could also be a 1 without changing behaviour. | |
| [[1 0 1 0 1 0 0 1 0 0]]: causal attention between 4 blocks. Tokens of a | |
| block can attend all previous blocks and all tokens on the same block. | |
| Args: | |
| input_mask: bool[B, N] true if its part of the input, false if padding. | |
| mask_ar: bool[?B, N] mask that's true where previous tokens cannot depend on | |
| it and false where it shares the same attention mask as the previous token. | |
| """ | |
| mask_ar = jnp.broadcast_to(mask_ar, input_mask.shape) | |
| cumsum = jnp.cumsum(mask_ar, axis=1) | |
| attn_mask = cumsum[:, None, :] <= cumsum[:, :, None] | |
| valid_mask = input_mask[:, None, :] * input_mask[:, :, None] | |
| return jnp.logical_and(attn_mask, valid_mask) | |
| def posemb_sincos( | |
| pos: at.Real[at.Array, " b"], embedding_dim: int, min_period: float, max_period: float | |
| ) -> at.Float[at.Array, "b {embedding_dim}"]: | |
| """Computes sine-cosine positional embedding vectors for scalar positions.""" | |
| if embedding_dim % 2 != 0: | |
| raise ValueError(f"embedding_dim ({embedding_dim}) must be divisible by 2") | |
| fraction = jnp.linspace(0.0, 1.0, embedding_dim // 2) | |
| period = min_period * (max_period / min_period) ** fraction | |
| sinusoid_input = jnp.einsum( | |
| "i,j->ij", | |
| pos, | |
| 1.0 / period * 2 * jnp.pi, | |
| precision=jax.lax.Precision.HIGHEST, | |
| ) | |
| return jnp.concatenate([jnp.sin(sinusoid_input), jnp.cos(sinusoid_input)], axis=-1) | |
| class Pi0(_model.BaseModel): | |
| def __init__(self, config: pi0_config.Pi0Config, rngs: nnx.Rngs): | |
| super().__init__(config.action_dim, config.action_horizon, config.max_token_len) | |
| self.pi05 = config.pi05 | |
| # Per-variant ranks override lora_rank; None falls back to each variant's built-in default. | |
| paligemma_lora_rank = config.paligemma_lora_rank if config.paligemma_lora_rank is not None else config.lora_rank | |
| action_expert_lora_rank = ( | |
| config.action_expert_lora_rank if config.action_expert_lora_rank is not None else config.lora_rank | |
| ) | |
| paligemma_config = _gemma.get_config(config.paligemma_variant, lora_rank=paligemma_lora_rank) | |
| action_expert_config = _gemma.get_config(config.action_expert_variant, lora_rank=action_expert_lora_rank) | |
| action_expert_overrides = {} | |
| if config.action_expert_num_moe_experts is not None: | |
| action_expert_overrides["num_local_experts"] = config.action_expert_num_moe_experts | |
| if config.action_expert_moe_layers is not None: | |
| action_expert_overrides["moe_layers"] = config.action_expert_moe_layers | |
| if config.action_expert_moe_layer_groups is not None: | |
| action_expert_overrides["moe_layer_groups"] = config.action_expert_moe_layer_groups | |
| if config.action_expert_num_experts_per_tok is not None: | |
| action_expert_overrides["num_experts_per_tok"] = config.action_expert_num_experts_per_tok | |
| if config.action_expert_router_aux_loss_coef is not None: | |
| action_expert_overrides["router_aux_loss_coef"] = config.action_expert_router_aux_loss_coef | |
| if config.action_expert_router_entropy_loss_coef is not None: | |
| action_expert_overrides["router_entropy_loss_coef"] = config.action_expert_router_entropy_loss_coef | |
| if config.action_expert_router_temperature is not None: | |
| action_expert_overrides["router_temperature"] = config.action_expert_router_temperature | |
| if config.action_expert_router_noise_std is not None: | |
| action_expert_overrides["router_noise_std"] = config.action_expert_router_noise_std | |
| if config.action_expert_moe_type is not None: | |
| action_expert_overrides["moe_type"] = config.action_expert_moe_type | |
| if config.action_expert_router_input is not None: | |
| action_expert_overrides["router_input"] = config.action_expert_router_input | |
| if config.action_expert_moe_gamma is not None: | |
| action_expert_overrides["moe_gamma"] = config.action_expert_moe_gamma | |
| if config.action_expert_router_type is not None: | |
| action_expert_overrides["router_type"] = config.action_expert_router_type | |
| if config.action_expert_router_prototype_dim is not None: | |
| action_expert_overrides["router_prototype_dim"] = config.action_expert_router_prototype_dim | |
| if config.action_expert_router_action_grad_coef is not None: | |
| action_expert_overrides["router_action_grad_coef"] = config.action_expert_router_action_grad_coef | |
| if config.action_expert_routing_contrastive_temperature is not None: | |
| action_expert_overrides["routing_contrastive_temperature"] = ( | |
| config.action_expert_routing_contrastive_temperature | |
| ) | |
| if config.action_expert_routing_contrastive_loss_coef is not None: | |
| action_expert_overrides["routing_contrastive_loss_coef"] = ( | |
| config.action_expert_routing_contrastive_loss_coef | |
| ) | |
| if config.action_expert_dead_expert_loss_coef is not None: | |
| action_expert_overrides["dead_expert_loss_coef"] = config.action_expert_dead_expert_loss_coef | |
| if config.action_expert_dead_expert_min_usage is not None: | |
| action_expert_overrides["dead_expert_min_usage"] = config.action_expert_dead_expert_min_usage | |
| if action_expert_overrides: | |
| action_expert_config = dataclasses.replace(action_expert_config, **action_expert_overrides) | |
| self.action_expert_num_moe_experts = action_expert_config.num_local_experts | |
| self.action_expert_router_aux_loss_coef = action_expert_config.router_aux_loss_coef | |
| self.action_expert_router_entropy_loss_coef = action_expert_config.router_entropy_loss_coef | |
| self.action_expert_routing_contrastive_loss_coef = action_expert_config.routing_contrastive_loss_coef | |
| self.action_expert_dead_expert_loss_coef = action_expert_config.dead_expert_loss_coef | |
| self.action_expert_router_action_grad_coef = action_expert_config.router_action_grad_coef | |
| self.action_expert_router_supervised_loss_coef = config.action_expert_router_supervised_loss_coef | |
| self.action_expert_router_action_loss_weight_coef = config.action_expert_router_action_loss_weight_coef | |
| self.action_expert_moe_layers = action_expert_config.moe_layers | |
| self.action_expert_router_input = action_expert_config.router_input | |
| self.action_expert_uses_router_context = ( | |
| self.action_expert_num_moe_experts > 0 | |
| and (self.action_expert_router_input != "token" or action_expert_config.router_type == "prototype") | |
| ) | |
| logger.info( | |
| "Pi0 effective config: pi05=%s, paligemma_variant=%s, action_expert_variant=%s, " | |
| "moe_experts=%s, top_k=%s, aux_coef=%s, entropy_coef=%s, router_temp=%s, " | |
| "router_noise_std=%s, moe_type=%s, router_input=%s, moe_gamma=%s, " | |
| "router_type=%s, router_prototype_dim=%s, router_action_grad_coef=%s, " | |
| "contrast_coef=%s, dead_coef=%s, " | |
| "router_supervised_coef=%s, action_class_weight_coef=%s, moe_layers=%s, " | |
| "moe_layer_groups=%s, " | |
| "global_router_context=%s", | |
| config.pi05, | |
| config.paligemma_variant, | |
| config.action_expert_variant, | |
| action_expert_config.num_local_experts, | |
| action_expert_config.num_experts_per_tok, | |
| action_expert_config.router_aux_loss_coef, | |
| action_expert_config.router_entropy_loss_coef, | |
| action_expert_config.router_temperature, | |
| action_expert_config.router_noise_std, | |
| action_expert_config.moe_type, | |
| action_expert_config.router_input, | |
| action_expert_config.moe_gamma, | |
| action_expert_config.router_type, | |
| action_expert_config.router_prototype_dim, | |
| action_expert_config.router_action_grad_coef, | |
| action_expert_config.routing_contrastive_loss_coef, | |
| action_expert_config.dead_expert_loss_coef, | |
| config.action_expert_router_supervised_loss_coef, | |
| config.action_expert_router_action_loss_weight_coef, | |
| action_expert_config.moe_layers, | |
| action_expert_config.moe_layer_groups, | |
| self.action_expert_uses_router_context, | |
| ) | |
| # TODO: rewrite gemma in NNX. For now, use bridge. | |
| llm = nnx_bridge.ToNNX( | |
| _gemma.Module( | |
| configs=[paligemma_config, action_expert_config], | |
| embed_dtype=config.dtype, | |
| adarms=config.pi05, | |
| ) | |
| ) | |
| llm.lazy_init(rngs=rngs, method="init", use_adarms=[False, True] if config.pi05 else [False, False]) | |
| img = nnx_bridge.ToNNX( | |
| _siglip.Module( | |
| num_classes=paligemma_config.width, | |
| variant="So400m/14", | |
| pool_type="none", | |
| scan=True, | |
| dtype_mm=config.dtype, | |
| ) | |
| ) | |
| img.lazy_init(next(iter(config.fake_obs().images.values())), train=False, rngs=rngs) | |
| self.PaliGemma = nnx.Dict(llm=llm, img=img) | |
| self.action_in_proj = nnx.Linear(config.action_dim, action_expert_config.width, rngs=rngs) | |
| if config.pi05: | |
| self.time_mlp_in = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs) | |
| self.time_mlp_out = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs) | |
| else: | |
| self.state_proj = nnx.Linear(config.action_dim, action_expert_config.width, rngs=rngs) | |
| self.action_time_mlp_in = nnx.Linear(2 * action_expert_config.width, action_expert_config.width, rngs=rngs) | |
| self.action_time_mlp_out = nnx.Linear(action_expert_config.width, action_expert_config.width, rngs=rngs) | |
| self.action_out_proj = nnx.Linear(action_expert_config.width, config.action_dim, rngs=rngs) | |
| # This attribute gets automatically set by model.train() and model.eval(). | |
| self.deterministic = True | |
| def embed_prefix( | |
| self, obs: _model.Observation | |
| ) -> tuple[at.Float[at.Array, "b s emb"], at.Bool[at.Array, "b s"], at.Bool[at.Array, " s"]]: | |
| input_mask = [] | |
| ar_mask = [] | |
| tokens = [] | |
| # embed images | |
| for name in obs.images: | |
| image_tokens, _ = self.PaliGemma.img(obs.images[name], train=False) | |
| tokens.append(image_tokens) | |
| input_mask.append( | |
| einops.repeat( | |
| obs.image_masks[name], | |
| "b -> b s", | |
| s=image_tokens.shape[1], | |
| ) | |
| ) | |
| # image tokens attend to each other | |
| ar_mask += [False] * image_tokens.shape[1] | |
| # add language (aka tokenized inputs) | |
| if obs.tokenized_prompt is not None: | |
| tokenized_inputs = self.PaliGemma.llm(obs.tokenized_prompt, method="embed") | |
| tokens.append(tokenized_inputs) | |
| input_mask.append(obs.tokenized_prompt_mask) | |
| # full attention between image and language inputs | |
| ar_mask += [False] * tokenized_inputs.shape[1] | |
| tokens = jnp.concatenate(tokens, axis=1) | |
| input_mask = jnp.concatenate(input_mask, axis=1) | |
| ar_mask = jnp.array(ar_mask) | |
| return tokens, input_mask, ar_mask | |
| def _masked_mean_pool(tokens: jnp.ndarray, mask: jnp.ndarray) -> jnp.ndarray: | |
| weights = mask.astype(tokens.dtype)[..., None] | |
| denom = jnp.clip(jnp.sum(weights, axis=1), min=1.0) | |
| return jnp.sum(tokens * weights, axis=1) / denom | |
| def _router_supervised_loss( | |
| router_metrics: dict[str, at.Array], | |
| router_label: at.Array | None, | |
| router_class_weight: at.Array | None, | |
| moe_layers: tuple[int, ...] | None = None, | |
| ) -> tuple[jnp.ndarray, jnp.ndarray]: | |
| if router_label is None or "router_route_logits" not in router_metrics: | |
| zero = jnp.array(0.0, dtype=jnp.float32) | |
| return zero, zero | |
| logits = router_metrics["router_route_logits"].astype(jnp.float32) | |
| raw_labels = jnp.asarray(router_label) | |
| weights = None if router_class_weight is None else jnp.asarray(router_class_weight, dtype=jnp.float32) | |
| if raw_labels.ndim == 2 and raw_labels.shape[-1] == logits.shape[-1]: | |
| if logits.ndim == 4: | |
| logits = jnp.mean(logits[:, :, 0, :], axis=0) | |
| elif logits.ndim == 3: | |
| logits = jnp.mean(logits[:, :, :], axis=0) | |
| if logits.ndim != 2: | |
| raise ValueError(f"Unsupported router logits shape for soft supervised loss: {logits.shape}") | |
| targets = raw_labels.astype(jnp.float32) | |
| target_mass = jnp.sum(targets, axis=-1) | |
| valid = target_mass > 0 | |
| targets = targets / jnp.clip(target_mass[:, None], a_min=1e-6) | |
| per_example = -jnp.sum(targets * jax.nn.log_softmax(logits, axis=-1), axis=-1) | |
| if weights is None: | |
| weights = jnp.ones_like(per_example, dtype=jnp.float32) | |
| weights = jnp.where(valid, weights, 0.0) | |
| loss = jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) | |
| accuracy = ( | |
| jnp.sum((jnp.argmax(logits, axis=-1) == jnp.argmax(targets, axis=-1)) * valid) | |
| / jnp.clip(jnp.sum(valid), a_min=1) | |
| ) | |
| return loss.astype(jnp.float32), accuracy.astype(jnp.float32) | |
| labels = raw_labels.astype(jnp.int32) | |
| if labels.ndim == 2: | |
| if logits.ndim != 4: | |
| raise ValueError(f"Layer router labels require per-layer logits, got {logits.shape}") | |
| if moe_layers is None: | |
| raise ValueError("Layer router labels require action_expert_moe_layers.") | |
| selected_logits = [] | |
| for layer in moe_layers: | |
| selected_logits.append(logits[layer, :, 0, :]) | |
| logits = jnp.stack(selected_logits, axis=1) | |
| if logits.shape[1] != labels.shape[1]: | |
| raise ValueError(f"Got {labels.shape[1]} label columns for {logits.shape[1]} MoE layers.") | |
| valid = labels >= 0 | |
| clipped = jnp.clip(labels, 0, logits.shape[-1] - 1) | |
| per_example = -jax.nn.log_softmax(logits, axis=-1)[ | |
| jnp.arange(labels.shape[0])[:, None], | |
| jnp.arange(labels.shape[1])[None, :], | |
| clipped, | |
| ] | |
| if weights is None: | |
| weights = jnp.ones_like(per_example, dtype=jnp.float32) | |
| weights = jnp.where(valid, weights, 0.0) | |
| loss = jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) | |
| accuracy = ( | |
| jnp.sum((jnp.argmax(logits, axis=-1) == labels) * valid) | |
| / jnp.clip(jnp.sum(valid), a_min=1) | |
| ) | |
| return loss.astype(jnp.float32), accuracy.astype(jnp.float32) | |
| if logits.ndim == 4: | |
| # Dense router stores [layers, batch, 1, experts]; prototype is the same shape. | |
| logits = jnp.mean(logits[:, :, 0, :], axis=0) | |
| elif logits.ndim == 3: | |
| logits = jnp.mean(logits[:, :, :], axis=0) | |
| if logits.ndim != 2: | |
| raise ValueError(f"Unsupported router logits shape for supervised loss: {logits.shape}") | |
| valid = labels >= 0 | |
| labels = jnp.clip(labels, 0, logits.shape[-1] - 1) | |
| per_example = -jax.nn.log_softmax(logits, axis=-1)[jnp.arange(labels.shape[0]), labels] | |
| if weights is None: | |
| weights = jnp.ones_like(per_example, dtype=jnp.float32) | |
| weights = jnp.where(valid, weights, 0.0) | |
| loss = jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) | |
| accuracy = jnp.sum((jnp.argmax(logits, axis=-1) == labels) * valid) / jnp.clip(jnp.sum(valid), a_min=1) | |
| return loss.astype(jnp.float32), accuracy.astype(jnp.float32) | |
| def _layer_router_supervised_metrics( | |
| router_metrics: dict[str, at.Array], | |
| router_label: at.Array | None, | |
| router_class_weight: at.Array | None, | |
| moe_layers: tuple[int, ...] | None, | |
| ) -> dict[str, jnp.ndarray]: | |
| if ( | |
| router_label is None | |
| or moe_layers is None | |
| or "router_route_logits" not in router_metrics | |
| ): | |
| return {} | |
| raw_labels = jnp.asarray(router_label) | |
| if raw_labels.ndim != 2: | |
| return {} | |
| logits_all = router_metrics["router_route_logits"].astype(jnp.float32) | |
| if raw_labels.shape[-1] == logits_all.shape[-1]: | |
| return {} | |
| labels = raw_labels.astype(jnp.int32) | |
| top_indices_all = router_metrics.get("router_route_top_indices") | |
| if logits_all.ndim != 4: | |
| return {} | |
| weights_all = ( | |
| jnp.ones_like(labels, dtype=jnp.float32) | |
| if router_class_weight is None | |
| else jnp.asarray(router_class_weight, dtype=jnp.float32) | |
| ) | |
| metrics = {} | |
| for col, layer in enumerate(moe_layers): | |
| logits = logits_all[layer, :, 0, :] | |
| layer_labels = labels[:, col] | |
| valid = layer_labels >= 0 | |
| clipped = jnp.clip(layer_labels, 0, logits.shape[-1] - 1) | |
| per_example = -jax.nn.log_softmax(logits, axis=-1)[jnp.arange(layer_labels.shape[0]), clipped] | |
| weights = jnp.where(valid, weights_all[:, col], 0.0) | |
| metrics[f"router_l{layer}_supervised_loss"] = ( | |
| jnp.sum(per_example * weights) / jnp.clip(jnp.sum(weights), a_min=1e-6) | |
| ).astype(jnp.float32) | |
| metrics[f"router_l{layer}_supervised_accuracy"] = ( | |
| jnp.sum((jnp.argmax(logits, axis=-1) == layer_labels) * valid) | |
| / jnp.clip(jnp.sum(valid), a_min=1) | |
| ).astype(jnp.float32) | |
| if top_indices_all is not None: | |
| top = top_indices_all[layer, :, 0, :] | |
| usage = jnp.mean(jax.nn.one_hot(top, logits.shape[-1], dtype=jnp.float32), axis=(0, 1)) | |
| for expert_idx in range(usage.shape[0]): | |
| metrics[f"router_l{layer}_usage_{expert_idx}"] = usage[expert_idx] | |
| return metrics | |
| def embed_suffix( | |
| self, obs: _model.Observation, noisy_actions: _model.Actions, timestep: at.Float[at.Array, " b"] | |
| ) -> tuple[ | |
| at.Float[at.Array, "b s emb"], | |
| at.Bool[at.Array, "b s"], | |
| at.Bool[at.Array, " s"], | |
| at.Float[at.Array, "b emb"] | None, | |
| ]: | |
| input_mask = [] | |
| ar_mask = [] | |
| tokens = [] | |
| if not self.pi05: | |
| # add a single state token | |
| state_token = self.state_proj(obs.state)[:, None, :] | |
| tokens.append(state_token) | |
| input_mask.append(jnp.ones((obs.state.shape[0], 1), dtype=jnp.bool_)) | |
| # image/language inputs do not attend to state or actions | |
| ar_mask += [True] | |
| action_tokens = self.action_in_proj(noisy_actions) | |
| # embed timestep using sine-cosine positional encoding with sensitivity in the range [0, 1] | |
| time_emb = posemb_sincos(timestep, self.action_in_proj.out_features, min_period=4e-3, max_period=4.0) | |
| if self.pi05: | |
| # time MLP (for adaRMS) | |
| time_emb = self.time_mlp_in(time_emb) | |
| time_emb = nnx.swish(time_emb) | |
| time_emb = self.time_mlp_out(time_emb) | |
| time_emb = nnx.swish(time_emb) | |
| action_expert_tokens = action_tokens | |
| adarms_cond = time_emb | |
| else: | |
| # mix timestep + action information using an MLP (no adaRMS) | |
| time_tokens = einops.repeat(time_emb, "b emb -> b s emb", s=self.action_horizon) | |
| action_time_tokens = jnp.concatenate([action_tokens, time_tokens], axis=-1) | |
| action_time_tokens = self.action_time_mlp_in(action_time_tokens) | |
| action_time_tokens = nnx.swish(action_time_tokens) | |
| action_time_tokens = self.action_time_mlp_out(action_time_tokens) | |
| action_expert_tokens = action_time_tokens | |
| adarms_cond = None | |
| tokens.append(action_expert_tokens) | |
| input_mask.append(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_)) | |
| # image/language/state inputs do not attend to action tokens | |
| ar_mask += [True] + ([False] * (self.action_horizon - 1)) | |
| tokens = jnp.concatenate(tokens, axis=1) | |
| input_mask = jnp.concatenate(input_mask, axis=1) | |
| ar_mask = jnp.array(ar_mask) | |
| return tokens, input_mask, ar_mask, adarms_cond | |
| def _prefix_router_contexts(self, prefix_out: jnp.ndarray, prefix_mask: jnp.ndarray) -> jnp.ndarray | None: | |
| if not self.action_expert_uses_router_context: | |
| return None | |
| pooled = self._masked_mean_pool(prefix_out, prefix_mask) | |
| # Old per-layer router implementation expanded this to | |
| # (depth, batch, width) so each MoE layer could consume its own context. | |
| # return jnp.broadcast_to(pooled, (self.PaliGemma.llm.module.configs[0].depth,) + pooled.shape) | |
| # The global router now consumes one shared prefix context for all layers. | |
| return pooled | |
| def compute_loss( | |
| self, rng: at.KeyArrayLike, observation: _model.Observation, actions: _model.Actions, *, train: bool = False | |
| ) -> at.Float[at.Array, "*b ah"]: | |
| loss, _ = self.compute_loss_with_info(rng, observation, actions, train=train) | |
| return loss | |
| def compute_loss_with_info( | |
| self, | |
| rng: at.KeyArrayLike, | |
| observation: _model.Observation, | |
| actions: _model.Actions, | |
| *, | |
| train: bool = False, | |
| train_step: at.Array | int | None = None, | |
| ) -> tuple[at.Float[at.Array, "*b ah"], dict[str, at.Array]]: | |
| preprocess_rng, noise_rng, time_rng, router_noise_rng = jax.random.split(rng, 4) | |
| observation = _model.preprocess_observation(preprocess_rng, observation, train=train) | |
| batch_shape = actions.shape[:-2] | |
| noise = jax.random.normal(noise_rng, actions.shape) | |
| time = jax.random.beta(time_rng, 1.5, 1, batch_shape) * 0.999 + 0.001 | |
| time_expanded = time[..., None, None] | |
| x_t = time_expanded * noise + (1 - time_expanded) * actions | |
| u_t = noise - actions | |
| prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation) | |
| prefix_attn_mask = make_attn_mask(prefix_mask, prefix_ar_mask) | |
| prefix_positions = jnp.cumsum(prefix_mask, axis=1) - 1 | |
| (prefix_out, _), _, _, _, _, _ = self.PaliGemma.llm( | |
| [prefix_tokens, None], | |
| mask=prefix_attn_mask, | |
| positions=prefix_positions, | |
| ) | |
| router_contexts = self._prefix_router_contexts(jax.lax.stop_gradient(prefix_out), prefix_mask) | |
| router_action_grad_coef = jnp.asarray(self.action_expert_router_action_grad_coef, dtype=jnp.float32) | |
| router_supervised_loss_coef = jnp.asarray(self.action_expert_router_supervised_loss_coef, dtype=jnp.float32) | |
| suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix(observation, x_t, time) | |
| input_mask = jnp.concatenate([prefix_mask, suffix_mask], axis=1) | |
| ar_mask = jnp.concatenate([prefix_ar_mask, suffix_ar_mask], axis=0) | |
| attn_mask = make_attn_mask(input_mask, ar_mask) | |
| positions = jnp.cumsum(input_mask, axis=1) - 1 | |
| (_, suffix_out), _, router_aux_loss, _, _, router_metrics = self.PaliGemma.llm( | |
| [prefix_tokens, suffix_tokens], | |
| mask=attn_mask, | |
| positions=positions, | |
| adarms_cond=[None, adarms_cond], | |
| router_contexts=router_contexts, | |
| router_noise_rng=router_noise_rng, | |
| router_action_grad_coef=router_action_grad_coef, | |
| deterministic=not train, | |
| ) | |
| v_t = self.action_out_proj(suffix_out[:, -self.action_horizon :]) | |
| action_loss = jnp.mean(jnp.square(v_t - u_t), axis=-1) | |
| if ( | |
| observation.router_class_weight is not None | |
| and self.action_expert_router_action_loss_weight_coef > 0.0 | |
| ): | |
| class_weight = jnp.asarray(observation.router_class_weight, dtype=action_loss.dtype) | |
| action_weight = 1.0 + self.action_expert_router_action_loss_weight_coef * (class_weight - 1.0) | |
| action_loss = action_loss * action_weight[:, None] | |
| weighted_router_aux_loss = self.action_expert_router_aux_loss_coef * router_aux_loss | |
| router_entropy_loss = router_metrics["router_entropy_loss"] | |
| weighted_router_entropy_loss = self.action_expert_router_entropy_loss_coef * router_entropy_loss | |
| routing_contrastive_loss = router_metrics["routing_contrastive_loss"] | |
| dead_expert_loss = router_metrics["dead_expert_loss"] | |
| router_supervised_loss, router_supervised_accuracy = self._router_supervised_loss( | |
| router_metrics, | |
| observation.router_label, | |
| observation.router_class_weight, | |
| self.action_expert_moe_layers, | |
| ) | |
| weighted_router_supervised_loss = router_supervised_loss_coef * router_supervised_loss | |
| weighted_routing_contrastive_loss = ( | |
| self.action_expert_routing_contrastive_loss_coef * routing_contrastive_loss | |
| ) | |
| weighted_dead_expert_loss = self.action_expert_dead_expert_loss_coef * dead_expert_loss | |
| total_loss = ( | |
| action_loss | |
| + weighted_router_aux_loss | |
| + weighted_router_entropy_loss | |
| + weighted_routing_contrastive_loss | |
| + weighted_dead_expert_loss | |
| + weighted_router_supervised_loss | |
| ) | |
| # Keep training logs focused on the losses that are active for supervised router training. | |
| # Uncomment these diagnostics when debugging router confidence / usage behavior. | |
| # scalar_router_metrics = { | |
| # k: v | |
| # for k, v in router_metrics.items() | |
| # if getattr(v, "ndim", 0) == 0 | |
| # } | |
| loss_info = { | |
| "action_loss": jnp.mean(action_loss), | |
| # "router_aux_loss": router_aux_loss, | |
| # "router_aux_loss_weighted": weighted_router_aux_loss, | |
| # "router_entropy_loss_weighted": weighted_router_entropy_loss, | |
| "router_supervised_loss": router_supervised_loss, | |
| "router_supervised_loss_weighted": weighted_router_supervised_loss, | |
| "router_supervised_coef": router_supervised_loss_coef, | |
| "router_supervised_accuracy": router_supervised_accuracy, | |
| "router_action_grad_coef": router_action_grad_coef, | |
| # "routing_contrastive_loss": routing_contrastive_loss, | |
| # "routing_contrastive_loss_weighted": weighted_routing_contrastive_loss, | |
| # "dead_expert_loss": dead_expert_loss, | |
| # "dead_expert_loss_weighted": weighted_dead_expert_loss, | |
| # **scalar_router_metrics, | |
| } | |
| loss_info.update( | |
| self._layer_router_supervised_metrics( | |
| router_metrics, | |
| observation.router_label, | |
| observation.router_class_weight, | |
| self.action_expert_moe_layers, | |
| ) | |
| ) | |
| if "router_usage" in router_metrics: | |
| for expert_idx in range(router_metrics["router_usage"].shape[0]): | |
| loss_info[f"router_usage_{expert_idx}"] = router_metrics["router_usage"][expert_idx] | |
| return total_loss, loss_info | |
| def sample_actions( | |
| self, | |
| rng: at.KeyArrayLike, | |
| observation: _model.Observation, | |
| *, | |
| num_steps: int | at.Int[at.Array, ""] = 10, | |
| noise: at.Float[at.Array, "b ah ad"] | None = None, | |
| ) -> _model.Actions: | |
| observation = _model.preprocess_observation(None, observation, train=False) | |
| x_0, _ = self._sample_actions_core(rng, observation, num_steps=num_steps, noise=noise, return_router_info=False) | |
| return x_0 | |
| def sample_actions_with_router_info( | |
| self, | |
| rng: at.KeyArrayLike, | |
| observation: _model.Observation, | |
| *, | |
| num_steps: int | at.Int[at.Array, ""] = 10, | |
| noise: at.Float[at.Array, "b ah ad"] | None = None, | |
| ffn_activation_topk: int = 0, | |
| ) -> tuple[_model.Actions, dict[str, at.Array]]: | |
| observation = _model.preprocess_observation(None, observation, train=False) | |
| return self._sample_actions_core( | |
| rng, | |
| observation, | |
| num_steps=num_steps, | |
| noise=noise, | |
| return_router_info=True, | |
| ffn_activation_topk=ffn_activation_topk, | |
| ) | |
| def _sample_actions_core( | |
| self, | |
| rng: at.KeyArrayLike, | |
| observation: _model.Observation, | |
| *, | |
| num_steps: int | at.Int[at.Array, ""] = 10, | |
| noise: at.Float[at.Array, "b ah ad"] | None = None, | |
| return_router_info: bool = False, | |
| ffn_activation_topk: int = 0, | |
| ) -> tuple[_model.Actions, dict[str, at.Array] | None]: | |
| # note that we use the convention more common in diffusion literature, where t=1 is noise and t=0 is the target | |
| # distribution. yes, this is the opposite of the pi0 paper, and I'm sorry. | |
| dt = -1.0 / num_steps | |
| batch_size = observation.state.shape[0] | |
| if noise is None: | |
| noise = jax.random.normal(rng, (batch_size, self.action_horizon, self.action_dim)) | |
| # first fill KV cache with a forward pass of the prefix | |
| prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation) | |
| prefix_attn_mask = make_attn_mask(prefix_mask, prefix_ar_mask) | |
| positions = jnp.cumsum(prefix_mask, axis=1) - 1 | |
| (prefix_out, _), kv_cache, _, _, _, _ = self.PaliGemma.llm( | |
| [prefix_tokens, None], | |
| mask=prefix_attn_mask, | |
| positions=positions, | |
| ) | |
| router_contexts = self._prefix_router_contexts(prefix_out, prefix_mask) | |
| def denoise_step(carry): | |
| x_t, time = carry | |
| suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix( | |
| observation, x_t, jnp.broadcast_to(time, batch_size) | |
| ) | |
| # `suffix_attn_mask` is shape (b, suffix_len, suffix_len) indicating how the suffix tokens can attend to each | |
| # other | |
| suffix_attn_mask = make_attn_mask(suffix_mask, suffix_ar_mask) | |
| # `prefix_attn_mask` is shape (b, suffix_len, prefix_len) indicating how the suffix tokens can attend to the | |
| # prefix tokens | |
| prefix_attn_mask = einops.repeat(prefix_mask, "b p -> b s p", s=suffix_tokens.shape[1]) | |
| # `combined_mask` is shape (b, suffix_len, prefix_len + suffix_len) indicating how the suffix tokens (which | |
| # generate the queries) can attend to the full prefix + suffix sequence (which generates the keys and values) | |
| full_attn_mask = jnp.concatenate([prefix_attn_mask, suffix_attn_mask], axis=-1) | |
| assert full_attn_mask.shape == ( | |
| batch_size, | |
| suffix_tokens.shape[1], | |
| prefix_tokens.shape[1] + suffix_tokens.shape[1], | |
| ) | |
| # `positions` is shape (b, suffix_len) indicating the positions of the suffix tokens | |
| positions = jnp.sum(prefix_mask, axis=-1)[:, None] + jnp.cumsum(suffix_mask, axis=-1) - 1 | |
| (prefix_out, suffix_out), _, _, router_info, _, _ = self.PaliGemma.llm( | |
| [None, suffix_tokens], | |
| mask=full_attn_mask, | |
| positions=positions, | |
| kv_cache=kv_cache, | |
| adarms_cond=[None, adarms_cond], | |
| router_contexts=router_contexts if self.action_expert_uses_router_context else None, | |
| return_router_info=return_router_info, | |
| ffn_activation_topk=ffn_activation_topk, | |
| ) | |
| if router_info is not None: | |
| router_info = { | |
| **router_info, | |
| "router_timestep": jnp.broadcast_to(time, (batch_size,)), | |
| } | |
| assert prefix_out is None | |
| v_t = self.action_out_proj(suffix_out[:, -self.action_horizon :]) | |
| return (x_t + dt * v_t, time + dt), router_info | |
| if return_router_info: | |
| def scan_step(carry, _): | |
| return denoise_step(carry) | |
| (x_0, _), router_info = jax.lax.scan(scan_step, (noise, 1.0), xs=None, length=num_steps) | |
| return x_0, router_info | |
| def while_step(carry): | |
| carry, _ = denoise_step(carry) | |
| return carry | |
| def cond(carry): | |
| x_t, time = carry | |
| # robust to floating-point error | |
| return time >= -dt / 2 | |
| x_0, _ = jax.lax.while_loop(cond, while_step, (noise, 1.0)) | |
| return x_0, None | |