beataasd's picture
Add validated inference model
92cea10 verified
Raw History Blame Contribute Delete
50.1 kB
# Copyright 2024 Big Vision Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Gemma adaptation for Pi, taken from big_vision. We follow this einsum axis naming convention: B: batch T: query length S: k/v length N: num query heads K: num k/v heads G: num query heads per k/v head H: head dim D: d_model ("features") """ from collections.abc import Sequence import dataclasses from typing import Literal, TypeAlias import einops import flax.linen as nn import jax import jax.numpy as jnp import openpi.models.lora as lora import openpi.shared.array_typing as at import openpi.training.sharding as sharding PALIGEMMA_VOCAB_SIZE = 257_152 MoeType = Literal["vanilla", "adamoe"] RouterInput = Literal["token", "pooled", "pooled_time", "token_pooled", "token_pooled_time"] RouterType = Literal["dense", "prototype"] FFNActivationTopK = int @dataclasses.dataclass class Config: width: int depth: int mlp_dim: int num_heads: int num_kv_heads: int head_dim: int lora_configs: dict[str, lora.LoRAConfig] = dataclasses.field(default_factory=dict) num_local_experts: int = 0 num_experts_per_tok: int = 2 router_aux_loss_coef: float = 0.0 router_entropy_loss_coef: float = 0.0 router_temperature: float = 1.0 router_noise_std: float = 0.0 moe_type: MoeType = "vanilla" router_input: RouterInput = "pooled" moe_gamma: float = 0.01 router_type: RouterType = "dense" router_prototype_dim: int = 512 router_action_grad_coef: float = 0.0 routing_contrastive_temperature: float = 0.07 routing_contrastive_loss_coef: float = 0.0 dead_expert_loss_coef: float = 0.0 dead_expert_min_usage: float = 0.03 moe_layers: tuple[int, ...] | None = None moe_layer_groups: tuple[tuple[int, ...], ...] | None = None Variant = Literal["dummy", "gemma_300m", "gemma_300m_lora", "gemma_2b", "gemma_2b_lora", "moe_gemma"] def get_config(variant: Variant, lora_rank: int | None = None) -> Config: """Returns config for specified gemma variant. Args: variant: Model variant name. lora_rank: LoRA rank to use for lora variants. If None, falls back to the per-variant default (gemma_2b_lora → 16, gemma_300m_lora → 32). Pass an explicit value to override (e.g. lora_rank=8) when tuning rank across experiments. alpha is always set equal to rank so scaling = 1.0. """ if variant == "dummy": return Config( width=64, depth=4, mlp_dim=128, num_heads=8, num_kv_heads=1, head_dim=16, ) if variant == "gemma_300m": # 311M params return Config( width=1024, depth=18, mlp_dim=4096, num_heads=8, num_kv_heads=1, head_dim=256, ) if variant == "gemma_2b": return Config( width=2048, depth=18, mlp_dim=16_384, num_heads=8, num_kv_heads=1, head_dim=256, ) if variant == "gemma_2b_lora": # 默认 rank=16;可通过 lora_rank 参数覆盖以便对比实验 rank = lora_rank if lora_rank is not None else 16 return Config( width=2048, depth=18, mlp_dim=16_384, num_heads=8, num_kv_heads=1, head_dim=256, lora_configs={"attn": lora.LoRAConfig(rank=rank, alpha=float(rank)), "ffn": lora.LoRAConfig(rank=rank, alpha=float(rank))}, ) if variant == "gemma_300m_lora": # 311M params;默认 rank=32;可通过 lora_rank 参数覆盖以便对比实验 rank = lora_rank if lora_rank is not None else 32 return Config( width=1024, depth=18, mlp_dim=4096, num_heads=8, num_kv_heads=1, head_dim=256, lora_configs={"attn": lora.LoRAConfig(rank=rank, alpha=float(rank)), "ffn": lora.LoRAConfig(rank=rank, alpha=float(rank))}, ) if variant == "moe_gemma": return Config( width=1024, depth=18, mlp_dim=4096, num_heads=8, num_kv_heads=1, head_dim=256, num_local_experts=4, num_experts_per_tok=2, router_aux_loss_coef=0.0, router_input="token_pooled", ) raise ValueError(f"Unknown variant: {variant}") @at.typecheck class RMSNorm(nn.Module): @nn.compact def __call__(self, x, cond): dtype = x.dtype # original dtype, could be half-precision var = jnp.mean(jnp.square(x.astype(jnp.float32)), axis=-1, keepdims=True) # compute variance in float32 normed_inputs = jnp.asarray(x * jnp.reciprocal(jnp.sqrt(var + 1e-06))) # compute normalization in float32 if cond is None: # regular RMSNorm scale = self.param("scale", nn.initializers.zeros_init(), (x.shape[-1])) normed_inputs = normed_inputs * ( 1 + scale ) # scale by learned parameter in float32 (matches Flax implementation) return normed_inputs.astype(dtype), None # return in original dtype # adaptive RMSNorm modulation = nn.Dense(x.shape[-1] * 3, kernel_init=nn.initializers.zeros, dtype=dtype)(cond) scale, shift, gate = jnp.split(modulation[:, None, :], 3, axis=-1) normed_inputs = normed_inputs * (1 + scale) + shift # scale and shift in float32 return normed_inputs.astype(dtype), gate @at.typecheck class Embedder(nn.Module): """Embedder module.""" vocab_size: int embed_dim: int def setup(self): self.input_embedding_table = self.param( "input_embedding", nn.initializers.normal(), (self.vocab_size, self.embed_dim), ) def encode(self, x): x = self.input_embedding_table[(x,)] x *= jnp.sqrt(self.embed_dim).astype(x.dtype) return x def decode(self, x): return jnp.dot(x, self.input_embedding_table.T) @at.typecheck class Attention(nn.Module): """Attention module.""" configs: Sequence[Config] @nn.compact def __call__(self, xs, positions, attn_mask, kv_cache): # all experts must share the same head dim, num heads, and num kv heads for self-attention to work assert all(config.head_dim == self.configs[0].head_dim for config in self.configs) assert all(config.num_heads == self.configs[0].num_heads for config in self.configs) assert all(config.num_kv_heads == self.configs[0].num_kv_heads for config in self.configs) dtype = next(x.dtype for x in xs if x is not None) # original dtype, could be half-precision qkvs = [] for i, (x, config) in enumerate(zip(xs, self.configs, strict=True)): if x is None: continue if config.num_kv_heads == config.num_heads: qkv_einsum = lora.Einsum( shape=(3, config.num_heads, config.width, config.head_dim), name=_name("qkv_einsum", i), init_fn=nn.initializers.lecun_normal(in_axis=-2, out_axis=-1, batch_axis=(0, 1)), lora_config=config.lora_configs.get("attn"), ) qkvs.append(qkv_einsum("BSD,3KDH->3BSKH", x)) else: q_einsum = lora.Einsum( shape=(config.num_heads, config.width, config.head_dim), name=_name("q_einsum", i), init_fn=nn.initializers.lecun_normal(in_axis=-2, out_axis=-1, batch_axis=(0,)), lora_config=config.lora_configs.get("attn"), ) q = q_einsum("BTD,NDH->BTNH", x) kv_einsum = lora.Einsum( shape=(2, config.num_kv_heads, config.width, config.head_dim), name=_name("kv_einsum", i), init_fn=nn.initializers.lecun_normal(in_axis=-2, out_axis=-1, batch_axis=(0, 1)), lora_config=config.lora_configs.get("attn"), ) k, v = kv_einsum("BSD,2KDH->2BSKH", x) qkvs.append((q, k, v)) q, k, v = (jnp.concatenate(y, axis=1) for y in zip(*qkvs, strict=True)) q = _apply_rope(q, positions=positions) q *= self.configs[0].head_dim ** -0.5 k = _apply_rope(k, positions=positions) # should still be half-precision here (if input was half-precision) assert q.dtype == k.dtype == v.dtype == dtype if kv_cache is not None: cache_k, cache_v = kv_cache k = jnp.concatenate([cache_k, k], axis=1) v = jnp.concatenate([cache_v, v], axis=1) q = einops.rearrange(q, "B T (K G) H -> B T K G H", K=self.configs[0].num_kv_heads) logits = jnp.einsum("BTKGH,BSKH->BKGTS", q, k, preferred_element_type=jnp.float32) if attn_mask.shape != (q.shape[0], 1, q.shape[1], k.shape[1]): raise ValueError( f"Attention mask with shape {attn_mask.shape} but shapes for q and k are: {q.shape} and {k.shape}" ) # big_neg = jnp.finfo(logits.dtype).min big_neg = -2.3819763e38 # See gemma/modules.py masked_logits = jnp.where(attn_mask[:, :, None, :, :], logits, big_neg) probs = jax.nn.softmax(masked_logits, axis=-1).astype(dtype) encoded = jnp.einsum("BKGTS,BSKH->BTKGH", probs, v) encoded = einops.rearrange(encoded, "B T K G H -> B T (K G) H") out = [] start = 0 for i, (x, config) in enumerate(zip(xs, self.configs, strict=True)): if x is not None: end = start + x.shape[1] out_einsum = lora.Einsum( shape=(config.num_heads, config.head_dim, config.width), name=_name("attn_vec_einsum", i), init_fn=nn.initializers.lecun_normal(in_axis=(-3, -2), out_axis=-1), lora_config=config.lora_configs.get("attn"), ) out.append(out_einsum("BTNH,NHD->BTD", encoded[:, start:end])) start = end else: out.append(None) return out, (k, v) @at.typecheck class FeedForward(nn.Module): """Feed forward module.""" features: int hidden_dim: int @nn.compact def __call__(self, x): dtype = x.dtype # original dtype, could be half-precision w_gating = self.param( "gating_einsum", nn.initializers.lecun_normal(in_axis=-2, out_axis=-1, batch_axis=(0,)), (2, self.features, self.hidden_dim), ).astype(dtype) ff_gate = jnp.dot(x, w_gating[0]) gate_value = nn.gelu(ff_gate) ff1 = jnp.dot(x, w_gating[1]) activations = gate_value * ff1 w_linear = self.param( "linear", nn.initializers.lecun_normal(in_axis=-2, out_axis=-1), (self.hidden_dim, self.features), ).astype(dtype) outputs = jnp.dot(activations, w_linear) assert outputs.dtype == dtype return outputs def _topk_routing( router_logits: jnp.ndarray, top_k: int, *, temperature: float = 1.0, action_grad_coef: float = 0.0, ) -> tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray, jnp.ndarray]: temperature = max(float(temperature), 1e-6) probs = jax.nn.softmax(router_logits / temperature, axis=-1) top_weights, top_indices = jax.lax.top_k(probs, k=top_k) one_hot = jax.nn.one_hot(top_indices, router_logits.shape[-1], dtype=router_logits.dtype) if top_k > 1: top_weights = top_weights / jnp.clip(jnp.sum(top_weights, axis=-1, keepdims=True), a_min=1e-6) combine_weights = jnp.sum(one_hot * top_weights[..., None], axis=-2) else: top_weights = jnp.ones_like(top_weights) hard_weights = jnp.sum(one_hot * top_weights[..., None], axis=-2) action_grad_coef = jnp.maximum(jnp.asarray(action_grad_coef, dtype=probs.dtype), 0.0) combine_weights = jax.lax.stop_gradient(hard_weights) + action_grad_coef * ( probs - jax.lax.stop_gradient(probs) ) return combine_weights, probs, top_indices, top_weights def _l2_normalize(x: jnp.ndarray, axis: int = -1) -> jnp.ndarray: return x * jax.lax.rsqrt(jnp.sum(jnp.square(x), axis=axis, keepdims=True) + 1e-6) def _broadcast_batch_vector(vector: jnp.ndarray, target: jnp.ndarray) -> jnp.ndarray: if vector.ndim == 3: vector = jnp.mean(vector, axis=0) return jnp.broadcast_to(vector[:, None, :], target.shape[:2] + vector.shape[-1:]) def _zeros_batch_vector_like(x: jnp.ndarray) -> jnp.ndarray: return jnp.zeros((x.shape[0], x.shape[-1]), dtype=x.dtype) def _masked_mean_pool(tokens: jnp.ndarray, mask: jnp.ndarray | None) -> jnp.ndarray: if mask is None: return jnp.mean(tokens, axis=1) weights = mask.astype(tokens.dtype)[..., None] denom = jnp.maximum(jnp.sum(weights, axis=1), 1.0) return jnp.sum(tokens * weights, axis=1) / denom def _project_router_context( router_context: jnp.ndarray | None, *, features: int, dtype: jnp.dtype, name: str, ) -> jnp.ndarray | None: if router_context is None: return None return nn.Dense(features=features, dtype=dtype, name=name)(router_context) def _build_router_input( x: jnp.ndarray, config: Config, router_context: jnp.ndarray | None, time_context: jnp.ndarray | None, ) -> jnp.ndarray: if router_context is None: router_context = _zeros_batch_vector_like(x) if time_context is None: time_context = _zeros_batch_vector_like(x) pooled_tokens = _broadcast_batch_vector(router_context.astype(x.dtype), x) time_tokens = _broadcast_batch_vector(time_context.astype(x.dtype), x) if config.router_input == "token": return x if config.router_input == "pooled": return pooled_tokens if config.router_input == "pooled_time": return jnp.concatenate([pooled_tokens, time_tokens], axis=-1) if config.router_input == "token_pooled": return jnp.concatenate([x, pooled_tokens], axis=-1) if config.router_input == "token_pooled_time": return jnp.concatenate([x, pooled_tokens, time_tokens], axis=-1) raise ValueError(f"Unsupported MoE router input: {config.router_input}") def _load_balance_loss(router_probs: jnp.ndarray, top_indices: jnp.ndarray, top_k: int) -> jnp.ndarray: num_experts = router_probs.shape[-1] selected = jax.nn.one_hot(top_indices, num_experts, dtype=router_probs.dtype) selected = jnp.sum(selected, axis=-2) tokens_per_expert = jnp.sum(selected, axis=1) * (num_experts / (router_probs.shape[1] * top_k)) prob_per_expert = jnp.mean(router_probs, axis=1) return jnp.mean(jnp.sum(tokens_per_expert * prob_per_expert, axis=-1)) def _router_entropy_loss(router_probs: jnp.ndarray) -> jnp.ndarray: entropy = -jnp.sum(router_probs * jnp.log(jnp.clip(router_probs, a_min=1e-9)), axis=-1) return jnp.mean(entropy).astype(jnp.float32) def _routing_contrastive_loss( router_embedding: jnp.ndarray, prototypes: jnp.ndarray, top_indices: jnp.ndarray, temperature: float, ) -> jnp.ndarray: num_experts = prototypes.shape[-2] top1 = top_indices[..., 0] selected = jax.nn.one_hot(top1, num_experts, dtype=router_embedding.dtype) counts = jnp.sum(selected, axis=0) cluster_means = jnp.einsum("be,bd->ed", selected, router_embedding) / jnp.clip(counts[:, None], a_min=1.0) cluster_means = _l2_normalize(cluster_means) prototypes = _l2_normalize(prototypes.astype(jnp.float32)) logits = jnp.einsum("ed,fd->ef", prototypes, cluster_means) / temperature labels = jnp.arange(num_experts) per_expert_loss = -jax.nn.log_softmax(logits, axis=-1)[labels, labels] active = counts > 0 return jnp.sum(jnp.where(active, per_expert_loss, 0.0)) / jnp.clip(jnp.sum(active), a_min=1.0) def _dead_expert_loss( top_indices: jnp.ndarray, router_probs: jnp.ndarray, num_experts: int, min_usage: float, ) -> jnp.ndarray: top1 = top_indices[..., 0] hard_usage = jnp.mean(jax.nn.one_hot(top1, num_experts, dtype=jnp.float32), axis=0) soft_usage = jnp.mean(router_probs.astype(jnp.float32), axis=0) usage = jax.lax.stop_gradient(hard_usage - soft_usage) + soft_usage return jnp.mean(jnp.square(jnp.maximum(0.0, min_usage - usage))) def _router_metrics(moe_routings: Sequence[dict[str, jnp.ndarray] | None]) -> dict[str, jnp.ndarray]: probs = [r["router_probs"].astype(jnp.float32) for r in moe_routings if r is not None] if not probs: zero = jnp.array(0.0, dtype=jnp.float32) return { "router_top1_prob": zero, "router_entropy_norm": zero, "router_top1_margin": zero, "router_usage_max": zero, "router_usage": jnp.array([], dtype=jnp.float32), } router_probs = jnp.concatenate([jnp.reshape(p, (-1, p.shape[-1])) for p in probs], axis=0) top1 = jnp.max(router_probs, axis=-1) top1_prob = jnp.mean(top1) masked_probs = jnp.where(router_probs == top1[:, None], -jnp.inf, router_probs) top2 = jnp.max(masked_probs, axis=-1) top1_margin = jnp.mean(jnp.where(jnp.isfinite(top2), top1 - top2, 0.0)) entropy = -jnp.sum(router_probs * jnp.log(jnp.clip(router_probs, a_min=1e-9)), axis=-1) entropy_denom = jnp.maximum(jnp.log(jnp.asarray(router_probs.shape[-1], dtype=jnp.float32)), 1e-9) entropy_norm = jnp.mean(entropy / entropy_denom) top_indices = [jnp.reshape(r["top_indices"], (-1, r["top_indices"].shape[-1])) for r in moe_routings if r is not None] top_indices = jnp.concatenate(top_indices, axis=0) hard_usage = jnp.mean(jax.nn.one_hot(top_indices, router_probs.shape[-1], dtype=jnp.float32), axis=(0, 1)) first_routing = next(r for r in moe_routings if r is not None) return { "router_top1_prob": top1_prob, "router_entropy_norm": entropy_norm, "router_top1_margin": top1_margin, "router_usage_max": jnp.max(hard_usage), "router_usage": hard_usage, "router_route_logits": first_routing["route_logits"], "router_route_probs": first_routing["route_probs"], "router_route_top_indices": first_routing["route_top_indices"], } def _moe_layer_mask(config: Config) -> jnp.ndarray: if config.moe_layers is None: return jnp.ones((config.depth,), dtype=bool) layer_ids = jnp.arange(config.depth) selected = jnp.asarray(config.moe_layers, dtype=jnp.int32) return jnp.any(layer_ids[:, None] == selected[None, :], axis=1) def _moe_layer_group_ids(config: Config) -> jnp.ndarray | None: if config.moe_layer_groups is None: return None ids = [-1] * config.depth for group_id, layers in enumerate(config.moe_layer_groups): for layer in layers: if not 0 <= layer < config.depth: raise ValueError(f"MoE layer group contains out-of-range layer {layer} for depth {config.depth}") if ids[layer] >= 0: raise ValueError(f"MoE layer group contains duplicate layer {layer}") ids[layer] = group_id if config.moe_layers is not None: expected = set(config.moe_layers) configured = {layer for layer, group_id in enumerate(ids) if group_id >= 0} if configured != expected: raise ValueError( f"MoE layer groups must match moe_layers: got {sorted(configured)}, expected {sorted(expected)}" ) return jnp.asarray(ids, dtype=jnp.int32) MoeRouting: TypeAlias = dict[str, jnp.ndarray] def _activation_values(activations: jnp.ndarray) -> jnp.ndarray: return activations.astype(jnp.float32) @at.typecheck class DenseMoeRouter(nn.Module): config: Config embed_dtype: str @nn.compact def __call__( self, x, router_contexts: jnp.ndarray | None, time_context: jnp.ndarray | None, router_noise_rng: jax.Array | None, deterministic: bool, router_action_grad_coef=None, ) -> MoeRouting: config = self.config num_experts = config.num_local_experts if router_action_grad_coef is None: router_action_grad_coef = config.router_action_grad_coef layer_router_context = ( nn.Dense(features=config.width, dtype=jnp.dtype(self.embed_dtype), name="context_proj")(router_contexts) if config.router_input != "token" and router_contexts is not None else None ) router_input = _build_router_input(x, config, layer_router_context, time_context) route_input = router_input[:, :1, :] if config.moe_layers is None: router_logits = nn.Dense(features=num_experts, use_bias=False, dtype=jnp.float32, name="router")(route_input) router_logits = jnp.broadcast_to(router_logits, (config.depth,) + router_logits.shape) else: layer_group_ids = _moe_layer_group_ids(config) router_count = config.depth if layer_group_ids is None else len(config.moe_layer_groups) kernel = self.param( "router_kernel", nn.initializers.lecun_normal(), (router_count, route_input.shape[-1], num_experts), jnp.float32, ) router_logits = jnp.einsum("btd,rde->rbte", route_input.astype(jnp.float32), kernel) if layer_group_ids is not None: gather_ids = jnp.clip(layer_group_ids, 0, router_count - 1) router_logits = jnp.take(router_logits, gather_ids, axis=0) if not deterministic and config.router_noise_std > 0.0: if router_noise_rng is None: raise ValueError("router_noise_rng must be provided when router noise is enabled") noise = jax.random.normal(router_noise_rng, router_logits.shape, dtype=router_logits.dtype) router_logits = router_logits + noise * config.router_noise_std top_k = config.num_experts_per_tok if config.moe_layers is None: combine_weights, router_probs, top_indices, top_weights = _topk_routing( router_logits[0], top_k, temperature=config.router_temperature, action_grad_coef=router_action_grad_coef, ) combine_weights = jnp.broadcast_to(combine_weights, x.shape[:2] + (num_experts,)) router_probs = jnp.broadcast_to(router_probs, x.shape[:2] + (num_experts,)) top_indices = jnp.broadcast_to(top_indices, x.shape[:2] + top_indices.shape[-1:]) top_weights = jnp.broadcast_to(top_weights, x.shape[:2] + top_weights.shape[-1:]) aux_loss = _load_balance_loss(router_probs, top_indices, top_k).astype(jnp.float32) entropy_loss = _router_entropy_loss(router_probs) combine_weights = jnp.broadcast_to(combine_weights, (config.depth,) + combine_weights.shape) router_probs = jnp.broadcast_to(router_probs, (config.depth,) + router_probs.shape) top_indices = jnp.broadcast_to(top_indices, (config.depth,) + top_indices.shape) top_weights = jnp.broadcast_to(top_weights, (config.depth,) + top_weights.shape) aux_loss = jnp.broadcast_to(aux_loss, (config.depth,)) entropy_loss = jnp.broadcast_to(entropy_loss, (config.depth,)) else: combine_weights, router_probs, top_indices, top_weights = jax.vmap( lambda layer_logits: _topk_routing( layer_logits, top_k, temperature=config.router_temperature, action_grad_coef=router_action_grad_coef, ) )(router_logits) combine_weights = jnp.broadcast_to(combine_weights, (config.depth,) + x.shape[:2] + (num_experts,)) router_probs = jnp.broadcast_to(router_probs, (config.depth,) + x.shape[:2] + (num_experts,)) top_indices = jnp.broadcast_to(top_indices, (config.depth,) + x.shape[:2] + top_indices.shape[-1:]) top_weights = jnp.broadcast_to(top_weights, (config.depth,) + x.shape[:2] + top_weights.shape[-1:]) aux_loss = jax.vmap(lambda p, ti: _load_balance_loss(p, ti, top_k).astype(jnp.float32))( router_probs, top_indices, ) entropy_loss = jax.vmap(_router_entropy_loss)(router_probs) if config.moe_type == "adamoe": if config.moe_layers is not None: raise ValueError("adamoe is not implemented for per-layer dense routers.") scale_logits = nn.Dense( features=num_experts, use_bias=True, dtype=jnp.float32, kernel_init=nn.initializers.zeros, bias_init=nn.initializers.zeros, name="scale_adapter", )(router_input) scale_weights = jax.nn.sigmoid(scale_logits) top_weights = top_weights * jnp.take_along_axis(scale_weights, top_indices, axis=-1) one_hot = jax.nn.one_hot(top_indices, num_experts, dtype=router_logits.dtype) combine_weights = jnp.sum(one_hot * top_weights[..., None], axis=-2) elif config.moe_type != "vanilla": raise ValueError(f"Unsupported MoE type: {config.moe_type}") layer_mask = _moe_layer_mask(config) return { "layer_mask": layer_mask, "combine_weights": combine_weights, "router_probs": router_probs, "top_indices": top_indices, "top_weights": top_weights, "route_input": jnp.broadcast_to(route_input, (config.depth,) + route_input.shape), "route_logits": router_logits, "route_probs": router_probs[:, :, :1, :], "route_top_indices": top_indices[:, :, :1, :], "route_top_weights": top_weights[:, :, :1, :], "aux_loss": aux_loss * layer_mask.astype(jnp.float32), "entropy_loss": entropy_loss * layer_mask.astype(jnp.float32), "routing_contrastive_loss": jnp.zeros((config.depth,), dtype=jnp.float32), "dead_expert_loss": jnp.zeros((config.depth,), dtype=jnp.float32), } @at.typecheck class PrototypeMoeRouter(nn.Module): config: Config @nn.compact def __call__( self, x, router_contexts: jnp.ndarray | None, time_context: jnp.ndarray | None, router_noise_rng: jax.Array | None, deterministic: bool, router_action_grad_coef=None, ) -> MoeRouting: del time_context, router_noise_rng, deterministic config = self.config num_experts = config.num_local_experts if router_action_grad_coef is None: router_action_grad_coef = config.router_action_grad_coef if router_contexts is None: raise ValueError("Prototype MoE routing requires router_contexts from the prefix encoder.") if config.num_experts_per_tok != 1: raise ValueError("Prototype MoE routing uses hard top-1 routing; set num_experts_per_tok=1.") if config.moe_type == "adamoe": raise ValueError("Prototype MoE routing does not support adamoe scale adapters.") if config.moe_type != "vanilla": raise ValueError(f"Unsupported MoE type: {config.moe_type}") route_input = nn.Dense(features=config.router_prototype_dim, dtype=jnp.float32, name="proj")(router_contexts) route_input = _l2_normalize(route_input.astype(jnp.float32)) prototypes = self.param( "prototypes", nn.initializers.normal(stddev=0.02), (config.depth, num_experts, config.router_prototype_dim), jnp.float32, ) prototypes = _l2_normalize(prototypes.astype(jnp.float32)) router_logits = jnp.einsum("bd,led->lbe", route_input, prototypes) route_probs = jax.nn.softmax(router_logits / config.routing_contrastive_temperature, axis=-1) route_top_indices = jnp.argmax(router_logits, axis=-1)[..., None] route_top_weights = jnp.ones(route_top_indices.shape, dtype=router_logits.dtype) hard_combine_weights = jax.nn.one_hot(route_top_indices[..., 0], num_experts, dtype=router_logits.dtype) action_grad_coef = jnp.maximum(jnp.asarray(router_action_grad_coef, dtype=route_probs.dtype), 0.0) combine_weights = jax.lax.stop_gradient(hard_combine_weights) + action_grad_coef * ( route_probs - jax.lax.stop_gradient(route_probs) ) contrastive_loss = jax.vmap( lambda p, ti: _routing_contrastive_loss( route_input, p, ti, config.routing_contrastive_temperature, ) )(prototypes, route_top_indices) dead_loss = jax.vmap( lambda ti, rp: _dead_expert_loss(ti, rp, num_experts, config.dead_expert_min_usage) )(route_top_indices, route_probs) route_input = jnp.broadcast_to(route_input[None, :, None, :], (config.depth, x.shape[0], 1, route_input.shape[-1])) router_logits = router_logits[:, :, None, :] route_probs = route_probs[:, :, None, :] route_top_indices = route_top_indices[:, :, None, :] route_top_weights = route_top_weights[:, :, None, :] layer_mask = _moe_layer_mask(config) return { "layer_mask": layer_mask, "combine_weights": jnp.broadcast_to( combine_weights[:, :, None, :], (config.depth, x.shape[0], x.shape[1], num_experts), ), "router_probs": jnp.broadcast_to(route_probs, (config.depth, x.shape[0], x.shape[1], num_experts)), "top_indices": jnp.broadcast_to(route_top_indices, (config.depth, x.shape[0], x.shape[1], 1)), "top_weights": jnp.broadcast_to(route_top_weights, (config.depth, x.shape[0], x.shape[1], 1)), "route_input": route_input, "route_logits": router_logits, "route_probs": route_probs, "route_top_indices": route_top_indices, "route_top_weights": route_top_weights, "aux_loss": jnp.zeros((config.depth,), dtype=jnp.float32), "entropy_loss": jnp.zeros((config.depth,), dtype=jnp.float32), "routing_contrastive_loss": contrastive_loss.astype(jnp.float32) * layer_mask.astype(jnp.float32), "dead_expert_loss": dead_loss.astype(jnp.float32) * layer_mask.astype(jnp.float32), } @at.typecheck class RoutedMoeFeedForward(nn.Module): """Token-routed extra FFN experts used as a residual branch.""" config: Config @nn.compact def __call__( self, x, routing: MoeRouting, return_router_info: bool = False, # noqa: FBT002 ffn_activation_topk: FFNActivationTopK = 0, ): num_extra_experts = self.config.num_local_experts dtype = x.dtype combine_weights = routing["combine_weights"] aux_loss = routing["aux_loss"] if self.config.moe_type not in ("vanilla", "adamoe"): raise ValueError(f"Unsupported MoE type: {self.config.moe_type}") routed_outputs = [] routed_activations = [] for expert_idx in range(num_extra_experts): expert_result = lora.FeedForward( features=self.config.width, hidden_dim=self.config.mlp_dim, name=f"expert_{expert_idx}", lora_config=self.config.lora_configs.get("ffn"), )(x, return_activations=return_router_info and ffn_activation_topk > 0) if return_router_info and ffn_activation_topk > 0: expert_out, expert_activations = expert_result routed_activations.append(_activation_values(expert_activations)) else: expert_out = expert_result routed_outputs.append(expert_out) routed_outputs = jnp.stack(routed_outputs, axis=-2) expert_diag = {} if return_router_info: expert_outputs_f32 = routed_outputs.astype(jnp.float32) expert_norms = jnp.linalg.norm(expert_outputs_f32, axis=-1) expert_unit = expert_outputs_f32 / jnp.clip(expert_norms[..., None], a_min=1e-6) expert_diag = { "expert_norms": expert_norms, "expert_cosine": jnp.einsum("bted,btfd->btef", expert_unit, expert_unit), } if ffn_activation_topk > 0: expert_diag = { **expert_diag, "expert_ffn_activations": jnp.stack(routed_activations, axis=-2), } routed = jnp.einsum("bte,bted->btd", combine_weights, routed_outputs) router_info = ( { **expert_diag, } if return_router_info else None ) return routed.astype(dtype), aux_loss, router_info @at.typecheck class Block(nn.Module): """Transformer block.""" configs: tuple[Config, ...] dropout: float = 0.0 dropout_bdims: tuple[int, ...] = () @nn.compact def __call__( self, xs, kv_cache, positions, attn_mask, adarms_cond, router_contexts, router_context_mask, moe_routings, deterministic=True, # noqa: FBT002 return_router_info: bool = False, # noqa: FBT002 ffn_activation_topk: FFNActivationTopK = 0, ): xs = sharding.activation_sharding_constraint(xs) drop = nn.Dropout(self.dropout, self.dropout_bdims) if self.dropout else lambda x, _: x attn = Attention(configs=self.configs, name="attn") pre_attn = [] gates = [] for i, x in enumerate(xs): if x is not None: x, gate = RMSNorm(name=_name("pre_attention_norm", i))(x, adarms_cond[i]) # noqa: PLW2901 pre_attn.append(x) gates.append(gate if x is not None else None) pre_attn = sharding.activation_sharding_constraint(pre_attn) post_attn, kv_cache = attn(pre_attn, positions, attn_mask, kv_cache) post_attn = jax.tree.map(lambda x: drop(x, deterministic), post_attn) post_attn = sharding.activation_sharding_constraint(post_attn) xs = [_gated_residual(x, y, gate) for x, y, gate in zip(xs, post_attn, gates, strict=True)] xs = sharding.activation_sharding_constraint(xs) out = [] gates = [] moe_aux_loss = jnp.array(0.0, dtype=jnp.float32) router_info = None for i, (x, config) in enumerate(zip(xs, self.configs, strict=True)): if x is not None: x, gate = RMSNorm(name=_name("pre_ffw_norm", i))(x, adarms_cond[i]) # noqa: PLW2901 if config.num_local_experts > 0: shared_result = lora.FeedForward( features=config.width, hidden_dim=config.mlp_dim, name=_name("mlp", i), lora_config=config.lora_configs.get("ffn"), )(x, return_activations=return_router_info and ffn_activation_topk > 0) shared_activation_topk = {} if return_router_info and ffn_activation_topk > 0: shared, shared_activations = shared_result shared_activation_topk = { "shared_ffn_activations": _activation_values(shared_activations), } else: shared = shared_result routed, aux_loss, info = RoutedMoeFeedForward(config=config, name=_name("moe", i))( x, routing=moe_routings[i], return_router_info=return_router_info, ffn_activation_topk=ffn_activation_topk, ) layer_enabled = jnp.asarray( moe_routings[i].get("layer_mask", True), dtype=shared.dtype, ) moe_aux_loss = moe_aux_loss + aux_loss routed_mix = 0.5 * shared + 0.5 * routed moe_output = shared + layer_enabled * (routed_mix - shared) if info is not None: shared_f32 = shared.astype(jnp.float32) routed_f32 = routed.astype(jnp.float32) moe_output_f32 = moe_output.astype(jnp.float32) shared_norm = jnp.linalg.norm(shared_f32, axis=-1) routed_norm = jnp.linalg.norm(routed_f32, axis=-1) moe_output_norm = jnp.linalg.norm(moe_output_f32, axis=-1) shared_routed_denom = jnp.clip(shared_norm * routed_norm, a_min=1e-6) routing = moe_routings[i] info = { **info, **shared_activation_topk, "router_input": routing["route_input"], "router_logits": routing["route_logits"], "router_probs": routing["route_probs"], "top_indices": routing["route_top_indices"], "top_weights": routing["route_top_weights"], "shared_norm": shared_norm, "routed_moe_norm": routed_norm, "moe_output_norm": moe_output_norm, "routed_moe_shared_ratio": routed_norm / jnp.clip(shared_norm, a_min=1e-6), "shared_routed_moe_cosine": jnp.sum(shared_f32 * routed_f32, axis=-1) / shared_routed_denom, } router_info = info x = moe_output # noqa: PLW2901 else: ffn_result = lora.FeedForward( features=config.width, hidden_dim=config.mlp_dim, name=_name("mlp", i), lora_config=config.lora_configs.get("ffn"), )(x, return_activations=return_router_info and ffn_activation_topk > 0) if return_router_info and ffn_activation_topk > 0: x, activations = ffn_result # noqa: PLW2901 router_info = { "ffn_activations": _activation_values(activations), } else: x = ffn_result # noqa: PLW2901 out.append(x) gates.append(gate if x is not None else None) out = sharding.activation_sharding_constraint(out) out = jax.tree.map(lambda x: drop(x, deterministic), out) xs = [_gated_residual(x, y, gate) for x, y, gate in zip(xs, out, gates, strict=True)] xs = sharding.activation_sharding_constraint(xs) return xs, (kv_cache, moe_aux_loss, router_info) KVCache: TypeAlias = tuple[at.Float[at.Array, "l b _t _k _h"], at.Float[at.Array, "l b _t _v _h"]] @at.typecheck class Module(nn.Module): """Transformer model, supporting a mixture of different weights for different tokens.""" configs: Sequence[Config] # list of configs, one for each expert embed_dtype: str dropout: float = 0.0 dropout_bdims: tuple[int, ...] = () # Every float is dropped independently. adarms: bool = False def setup(self): # all experts must have the same depth assert all(config.depth == self.configs[0].depth for config in self.configs) self.embedder = Embedder( vocab_size=PALIGEMMA_VOCAB_SIZE, embed_dim=self.configs[0].width, # embedder for first expert only name="embedder", ) self.moe_routers = [ ( DenseMoeRouter(config=config, embed_dtype=self.embed_dtype, name=_name("global_router", i)) if config.router_type == "dense" else PrototypeMoeRouter(config=config, name=_name("prototype_router", i)) ) if config.num_local_experts > 0 else None for i, config in enumerate(self.configs) ] block_cls = nn.remat( Block, prevent_cse=False, static_argnums=(9, 10, 11), # 0=self; 9/10/11 are deterministic and diagnostics flags. policy=jax.checkpoint_policies.nothing_saveable, ) self.layers = nn.scan( block_cls, variable_axes={"params": 0}, split_rngs={"params": True, "dropout": True}, in_axes=( 0, nn.broadcast, nn.broadcast, nn.broadcast, nn.broadcast, nn.broadcast, 0, nn.broadcast, nn.broadcast, nn.broadcast, ), length=self.configs[0].depth, )( configs=self.configs, dropout=self.dropout, dropout_bdims=self.dropout_bdims, ) self.final_norms = [RMSNorm(name=_name("final_norm", i)) for i in range(len(self.configs))] @at.typecheck def embed(self, tokens: at.Int[at.Array, "b t"]) -> at.Float[at.Array, "b t d"]: return self.embedder.encode(tokens).astype(self.embed_dtype) def _global_router_context( self, embedded: Sequence[jnp.ndarray | None], router_contexts: jnp.ndarray | None, router_context_mask: jnp.ndarray | None, ) -> jnp.ndarray | None: if router_contexts is not None: return router_contexts needs_router_context = any( c.num_local_experts > 0 and (c.router_input != "token" or c.router_type == "prototype") for c in self.configs ) if not needs_router_context: return None if router_context_mask is not None and embedded[0] is not None: return _masked_mean_pool(embedded[0], router_context_mask) return None def _global_moe_routings( self, embedded: Sequence[jnp.ndarray | None], router_contexts: jnp.ndarray | None, adarms_cond: Sequence[jnp.ndarray | None], router_noise_rng: jax.Array | None, deterministic: bool, router_action_grad_coef=None, ) -> tuple[list[MoeRouting | None], dict[str, jnp.ndarray]]: routings = [] for i, (x, config) in enumerate(zip(embedded, self.configs, strict=True)): if x is None or config.num_local_experts <= 0: routings.append(None) continue layer_rng = None if router_noise_rng is not None: router_noise_rng, layer_rng = jax.random.split(router_noise_rng) routings.append( self.moe_routers[i]( x, router_contexts, adarms_cond[i], layer_rng, deterministic, router_action_grad_coef, ) ) active = [r for r in routings if r is not None] zero = jnp.array(0.0, dtype=jnp.float32) extra_losses = { "router_entropy_loss": ( jnp.mean(jnp.concatenate([r["entropy_loss"].reshape(-1) for r in active])).astype(jnp.float32) if active else zero ), "routing_contrastive_loss": ( jnp.mean(jnp.concatenate([r["routing_contrastive_loss"].reshape(-1) for r in active])).astype(jnp.float32) if active else zero ), "dead_expert_loss": ( jnp.mean(jnp.concatenate([r["dead_expert_loss"].reshape(-1) for r in active])).astype(jnp.float32) if active else zero ), } return routings, extra_losses @at.typecheck def __call__( self, # list of token arrays, one for each expert, or None if that expert should not be run embedded: Sequence[at.Float[at.Array, "b _t _d"] | None], positions: at.Int[at.Array, "b t"], mask: at.Bool[at.Array, "b t s"], adarms_cond: Sequence[at.Float[at.Array, "b _d"] | None] | None = None, *, kv_cache: KVCache | None = None, router_contexts: at.Float[at.Array, "b _d"] | None = None, router_context_mask: at.Bool[at.Array, "b _t"] | None = None, router_noise_rng: at.KeyArrayLike | None = None, router_action_grad_coef=None, deterministic: bool = True, return_router_info: bool = False, ffn_activation_topk: FFNActivationTopK = 0, ) -> tuple[ Sequence[at.Float[at.Array, "b _t _d"] | None], KVCache, at.Float[at.Array, ""], dict[str, at.Array] | None, at.Float[at.Array, "b _d"] | None, dict[str, at.Array], ]: embedded = jax.tree.map(lambda e: e.astype(self.embed_dtype), embedded) mask = jnp.asarray(mask)[:, None, :, :] if adarms_cond is None: adarms_cond = [None] * len(self.configs) router_contexts = self._global_router_context(embedded, router_contexts, router_context_mask) moe_routings, router_extra_losses = self._global_moe_routings( embedded, router_contexts, adarms_cond, router_noise_rng, deterministic, router_action_grad_coef, ) router_metrics = _router_metrics(moe_routings) embedded, (kv_cache, moe_aux_losses, router_info) = self.layers( embedded, kv_cache, positions, mask, adarms_cond, router_contexts, router_context_mask, moe_routings, deterministic, return_router_info, ffn_activation_topk, ) moe_aux_loss = jnp.mean(moe_aux_losses) # 现在每block返回的损失是一致的,所以mean之后相当于一份 assert all(e.dtype == jnp.dtype(self.embed_dtype) for e in embedded if e is not None) router_metrics = { **router_metrics, **router_extra_losses, } return [ f(e, a)[0] if e is not None else e for f, e, a in zip(self.final_norms, embedded, adarms_cond, strict=True) ], kv_cache, moe_aux_loss, router_info, router_contexts, router_metrics def init(self, use_adarms: Sequence[bool]): """Convenience method for initializing all parameters, necessary due to the quirks of linen.""" self.embed(jnp.zeros((1, 1), dtype=jnp.int32)) needs_router_context = any( c.num_local_experts > 0 and (c.router_input != "token" or c.router_type == "prototype") for c in self.configs ) self( [jnp.zeros((1, 1, c.width)) for c in self.configs], jnp.zeros((1, len(self.configs)), dtype=jnp.int32), jnp.zeros((1, len(self.configs), len(self.configs)), dtype=bool), adarms_cond=[jnp.zeros((1, c.width)) if u else None for u, c in zip(use_adarms, self.configs, strict=True)], router_context_mask=jnp.ones((1, 1), dtype=bool) if needs_router_context else None, ) def _apply_rope(x, *, positions, max_wavelength=10_000): """Applies RoPE positions [B, L] to x [B, L, H, D].""" freq_exponents = (2.0 / x.shape[-1]) * jnp.arange(x.shape[-1] // 2, dtype=jnp.float32) timescale = max_wavelength**freq_exponents radians = positions[..., None] / timescale[None, None, :] radians = radians[..., None, :] assert radians.dtype == jnp.float32 # radians.shape = [...,L,1,d=D/2] sin, cos = jnp.sin(radians), jnp.cos(radians) x1, x2 = jnp.split(x, 2, axis=-1) res = jnp.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1) assert res.dtype == jnp.float32 # The original bigvision impl allows RoPE to upcast to float32. It is then immediately downcast again to the cache # dtype when in inference mode (but not in training mode). I don't think any of this was intentional. Based on the # original DeepMind impl, as well as the widely-used transformers impl, it is ok to always downcast back to bfloat16 # here. return res.astype(x.dtype) def _name(name, i): # we name layers like this because we want the first expert's weights to have no suffix (e.g., "attn"), so that they # can be loaded seamlessly from the existing PaliGemma checkpoint. subsequent experts will have a suffix (e.g., # "attn_1") and their weights will be initialized from scratch. in practice, we only use two experts -- PaliGemma, # and the action expert. if i == 0: return name return f"{name}_{i}" def _gated_residual(x, y, gate): assert (x is None) == (y is None) if x is None: return None if gate is None: return x + y return x + y * gate
# Copyright 2024 Big Vision Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Gemma adaptation for Pi, taken from big_vision.
We follow this einsum axis naming convention:
B: batch
T: query length
S: k/v length
N: num query heads
K: num k/v heads
G: num query heads per k/v head
H: head dim
D: d_model ("features")
"""
from collections.abc import Sequence
import dataclasses
from typing import Literal, TypeAlias
import einops
import flax.linen as nn
import jax
import jax.numpy as jnp
import openpi.models.lora as lora
import openpi.shared.array_typing as at
import openpi.training.sharding as sharding
PALIGEMMA_VOCAB_SIZE = 257_152
MoeType = Literal["vanilla", "adamoe"]
RouterInput = Literal["token", "pooled", "pooled_time", "token_pooled", "token_pooled_time"]
RouterType = Literal["dense", "prototype"]
FFNActivationTopK = int
@dataclasses.dataclass
class Config:
width: int
depth: int
mlp_dim: int
num_heads: int
num_kv_heads: int
head_dim: int
lora_configs: dict[str, lora.LoRAConfig] = dataclasses.field(default_factory=dict)
num_local_experts: int = 0
num_experts_per_tok: int = 2
router_aux_loss_coef: float = 0.0
router_entropy_loss_coef: float = 0.0
router_temperature: float = 1.0
router_noise_std: float = 0.0
moe_type: MoeType = "vanilla"
router_input: RouterInput = "pooled"
moe_gamma: float = 0.01
router_type: RouterType = "dense"
router_prototype_dim: int = 512
router_action_grad_coef: float = 0.0
routing_contrastive_temperature: float = 0.07
routing_contrastive_loss_coef: float = 0.0
dead_expert_loss_coef: float = 0.0
dead_expert_min_usage: float = 0.03
moe_layers: tuple[int, ...] | None = None
moe_layer_groups: tuple[tuple[int, ...], ...] | None = None
Variant = Literal["dummy", "gemma_300m", "gemma_300m_lora", "gemma_2b", "gemma_2b_lora", "moe_gemma"]
def get_config(variant: Variant, lora_rank: int | None = None) -> Config:
"""Returns config for specified gemma variant.
Args:
variant: Model variant name.
lora_rank: LoRA rank to use for lora variants. If None, falls back to the
per-variant default (gemma_2b_lora → 16, gemma_300m_lora → 32).
Pass an explicit value to override (e.g. lora_rank=8) when tuning rank
across experiments. alpha is always set equal to rank so scaling = 1.0.
"""
if variant == "dummy":
return Config(
width=64,
depth=4,
mlp_dim=128,
num_heads=8,
num_kv_heads=1,
head_dim=16,
)
if variant == "gemma_300m":