Download task 57/policy_code/src/b1k/models/pi_behavior_config.py from Luke-Xing/task57b1k: direct link, hf CLI and curl.
- Browser
- Download file 7.11 kB
-
https://huggingface.co/Luke-Xing/task57b1k/resolve/main/task%2057/policy_code/src/b1k/models/pi_behavior_config.py
- Command line
-
hf download 'hf://Luke-Xing/task57b1k/task 57/policy_code/src/b1k/models/pi_behavior_config.py'
-
curl -L -o pi_behavior_config.py https://huggingface.co/Luke-Xing/task57b1k/resolve/main/task%2057/policy_code/src/b1k/models/pi_behavior_config.py
7.11 kB
| """PI_BEHAVIOR Model Configuration | |
| Configuration for PI_BEHAVIOR model on BEHAVIOR-1K challenge. | |
| """ | |
| import dataclasses | |
| import json | |
| import pathlib | |
| from typing import TYPE_CHECKING | |
| import flax.nnx as nnx | |
| import jax | |
| import jax.numpy as jnp | |
| from typing_extensions import override | |
| from openpi.models import model as _model | |
| from openpi.models import gemma as _gemma | |
| from openpi.shared import array_typing as at | |
| import openpi.shared.nnx_utils as nnx_utils | |
| from b1k.models.observation import Observation | |
| if TYPE_CHECKING: | |
| from b1k.models.pi_behavior import PiBehavior | |
| # Per-task stage counts (based on avg_episode_length / 900, capped between 5-15) | |
| # Use tuple for immutability and to avoid JAX device allocation at import time | |
| TASK_NUM_STAGES = ( | |
| 5, 6, 15, 15, 14, 12, 9, 15, 10, 15, # Tasks 0-9 | |
| 7, 13, 10, 15, 15, 15, 15, 11, 13, 12, # Tasks 10-19 | |
| 14, 15, 9, 15, 15, 15, 15, 15, 15, 15, # Tasks 20-29 | |
| 11, 10, 10, 13, 5, 5, 14, 6, 8, 10, # Tasks 30-39 | |
| 5, 15, 8, 15, 12, 11, 9, 14, 15, 15, # Tasks 40-49 | |
| ) | |
| MAX_NUM_STAGES = 15 # Maximum stages per task | |
| TOTAL_TASK_STAGE_EMBEDDINGS = sum(TASK_NUM_STAGES) # 596 total embeddings | |
| # Cumulative offsets for indexing into task_stage_embeddings (as tuple) | |
| TASK_STAGE_OFFSETS = tuple([0] + [sum(TASK_NUM_STAGES[:i+1]) for i in range(len(TASK_NUM_STAGES) - 1)]) | |
| class PiBehaviorConfig(_model.BaseModelConfig): | |
| dtype: str = "bfloat16" | |
| paligemma_variant: _gemma.Variant = "gemma_2b" | |
| action_expert_variant: _gemma.Variant = "gemma_300m" | |
| # Set the model specific defaults. | |
| action_dim: int = 32 | |
| action_horizon: int = 30 | |
| max_token_len: int = 200 # Only used for compatibility, not for actual tokenization | |
| # Number of tasks in the behavior dataset | |
| num_tasks: int = 50 | |
| # Task embedding dimension - will match the paligemma width | |
| task_embedding_dim: int = None # type: ignore | |
| # Maximum number of subtask states across all tasks | |
| max_num_subtask_states: int = MAX_NUM_STAGES | |
| # Path to task data JSON file for initialization | |
| task_data_path: str = "b1k/BEHAVIOR-1K/docs/challenge/task_data.json" | |
| # Whether to use correlated noise matching action covariance structure | |
| # Requires correlation matrix in norm_stats (computed by compute_norm_stats.py) | |
| use_correlated_noise: bool = True | |
| # Shrinkage parameter for correlation regularization | |
| # Applied as: S_regularized = beta * S + (1-beta) * I | |
| # beta=1.0 means full correlation (no shrinkage) | |
| # beta=0.7 means 70% correlation + 30% independence (recommended for robustness) | |
| # beta=0.0 means independence (no correlation) | |
| correlation_beta: float = 0.5 | |
| # FAST auxiliary training configuration | |
| use_fast_auxiliary: bool = False # Enable FAST during training | |
| fast_loss_weight: float = 0.1 # Weight for FAST loss (vs flow loss) | |
| # Action dimensions to encode with FAST (default: 0:6, 7:23 = 22 dims) | |
| # Format: "0:6,7:23" or list of tuples [(0, 6), (7, 23)] | |
| fast_encoded_dims: str | list[tuple[int, int]] = "0:6,7:23" | |
| # FAST tokenizer vocab size | |
| fast_vocab_size: int = 1024 | |
| # Max FAST tokens to predict (truncate if exceeded) | |
| max_fast_tokens: int = 32 | |
| # FAST tokenizer path (set during initialization, relative to assets_dir/asset_id) | |
| fast_tokenizer_path: str | None = None | |
| # KV cache transformation for cross-layer attention between VLM and action expert | |
| # Allows each action expert layer to attend to a learned combination of all VLM layers | |
| use_kv_transform: bool = True | |
| # Knowledge insulation: stop action expert gradients from flowing to VLM backbone | |
| # VLM trains on FAST tokens only, action expert on flow matching with frozen VLM features | |
| # Implements approach from https://www.physicalintelligence.company/research/knowledge_insulation | |
| use_knowledge_insulation: bool = True | |
| # Subtask/stage prediction auxiliary loss weight (relative to action loss) | |
| # Higher values emphasize stage prediction accuracy at the expense of action quality | |
| subtask_loss_weight: float = 0.1 | |
| # Time threshold for inpainting during inference | |
| # Stop enforcing inpainting constraint when t < threshold (let model be free in final steps) | |
| time_threshold_inpaint: float = 0.3 | |
| # Vision backbone finetuning control | |
| freeze_vision_backbone: bool = True | |
| def __post_init__(self): | |
| if self.task_embedding_dim is None: | |
| paligemma_config = _gemma.get_config(self.paligemma_variant) | |
| object.__setattr__(self, "task_embedding_dim", paligemma_config.width) | |
| def get_fast_dim_ranges(self) -> list[tuple[int, int]]: | |
| """Parse fast_encoded_dims into list of ranges.""" | |
| if isinstance(self.fast_encoded_dims, str): | |
| ranges = [] | |
| for range_str in self.fast_encoded_dims.split(','): | |
| start, end = map(int, range_str.strip().split(':')) | |
| ranges.append((start, end)) | |
| return ranges | |
| return self.fast_encoded_dims | |
| def get_total_fast_dims(self) -> int: | |
| """Get total number of dimensions encoded by FAST.""" | |
| return sum(end - start for start, end in self.get_fast_dim_ranges()) | |
| def model_type(self): | |
| return "pi_behavior" | |
| def create(self, rng: at.KeyArrayLike) -> "PiBehavior": | |
| from b1k.models.pi_behavior import PiBehavior | |
| return PiBehavior(self, rngs=nnx.Rngs(rng)) | |
| def inputs_spec(self, *, batch_size: int = 1) -> tuple["Observation", _model.Actions]: | |
| image_spec = jax.ShapeDtypeStruct([batch_size, *_model.IMAGE_RESOLUTION, 3], jnp.float32) | |
| image_mask_spec = jax.ShapeDtypeStruct([batch_size], jnp.bool_) | |
| with at.disable_typechecking(): | |
| obs_kwargs = { | |
| "images": { | |
| "base_0_rgb": image_spec, | |
| "left_wrist_0_rgb": image_spec, | |
| "right_wrist_0_rgb": image_spec, | |
| }, | |
| "image_masks": { | |
| "base_0_rgb": image_mask_spec, | |
| "left_wrist_0_rgb": image_mask_spec, | |
| "right_wrist_0_rgb": image_mask_spec, | |
| }, | |
| "state": jax.ShapeDtypeStruct([batch_size, self.action_dim], jnp.float32), | |
| "tokenized_prompt": jax.ShapeDtypeStruct([batch_size, 2], jnp.int32), | |
| "tokenized_prompt_mask": jax.ShapeDtypeStruct([batch_size, 2], bool), | |
| } | |
| if self.use_fast_auxiliary: | |
| obs_kwargs["fast_tokens"] = jax.ShapeDtypeStruct([batch_size, self.max_fast_tokens], jnp.int32) | |
| obs_kwargs["fast_token_mask"] = jax.ShapeDtypeStruct([batch_size, self.max_fast_tokens], bool) | |
| observation_spec = Observation(**obs_kwargs) | |
| action_spec = jax.ShapeDtypeStruct([batch_size, self.action_horizon, self.action_dim], jnp.float32) | |
| return observation_spec, action_spec |