Download task 57/policy_code/src/b1k/models/observation.py from Luke-Xing/task57b1k: direct link, hf CLI and curl.
- Browser
- Download file 5.15 kB
-
https://huggingface.co/Luke-Xing/task57b1k/resolve/main/task%2057/policy_code/src/b1k/models/observation.py
- Command line
-
hf download 'hf://Luke-Xing/task57b1k/task 57/policy_code/src/b1k/models/observation.py'
-
curl -L -o observation.py https://huggingface.co/Luke-Xing/task57b1k/resolve/main/task%2057/policy_code/src/b1k/models/observation.py
5.15 kB
| """Observation class and preprocessing with FAST auxiliary fields support. | |
| Based on openpi with FAST fields added for PI_BEHAVIOR model. | |
| Reference: https://github.com/wensi-ai/openpi/blob/behavior/src/openpi/models/model.py | |
| """ | |
| from collections.abc import Sequence | |
| from typing import Generic, TypeVar | |
| import dataclasses | |
| import augmax | |
| from flax import struct | |
| import jax | |
| import jax.numpy as jnp | |
| import numpy as np | |
| import torch | |
| from openpi.shared import image_tools | |
| from openpi.shared import array_typing as at | |
| ArrayT = TypeVar("ArrayT", bound=jax.Array | torch.Tensor | np.ndarray) | |
| IMAGE_KEYS = ( | |
| "base_0_rgb", | |
| "left_wrist_0_rgb", | |
| "right_wrist_0_rgb", | |
| ) | |
| IMAGE_RESOLUTION = (224, 224) | |
| class Observation(Generic[ArrayT]): | |
| """Observation with FAST auxiliary fields.""" | |
| images: dict[str, at.Float[ArrayT, "*b h w c"]] | |
| image_masks: dict[str, at.Bool[ArrayT, "*b"]] | |
| state: at.Float[ArrayT, "*b s"] | |
| tokenized_prompt: at.Int[ArrayT, "*b l"] | None = None | |
| tokenized_prompt_mask: at.Bool[ArrayT, "*b l"] | None = None | |
| token_ar_mask: at.Int[ArrayT, "*b l"] | None = None | |
| token_loss_mask: at.Bool[ArrayT, "*b l"] | None = None | |
| fast_tokens: at.Int[ArrayT, "*b t"] | None = None | |
| fast_token_mask: at.Bool[ArrayT, "*b t"] | None = None | |
| def from_dict(cls, data: at.PyTree[ArrayT]) -> "Observation[ArrayT]": | |
| """Convert dict to Observation.""" | |
| if ("tokenized_prompt" in data) != ("tokenized_prompt_mask" in data): | |
| raise ValueError("tokenized_prompt and tokenized_prompt_mask must be provided together.") | |
| # Convert uint8 images to float32 [-1, 1] | |
| for key in data["image"]: | |
| if data["image"][key].dtype == np.uint8: | |
| data["image"][key] = data["image"][key].astype(np.float32) / 255.0 * 2.0 - 1.0 | |
| elif hasattr(data["image"][key], "dtype") and data["image"][key].dtype == torch.uint8: | |
| data["image"][key] = data["image"][key].to(torch.float32).permute(0, 3, 1, 2) / 255.0 * 2.0 - 1.0 | |
| return cls( | |
| images=data["image"], | |
| image_masks=data["image_mask"], | |
| state=data["state"], | |
| tokenized_prompt=data.get("tokenized_prompt"), | |
| tokenized_prompt_mask=data.get("tokenized_prompt_mask"), | |
| token_ar_mask=data.get("token_ar_mask"), | |
| token_loss_mask=data.get("token_loss_mask"), | |
| fast_tokens=data.get("fast_tokens"), | |
| fast_token_mask=data.get("fast_token_mask"), | |
| ) | |
| def to_dict(self) -> at.PyTree[ArrayT]: | |
| """Convert Observation to dict.""" | |
| result = dataclasses.asdict(self) | |
| result["image"] = result.pop("images") | |
| result["image_mask"] = result.pop("image_masks") | |
| return result | |
| def preprocess_observation( | |
| rng: at.KeyArrayLike | None, | |
| observation: Observation, | |
| *, | |
| train: bool = False, | |
| image_keys: Sequence[str] = IMAGE_KEYS, | |
| image_resolution: tuple[int, int] = IMAGE_RESOLUTION, | |
| ) -> Observation: | |
| """Preprocess observations with image augmentation and FAST fields preservation.""" | |
| if not set(image_keys).issubset(observation.images): | |
| raise ValueError(f"images dict missing keys: expected {image_keys}, got {list(observation.images)}") | |
| batch_shape = observation.state.shape[:-1] | |
| out_images = {} | |
| for key in image_keys: | |
| image = observation.images[key] | |
| if image.shape[1:3] != image_resolution: | |
| image = image_tools.resize_with_pad(image, *image_resolution) | |
| if train: | |
| # Convert from [-1, 1] to [0, 1] for augmax | |
| image = image / 2.0 + 0.5 | |
| transforms = [] | |
| if "wrist" not in key: | |
| height, width = image.shape[1:3] | |
| transforms += [ | |
| augmax.RandomCrop(int(width * 0.95), int(height * 0.95)), | |
| augmax.Resize(width, height), | |
| augmax.Rotate((-5, 5)), | |
| ] | |
| transforms += [ | |
| augmax.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.5), | |
| ] | |
| sub_rngs = jax.random.split(rng, image.shape[0]) | |
| image = jax.vmap(augmax.Chain(*transforms))(sub_rngs, image) | |
| # Back to [-1, 1] | |
| image = image * 2.0 - 1.0 | |
| out_images[key] = image | |
| # Obtain masks | |
| out_masks = {} | |
| for key in out_images: | |
| if key not in observation.image_masks: | |
| out_masks[key] = jnp.ones(batch_shape, dtype=jnp.bool) | |
| else: | |
| out_masks[key] = jnp.asarray(observation.image_masks[key]) | |
| return Observation( | |
| images=out_images, | |
| image_masks=out_masks, | |
| state=observation.state, | |
| tokenized_prompt=observation.tokenized_prompt, | |
| tokenized_prompt_mask=observation.tokenized_prompt_mask, | |
| token_ar_mask=observation.token_ar_mask, | |
| token_loss_mask=observation.token_loss_mask, | |
| fast_tokens=getattr(observation, 'fast_tokens', None), | |
| fast_token_mask=getattr(observation, 'fast_token_mask', None), | |
| ) | |