Add N0-VTLA UniVTAC single-arm policy

README.md

@@ -0,0 +1,70 @@
+---
+license: other
+license_name: gemma-terms-of-use
+license_link: https://ai.google.dev/gemma/terms
+library_name: n0vtla
+tags:
+  - robotics
+  - vision-language-action
+  - tactile
+  - simulation
+---
+
+# 𝒩₀-VTLA — UniVTAC single-arm
+
+A 𝒩₀-VTLA single-arm policy for the [UniVTAC](https://github.com/anlorla/UniVTAC) simulator,
+post-trained from the [𝒩₀-VTLA pretrained base](https://huggingface.co/NeoteAI/n0-vtla-base).
+This card documents the `insert_hole` setup.
+
+| | |
+|---|---|
+| Embodiment | single-arm |
+| Action space | joint, 8-dim (7 joints + gripper) |
+| Action horizon | 50 |
+| Tactile views | 2 |
+| Latent tactile tokens | 5 |
+
+## Files
+
+```
+model.safetensors                          the full checkpoint (~8.25 GB)
+assets/n0_insert_hole_norm/norm_stats.json
+config.json                                architecture summary
+```
+
+## Serve
+
+The evaluation adapter speaks ZMQ + msgpack, so use `scripts/serve_zmq.py` from the
+[code repository](https://github.com/neoteai/N0-VTLA):
+
+```bash
+hf download NeoteAI/n0_VTLA_insert_hole --local-dir checkpoints/n0_VTLA_insert_hole
+
+VTLA_ASSET_ID=n0_insert_hole_norm python scripts/serve_zmq.py \
+  --config sim_single_arm_tactile \
+  --ckpt checkpoints/n0_VTLA_insert_hole \
+  --addr "tcp://*:5557" \
+  --default-prompt "insert hole"
+```
+
+Serving reads only `model.safetensors` and `assets/<asset-id>/norm_stats.json`; no dataset is
+needed. A correct load prints `tactile=True` with 2 views and no missing or unexpected
+state-dict keys. If it reports either, the config does not match the checkpoint.
+
+Then run the evaluation from a UniVTAC checkout with a deploy YAML pointing at port 5557. Set
+`exec_horizon` to 50, the model's action horizon: executing fewer steps clips the tail of each
+chunk, which is where the grasp-closing motion lives.
+
+## Action space
+
+Unlike the pretrained base, which predicts end-effector deltas in a canonical 32-dim rot6d
+container, this policy predicts **joint** actions: 7 joints plus gripper. The joint dims are
+element-wise deltas against the current state; the gripper column is absolute. Do not feed it
+end-effector data or reuse an end-effector normalization asset.
+
+## License
+
+These weights are derived from Google's PaliGemma/Gemma parameters and are made available under
+the [Gemma Terms of Use](https://ai.google.dev/gemma/terms) and the
+[Gemma Prohibited Use Policy](https://ai.google.dev/gemma/prohibited_use_policy), not under the
+Apache-2.0 licence that covers the source code. This is inherited from the base model.

assets/n0_insert_hole_norm/norm_stats.json

@@ -0,0 +1,88 @@
+{
+  "norm_stats": {
+    "state": {
+      "mean": [
+        0.06630285829305649,
+        0.23359940946102142,
+        -0.06392768770456314,
+        -2.292025089263916,
+        -0.04158048331737518,
+        2.870220422744751,
+        0.8207927942276001,
+        0.008305604569613934
+      ],
+      "std": [
+        0.41193342208862305,
+        0.3409915566444397,
+        0.3874863088130951,
+        0.21225132048130035,
+        0.7690718173980713,
+        0.391698956489563,
+        0.6687915921211243,
+        0.00635269470512867
+      ],
+      "q01": [
+        -1.1231915211677552,
+        -1.090289604127407,
+        -1.5997005356788634,
+        -2.851706860399246,
+        -2.6326723899841307,
+        2.144908622741699,
+        -1.9642710106372834,
+        0.0006780616328120231
+      ],
+      "q99": [
+        1.7049519143104552,
+        0.6554751669764518,
+        1.1072352698802947,
+        -1.8351004985332489,
+        2.8525589485168457,
+        3.6928259750366212,
+        2.8887114732742303,
+        0.031380061611533165
+      ]
+    },
+    "actions": {
+      "mean": [
+        0.004890867974609137,
+        0.0018891234649345279,
+        -0.004484705626964569,
+        0.008738973177969456,
+        -0.00618957681581378,
+        0.019451549276709557,
+        -0.013864942826330662,
+        0.008818385191261768
+      ],
+      "std": [
+        0.08330520987510681,
+        0.05834702029824257,
+        0.07885386049747467,
+        0.05567995086312294,
+        0.14548058807849884,
+        0.13775849342346191,
+        0.15702897310256958,
+        0.0073010860942304134
+      ],
+      "q01": [
+        -0.21749016952514644,
+        -0.20817201356887804,
+        -0.3077045212745667,
+        -0.2091159963607788,
+        -0.43886827497482295,
+        -0.13781279792785645,
+        -0.7047383863449097,
+        0.0006780616328120231
+      ],
+      "q99": [
+        0.31995077896118174,
+        0.16927316443324103,
+        0.19594545202255254,
+        0.16887029111385343,
+        0.1517598106384277,
+        0.6878627071380616,
+        0.32376871776580796,
+        0.03923296145200729
+      ]
+    }
+  }
+}

config.json

@@ -0,0 +1,27 @@
+{
+  "_note": "N0-VTLA UniVTAC single-arm policy. Load with the sim_single_arm_tactile config in https://github.com/neoteai/N0-VTLA",
+  "type": "n0vtla",
+  "model_class": "n0vtla.models_pytorch.n0vtla_policy.N0VTLAPolicy",
+  "config_name": "sim_single_arm_tactile",
+  "pi05": true,
+  "paligemma_variant": "gemma_2b",
+  "action_expert_variant": "gemma_300m",
+  "action_dim": 32,
+  "action_horizon": 50,
+  "tactile_predictor_enabled": true,
+  "tactile_mode": "latent",
+  "n_latent": 5,
+  "predictor_arch": "tactile_kv",
+  "z_gate_zero_init": false,
+  "tactile_encoder": "frozen DINOv2 (facebook/dinov2-base)",
+  "tactile_image_keys": [
+    "tactile_a",
+    "tactile_b"
+  ],
+  "embodiment": "single-arm",
+  "action_space": "joint",
+  "raw_action_dim": 8,
+  "action_representation": "element-wise joint delta, gripper absolute",
+  "norm_stats_asset_id": "n0_insert_hole_norm",
+  "n_execute_at_serve": 50
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a66b9d914122e21274c9c6ae28a0e451a7dae072080377b75e09b12c4ea235e3
+size 8253873200