update

action_head.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5971a6c6f805bf4c622a30fc3a127f85011577eb297b670ad6491f09d64d7fc0
-size 109199860
+oid sha256:70e39851fc59d264813679e439f4efceacacb484c5fcd6ded694400bc503dad4
+size 37822956

flock_robotics_adapter.py

@@ -13,16 +13,50 @@ from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
 ACTION_DIM = 7
 
 
+def format_user_text(obs: dict) -> str:
+    """Build the text the VLM sees — must match training."""
+    instruction = str(obs.get("instruction", "")).strip()
+    task = str(obs.get("task", "")).strip()
+    difficulty = str(obs.get("difficulty", "")).strip()
+    step = obs.get("step", obs.get("step_index", ""))
+    horizon = obs.get("horizon", "")
+    parts = [instruction] if instruction else []
+    meta = []
+    if task:
+        meta.append(f"task={task}")
+    if difficulty:
+        meta.append(f"difficulty={difficulty}")
+    if step != "" and step is not None:
+        meta.append(f"step={step}" + (f"/{horizon}" if horizon not in ("", None) else ""))
+    if meta:
+        parts.append(" ".join(meta))
+    proprio = obs.get("proprio")
+    if proprio is not None:
+        arr = np.asarray(proprio, dtype=np.float32).reshape(-1)
+        # Compact fixed-width floats keep tokenizer length stable
+        props = " ".join(f"{x:.3f}" for x in arr.tolist())
+        parts.append(f"proprio: {props}")
+    return "\n".join(parts) if parts else "act"
+
+
 def build_action_head(hidden_size, head_hidden):
     return nn.Sequential(
         nn.Linear(hidden_size, head_hidden),
         nn.GELU(),
+        nn.Dropout(0.05),
         nn.Linear(head_hidden, head_hidden),
         nn.GELU(),
         nn.Linear(head_hidden, ACTION_DIM),
     )
 
 
+def last_token_hidden(hidden, attention_mask):
+    # hidden: [B, T, H], mask: [B, T]
+    idx = attention_mask.long().sum(dim=1).clamp(min=1) - 1
+    b = torch.arange(hidden.size(0), device=hidden.device)
+    return hidden[b, idx]
+
+
 class QwenVLAPolicy:
     def __init__(self, model, processor, head, device):
         self.model = model
@@ -35,14 +69,14 @@ class QwenVLAPolicy:
     @torch.no_grad()
     def act(self, obs):
         image = np.asarray(obs["image"], dtype=np.uint8)
-        instruction = str(obs.get("instruction", ""))
         pil = Image.fromarray(image)
+        user_text = format_user_text(obs)
         messages = [
             {
                 "role": "user",
                 "content": [
                     {"type": "image"},
-                    {"type": "text", "text": instruction},
+                    {"type": "text", "text": user_text},
                 ],
             }
         ]
@@ -55,8 +89,7 @@ class QwenVLAPolicy:
         inputs = {k: v.to(self.device) for k, v in inputs.items()}
         out = self.model(**inputs, output_hidden_states=True, use_cache=False)
         hidden = out.hidden_states[-1]
-        mask = inputs["attention_mask"].unsqueeze(-1).to(hidden.dtype)
-        pooled = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1)
+        pooled = last_token_hidden(hidden, inputs["attention_mask"])
         action = torch.tanh(self.head(pooled.float())).squeeze(0).cpu().numpy()
         return np.clip(action, -1.0, 1.0).astype(np.float32)
 

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a92011b869ded0ebf4deb4fc67dfda81f16c8c2bbcd64a557aa160d71cd24714
+oid sha256:d763700903b08fc9f6c1b7de760be9fac7f8d7a5b824c1fc1c1e11e348e2c7ea
 size 8875719408

model_config.json

@@ -1,6 +1,10 @@
 {
   "base_model": "/workspace/testnet-training-node-quickstart/models/Qwen3-VL-4B-Instruct",
   "hidden_size": 2560,
-  "head_hidden": 4096,
-  "action_dim": 7
+  "head_hidden": 2048,
+  "action_dim": 7,
+  "pool": "last_token",
+  "loss": "smooth_l1",
+  "best_val": 0.012188794997055084,
+  "train_avg": 0.00440433465039604
 }