update

.gitattributes

@@ -1,3 +0,0 @@
-action_head.safetensors filter=lfs diff=lfs merge=lfs -text
-model.safetensors filter=lfs diff=lfs merge=lfs -text
-tokenizer.json filter=lfs diff=lfs merge=lfs -text

action_head.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2ea073913c7192233212994e8a31860643dfd3a61b86e4bc98a5d5d0cdee40e4
-size 1208549892

chat_template.jinja

@@ -1,7 +0,0 @@
-{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
-You are a helpful assistant.<|im_end|>
-{% endif %}<|im_start|>{{ message['role'] }}
-{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
-{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
-{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
-{% endif %}

config.json

@@ -1,109 +0,0 @@
-{
-  "architectures": [
-    "Qwen2_5_VLForConditionalGeneration"
-  ],
-  "dtype": "bfloat16",
-  "image_token_id": 151655,
-  "model_type": "qwen2_5_vl",
-  "text_config": {
-    "attention_dropout": 0.0,
-    "bos_token_id": 151643,
-    "dtype": "bfloat16",
-    "eos_token_id": 151645,
-    "hidden_act": "silu",
-    "hidden_size": 2048,
-    "initializer_range": 0.02,
-    "intermediate_size": 11008,
-    "layer_types": [
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention",
-      "full_attention"
-    ],
-    "max_position_embeddings": 128000,
-    "max_window_layers": 70,
-    "model_type": "qwen2_5_vl_text",
-    "num_attention_heads": 16,
-    "num_hidden_layers": 36,
-    "num_key_value_heads": 2,
-    "pad_token_id": null,
-    "rms_norm_eps": 1e-06,
-    "rope_parameters": {
-      "mrope_section": [
-        16,
-        24,
-        24
-      ],
-      "rope_theta": 1000000.0,
-      "rope_type": "default",
-      "type": "mrope"
-    },
-    "sliding_window": null,
-    "use_cache": true,
-    "use_sliding_window": false,
-    "vocab_size": 151936
-  },
-  "tie_word_embeddings": true,
-  "transformers_version": "5.3.0",
-  "video_token_id": 151656,
-  "vision_config": {
-    "depth": 32,
-    "dtype": "bfloat16",
-    "fullatt_block_indexes": [
-      7,
-      15,
-      23,
-      31
-    ],
-    "hidden_act": "silu",
-    "hidden_size": 1280,
-    "in_channels": 3,
-    "in_chans": 3,
-    "initializer_range": 0.02,
-    "intermediate_size": 3420,
-    "model_type": "qwen2_5_vl",
-    "num_heads": 16,
-    "out_hidden_size": 2048,
-    "patch_size": 14,
-    "spatial_merge_size": 2,
-    "spatial_patch_size": 14,
-    "temporal_patch_size": 2,
-    "tokens_per_second": 2,
-    "window_size": 112
-  },
-  "vision_end_token_id": 151653,
-  "vision_start_token_id": 151652,
-  "vision_token_id": 151654
-}

flock_robotics_adapter.py

@@ -1,84 +0,0 @@
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-import numpy as np
-import torch
-import torch.nn as nn
-from PIL import Image
-from safetensors.torch import load_file
-from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration
-
-ACTION_DIM = 7
-
-
-def build_action_head(hidden_size, head_hidden):
-    return nn.Sequential(
-        nn.Linear(hidden_size, head_hidden),
-        nn.GELU(),
-        nn.Linear(head_hidden, head_hidden),
-        nn.GELU(),
-        nn.Linear(head_hidden, ACTION_DIM),
-    )
-
-
-class QwenVLAPolicy:
-    def __init__(self, model, processor, head, device):
-        self.model = model
-        self.processor = processor
-        self.head = head
-        self.device = device
-        self.model.eval()
-        self.head.eval()
-
-    @torch.no_grad()
-    def act(self, obs):
-        image = np.asarray(obs["image"], dtype=np.uint8)
-        instruction = str(obs.get("instruction", ""))
-        pil = Image.fromarray(image)
-        messages = [
-            {
-                "role": "user",
-                "content": [
-                    {"type": "image"},
-                    {"type": "text", "text": instruction},
-                ],
-            }
-        ]
-        text = self.processor.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True
-        )
-        inputs = self.processor(
-            text=[text], images=[pil], return_tensors="pt", padding=True
-        )
-        inputs = {k: v.to(self.device) for k, v in inputs.items()}
-        out = self.model(**inputs, output_hidden_states=True, use_cache=False)
-        hidden = out.hidden_states[-1]
-        mask = inputs["attention_mask"].unsqueeze(-1).to(hidden.dtype)
-        pooled = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1)
-        action = torch.tanh(self.head(pooled.float())).squeeze(0).cpu().numpy()
-        return np.clip(action, -1.0, 1.0).astype(np.float32)
-
-
-def load_policy(model_dir, device, dtype):
-    model_root = Path(model_dir)
-    cfg = json.loads((model_root / "model_config.json").read_text())
-    use_cuda = device == "cuda" and torch.cuda.is_available()
-    torch_device = torch.device("cuda" if use_cuda else "cpu")
-    torch_dtype = (
-        torch.bfloat16
-        if (use_cuda and str(dtype) == "bfloat16")
-        else torch.float32
-    )
-    model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
-        model_root, dtype=torch_dtype
-    ).to(torch_device)
-    processor = AutoProcessor.from_pretrained(model_root)
-    head = build_action_head(int(cfg["hidden_size"]), int(cfg["head_hidden"])).to(
-        torch_device
-    )
-    head.load_state_dict(
-        load_file(str(model_root / "action_head.safetensors"), device=str(torch_device))
-    )
-    return QwenVLAPolicy(model, processor, head, torch_device)

generation_config.json

@@ -1,12 +0,0 @@
-{
-  "bos_token_id": 151643,
-  "do_sample": true,
-  "eos_token_id": [
-    151645,
-    151643
-  ],
-  "pad_token_id": 151643,
-  "repetition_penalty": 1.05,
-  "temperature": 1e-06,
-  "transformers_version": "5.3.0"
-}

model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:05a02cbf57990c4d9b176489083c8509eec59daea405ce86043f199e1b18aea2
-size 7509338080

model_config.json

@@ -1,6 +0,0 @@
-{
-  "base_model": "Qwen/Qwen2.5-VL-3B-Instruct",
-  "hidden_size": 2048,
-  "head_hidden": 16384,
-  "action_dim": 7
-}

processor_config.json

@@ -1,63 +0,0 @@
-{
-  "image_processor": {
-    "data_format": "channels_first",
-    "do_convert_rgb": true,
-    "do_normalize": true,
-    "do_rescale": true,
-    "do_resize": true,
-    "image_mean": [
-      0.48145466,
-      0.4578275,
-      0.40821073
-    ],
-    "image_processor_type": "Qwen2VLImageProcessorFast",
-    "image_std": [
-      0.26862954,
-      0.26130258,
-      0.27577711
-    ],
-    "merge_size": 2,
-    "patch_size": 14,
-    "resample": 3,
-    "rescale_factor": 0.00392156862745098,
-    "size": {
-      "longest_edge": 12845056,
-      "shortest_edge": 3136
-    },
-    "temporal_patch_size": 2
-  },
-  "processor_class": "Qwen2_5_VLProcessor",
-  "video_processor": {
-    "data_format": "channels_first",
-    "default_to_square": true,
-    "do_convert_rgb": true,
-    "do_normalize": true,
-    "do_rescale": true,
-    "do_resize": true,
-    "do_sample_frames": false,
-    "image_mean": [
-      0.48145466,
-      0.4578275,
-      0.40821073
-    ],
-    "image_processor_type": "Qwen2VLImageProcessor",
-    "image_std": [
-      0.26862954,
-      0.26130258,
-      0.27577711
-    ],
-    "max_frames": 768,
-    "merge_size": 2,
-    "min_frames": 4,
-    "patch_size": 14,
-    "resample": 3,
-    "rescale_factor": 0.00392156862745098,
-    "return_metadata": false,
-    "size": {
-      "longest_edge": 12845056,
-      "shortest_edge": 3136
-    },
-    "temporal_patch_size": 2,
-    "video_processor_type": "Qwen2VLVideoProcessor"
-  }
-}

tokenizer.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
-size 11421892

tokenizer_config.json

@@ -1,30 +0,0 @@
-{
-  "add_prefix_space": false,
-  "backend": "tokenizers",
-  "bos_token": null,
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "extra_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
-  "is_local": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "processor_class": "Qwen2_5_VLProcessor",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
-}