add fg-clip large u16

.gitattributes

@@ -36,3 +36,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 image_encoder.axmodel filter=lfs diff=lfs merge=lfs -text
 text_encoder.axmodel filter=lfs diff=lfs merge=lfs -text
 fg-clip2-base/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text
+*.axmodel filter=lfs diff=lfs merge=lfs -text
+*.onnx.data filter=lfs diff=lfs merge=lfs -text

fg-clip2-large/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

fg-clip2-large/README.md

@@ -0,0 +1,221 @@
+---
+language:
+- en
+- zh
+library_name: transformers
+license: apache-2.0
+pipeline_tag: zero-shot-image-classification
+tags:
+- clip
+---
+
+# FG-CLIP 2: A Bilingual Fine-grained Vision-language Alignment Model
+
+Code: https://github.com/360CVGroup/FG-CLIP
+Project page: https://360cvgroup.github.io/FG-CLIP
+
+FG-CLIP 2 is the foundation model for fine-grained vision-language understanding in both English and Chinese. 
+Across 29 datasets and 8 diverse tasks, it consistently surpasses recent strong baselines such as SigLIP 2 and MetaCLIP 2, achieving the best reported performance to date in both languages. 
+
+**[FG-CLIP 2: A Bilingual Fine-grained Vision-language Alignment Model](https://arxiv.org/abs/2510.10921)** 
+</br>
+Chunyu Xie*, Bin Wang*, Fanjing Kong, Jincheng Li, Dawei Liang, Ji Ao, Dawei Leng†, Yuhui Yin(*Equal Contribution, †Corresponding Author)
+</br>
+[![arXiv](https://img.shields.io/badge/arXiv-2510.10921-b31b1b.svg)](https://arxiv.org/abs/2510.10921)
+[![HF-model](https://img.shields.io/badge/Model-Collection🤗-yellow.svg)](https://huggingface.co/collections/qihoo360/fg-clip-2-68ecbf9c548623bb78bc7913)
+[![HF-data](https://img.shields.io/badge/Benchmark-Collection🤗-yellow.svg)](https://huggingface.co/collections/qihoo360/fg-clip-2-68ecbf9c548623bb78bc7913)
+[![API+MCP](https://img.shields.io/badge/API/MCP-FG--CLIPv2-green.svg)](https://research.360.cn/sass/index)
+
+**[FG-CLIP: Fine-Grained Visual and Textual Alignment](https://arxiv.org/abs/2505.05071)** ([code branch: v1.0](https://github.com/360CVGroup/FG-CLIP/tree/v1.0))
+</br>
+Chunyu Xie*, Bin Wang*, Fanjing Kong, Jincheng Li, Dawei Liang, Gengshen Zhang, Dawei Leng†, Yuhui Yin (*Equal Contribution, †Corresponding Author)
+</br>
+[![arXiv](https://img.shields.io/badge/arXiv-2505.05071-b31b1b.svg)](https://arxiv.org/abs/2505.05071)
+[![ICML](https://img.shields.io/badge/ICML-2025-blue.svg)](https://icml.cc/Conferences/2025)
+[![HF-model](https://img.shields.io/badge/Model-Collection🤗-yellow.svg)](https://huggingface.co/collections/qihoo360/fg-clip-681da45d4acfb65c240a6d08)
+[![HF-data](https://img.shields.io/badge/Data-FineHARD🤗-yellow.svg)](https://huggingface.co/datasets/qihoo360/FineHARD)
+[![DeepWiki](https://img.shields.io/badge/DeepWiki-FG--CLIP-blue.svg?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAyCAYAAAAnWDnqAAAAAXNSR0IArs4c6QAAA05JREFUaEPtmUtyEzEQhtWTQyQLHNak2AB7ZnyXZMEjXMGeK/AIi+QuHrMnbChYY7MIh8g01fJoopFb0uhhEqqcbWTp06/uv1saEDv4O3n3dV60RfP947Mm9/SQc0ICFQgzfc4CYZoTPAswgSJCCUJUnAAoRHOAUOcATwbmVLWdGoH//PB8mnKqScAhsD0kYP3j/Yt5LPQe2KvcXmGvRHcDnpxfL2zOYJ1mFwrryWTz0advv1Ut4CJgf5uhDuDj5eUcAUoahrdY/56ebRWeraTjMt/00Sh3UDtjgHtQNHwcRGOC98BJEAEymycmYcWwOprTgcB6VZ5JK5TAJ+fXGLBm3FDAmn6oPPjR4rKCAoJCal2eAiQp2x0vxTPB3ALO2CRkwmDy5WohzBDwSEFKRwPbknEggCPB/imwrycgxX2NzoMCHhPkDwqYMr9tRcP5qNrMZHkVnOjRMWwLCcr8ohBVb1OMjxLwGCvjTikrsBOiA6fNyCrm8V1rP93iVPpwaE+gO0SsWmPiXB+jikdf6SizrT5qKasx5j8ABbHpFTx+vFXp9EnYQmLx02h1QTTrl6eDqxLnGjporxl3NL3agEvXdT0WmEost648sQOYAeJS9Q7bfUVoMGnjo4AZdUMQku50McDcMWcBPvr0SzbTAFDfvJqwLzgxwATnCgnp4wDl6Aa+Ax283gghmj+vj7feE2KBBRMW3FzOpLOADl0Isb5587h/U4gGvkt5v60Z1VLG8BhYjbzRwyQZemwAd6cCR5/XFWLYZRIMpX39AR0tjaGGiGzLVyhse5C9RKC6ai42ppWPKiBagOvaYk8lO7DajerabOZP46Lby5wKjw1HCRx7p9sVMOWGzb/vA1hwiWc6jm3MvQDTogQkiqIhJV0nBQBTU+3okKCFDy9WwferkHjtxib7t3xIUQtHxnIwtx4mpg26/HfwVNVDb4oI9RHmx5WGelRVlrtiw43zboCLaxv46AZeB3IlTkwouebTr1y2NjSpHz68WNFjHvupy3q8TFn3Hos2IAk4Ju5dCo8B3wP7VPr/FGaKiG6T+v+TQqIrOqMTL1VdWV1DdmcbO8KXBz6esmYWYKPwDL5b5FA1a0hwapHiom0r/cKaoqr+27/XcrS5UwSMbQAAAABJRU5ErkJggg==)](https://deepwiki.com/360CVGroup/FG-CLIP)
+
+<p align="center">
+  <img src="https://huggingface.co/qihoo360/fg-clip2-base/resolve/main/FGCLIP2_compare_all_n.png" width="500" height="440"/>
+</p>
+
+## Quick Start 🤗
+
+### Load Model
+```python
+import torch
+from PIL import Image
+from transformers import (
+    AutoImageProcessor,
+    AutoTokenizer,
+    AutoModelForCausalLM,
+)
+
+
+model_root = "qihoo360/fg-clip2-large"
+model = AutoModelForCausalLM.from_pretrained(model_root,trust_remote_code=True).cuda()
+
+device = model.device
+
+tokenizer = AutoTokenizer.from_pretrained(model_root)
+image_processor = AutoImageProcessor.from_pretrained(model_root)
+
+```
+
+
+### Retrieval
+
+```python
+def determine_max_value(image):
+    w,h = image.size
+    max_val = (w//16)*(h//16)
+    if max_val > 784:
+        return 1024
+    elif max_val > 576:
+        return 784
+    elif max_val > 256:
+        return 576
+    elif max_val > 128:
+        return 256
+    else:
+        return 128
+
+img_root = "cat_dfclor.jpg"
+image = Image.open(img_root).convert("RGB")
+
+image_input = image_processor(images=image, max_num_patches=determine_max_value(image), return_tensors="pt").to(device)
+
+# NOTE Short captions: max_length=64 walk_type="short"(default)
+# NOTE Long captions: max_length=196 walk_type="long"
+
+captions = [
+"一个简约风格的卧室角落，黑色金属衣架上挂着多件米色和白色的衣物，下方架子放着两双浅色鞋子，旁边是一盆绿植，左侧可见一张铺有白色床单和灰色枕���的床。",
+"一个简约风格的卧室角落，黑色金属衣架上挂着多件红色和蓝色的衣物，下方架子放着两双黑色高跟鞋，旁边是一盆绿植，左侧可见一张铺有白色床单和灰色枕头的床。",
+"一个简约风格的卧室角落，黑色金属衣架上挂着多件米色和白色的衣物，下方架子放着两双运动鞋，旁边是一盆仙人掌，左侧可见一张铺有白色床单和灰色枕头的床。",
+"一个繁忙的街头市场，摊位上摆满水果，背景是高楼大厦，人们在喧闹中购物。"
+]
+captions = [caption.lower() for caption in captions]
+
+caption_input = tokenizer(captions, padding="max_length", max_length=196, truncation=True, return_tensors="pt").to(device)
+
+
+with torch.no_grad():
+  image_feature = model.get_image_features(**image_input)
+  text_feature = model.get_text_features(**caption_input,walk_type="long")
+  image_feature = image_feature / image_feature.norm(p=2, dim=-1, keepdim=True)
+  text_feature = text_feature / text_feature.norm(p=2, dim=-1, keepdim=True)
+
+logits_per_image = image_feature @ text_feature.T
+logit_scale, logit_bias = model.logit_scale.to(text_feature.device), model.logit_bias.to(text_feature.device)
+logits_per_image = logits_per_image * logit_scale.exp() + logit_bias
+# The original Github example does not print probabilities for retrieval, keeping consistency.
+```
+ <p align="left">
+  <img src="https://huggingface.co/qihoo360/fg-clip2-base/resolve/main/cn_re_demo.png" width=100%/>
+</p>
+
+### Dense feature effect display
+
+```python
+
+import math
+import matplotlib
+matplotlib.use('Agg') 
+import matplotlib.pyplot as plt
+
+
+img_root = "cat_dfclor.jpg"
+image = Image.open(img_root).convert("RGB")
+# The 'resize_short_edge' function is not defined in the snippet or provided context.
+# Assuming 'cat_dfclor.jpg' is pre-processed or the model handles sizing.
+# image = resize_short_edge(image,target_size=2048) 
+
+image_input = image_processor(images=image, max_num_patches=16384, return_tensors="pt").to(device)
+captions = ["电脑","黑猫","窗户","window","white cat","book"]
+
+with torch.no_grad():
+    dense_image_feature = model.get_image_dense_feature(**image_input)
+    
+    spatial_values = image_input["spatial_shapes"][0]
+    real_h = spatial_values[0].item()
+    real_w = spatial_values[1].item()
+    real_pixel_tokens_num = real_w*real_h
+    dense_image_feature = dense_image_feature[0][:real_pixel_tokens_num]
+    captions = [caption.lower() for caption in captions]
+    caption_input = tokenizer(captions, padding="max_length", max_length=64, truncation=True, return_tensors="pt").to(device)
+
+    text_feature = model.get_text_features(**caption_input, walk_type="box")
+    text_feature = text_feature / text_feature.norm(p=2, dim=-1, keepdim=True)
+    dense_image_feature = dense_image_feature / dense_image_feature.norm(p=2, dim=-1, keepdim=True)
+
+similarity = dense_image_feature @ text_feature.T
+similarity = similarity.cpu()
+
+
+num_classes = len(captions)
+cols = 3
+rows = (num_classes + cols - 1) // cols
+
+
+aspect_ratio = real_w / real_h 
+
+fig_width_inch = 3 * cols        
+fig_height_inch = fig_width_inch / aspect_ratio * rows / cols  
+
+fig, axes = plt.subplots(rows, cols, figsize=(fig_width_inch, fig_height_inch))
+fig.subplots_adjust(wspace=0.01, hspace=0.01)
+
+if num_classes == 1:
+    axes = [axes]
+else:
+    axes = axes.flatten()
+
+for cls_index in range(num_classes):
+    similarity_map = similarity[:, cls_index].cpu().numpy()
+    show_image = similarity_map.reshape((real_h, real_w))
+
+    ax = axes[cls_index]
+    ax.imshow(show_image, cmap='viridis', aspect='equal')  
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.axis('off')
+
+
+for idx in range(num_classes, len(axes)):
+    axes[idx].axis('off')
+
+savename = "FGCLIP2_dfcolor_cat_all_2K.png"
+plt.savefig(savename, dpi=150, bbox_inches='tight', pad_inches=0.05)
+plt.close()
+```
+
+ <p align="left">
+  <img src="https://huggingface.co/qihoo360/fg-clip2-base/resolve/main/FGCLIP2_dfcolor_cat_all_2K.png" width=100%/>
+</p>
+
+## Citation
+If you find FG-CLIP 2 useful for your research and applications, please cite using this BibTeX:
+
+```
+@article{xie2025fg2,
+  title={FG-CLIP 2: A Bilingual Fine-grained Vision-language Alignment Model},
+  author={Xie, Chunyu and Wang, Bin and Kong, Fanjing and Li, Jincheng and Liang, Dawei and Ao, Ji and Leng, Dawei and Yin, Yuhui},
+  journal={arXiv preprint arXiv:2510.10921},
+  year={2025}
+}
+```
+```
+@article{xie2025fg,
+  title={FG-CLIP: Fine-Grained Visual and Textual Alignment},
+  author={Xie, Chunyu and Wang, Bin and Kong, Fanjing and Li, Jincheng and Liang, Dawei and Zhang, Gengshen and Leng, Dawei and Yin, Yuhui},
+  journal={arXiv preprint arXiv:2505.05071},
+  year={2025}
+}
+```
+
+
+## License
+
+This project utilizes certain datasets and checkpoints that are subject to their respective original licenses. Users must comply with all terms and conditions of these original licenses.
+The content of this project itself is licensed under the [Apache license 2.0](https://huggingface.co/qihoo360/fg-clip2-base/resolve/main/cn_re_demo.pngLICENSE).

fg-clip2-large/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a3921ec33c280cbbad5a61995cd31ef700491f8d088a8a167d67963a16a925c
+size 1110

fg-clip2-large/configuration_fgclip2.py

@@ -0,0 +1,274 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/fgclip2/modular_fgclip2.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_fgclip2.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class Fgclip2TextConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`Fgclip2TextModel`]. It is used to instantiate a
+    Fgclip2 text encoder according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the text encoder of the Fgclip2
+    [qihoo360/fg-clip2-base](https://huggingface.co/qihoo360/fg-clip2-base) architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the Fgclip2 text model. Defines the number of different tokens that can be represented by
+            the `inputs_ids` passed when calling [`Fgclip2Model`].
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        intermediate_size (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        max_position_embeddings (`int`, *optional*, defaults to 64):
+            The maximum sequence length that this model might ever be used with. Typically set this to something large
+            just in case (e.g., 512 or 1024 or 2048).
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the layer normalization layers.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        pad_token_id (`int`, *optional*, defaults to 1):
+            The id of the padding token in the vocabulary.
+        bos_token_id (`int`, *optional*, defaults to 49406):
+            The id of the beginning-of-sequence token in the vocabulary.
+        eos_token_id (`int`, *optional*, defaults to 49407):
+            The id of the end-of-sequence token in the vocabulary.
+        projection_size (`int`, *optional*, defaults to `hidden_size`):
+            The size of the projection head.
+        keep_len (`int`, *optional*, defaults to 20):
+            When processing long texts, the retained tokens are used for handling short text lengths.
+            For details, please refer to the FG-CLIP 'https://arxiv.org/abs/2505.05071' paper.
+        longtext_len (`int`, *optional*, defaults to 196):
+            The maximum number of tokens in long texts that can be processed
+
+
+    Example:
+
+    ```python
+    >>> from transformers import Fgclip2TextConfig, Fgclip2TextModel
+
+    >>> # Initializing a Fgclip2TextConfig with qihoo/fgclip2-base-patch16 style configuration
+    >>> configuration = Fgclip2TextConfig()
+
+    >>> # Initializing a Fgclip2TextModel (with random weights) from the qihoo/fgclip2-base-patch16 style configuration
+    >>> model = Fgclip2TextModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "fgclip2_text_model"
+    base_config_key = "text_config"
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        max_position_embeddings=64,
+        hidden_act="gelu_pytorch_tanh",
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        # This differs from `CLIPTokenizer`'s default and from openai/fgclip2
+        # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
+        pad_token_id=1,
+        bos_token_id=49406,
+        eos_token_id=49407,
+        projection_size=None,
+        keep_len=20,
+        longtext_len=196,
+        **kwargs,
+    ):
+        super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.max_position_embeddings = max_position_embeddings
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+        self.attention_dropout = attention_dropout
+        self.projection_size = projection_size if projection_size is not None else hidden_size
+        self.keep_len = keep_len
+        self.longtext_len = longtext_len
+
+
+class Fgclip2VisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`Fgclip2VisionModel`]. It is used to instantiate a
+    Fgclip2 vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the vision encoder of the Fgclip2
+    [qihoo/fgclip2-base-patch16](https://huggingface.co/qihoo/fgclip2-base-patch16) architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        intermediate_size (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_channels (`int`, *optional*, defaults to 3):
+            Number of channels in the input images.
+        num_patches (`int`, *optional*, defaults to 256):
+            The number of patches in the image with the size of (`patch_size`, `patch_size`).
+            The image is resized to fill maximum of this number of patches, and to preserve
+            the aspect ratio. In case the resulted number of patches is lower, the image is
+            padded in "patch" dimension.
+        patch_size (`int`, *optional*, defaults to 16):
+            The size (resolution) of each patch.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the layer normalization layers.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+
+    Example:
+
+    ```python
+    >>> from transformers import Fgclip2VisionConfig, Fgclip2VisionModel
+
+    >>> # Initializing a Fgclip2VisionConfig with qihoo/fgclip2-base-patch16 style configuration
+    >>> configuration = Fgclip2VisionConfig()
+
+    >>> # Initializing a Fgclip2VisionModel (with random weights) from the qihoo/fgclip2-base-patch16 style configuration
+    >>> model = Fgclip2VisionModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "fgclip2_vision_model"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        num_patches=256,
+        patch_size=16,
+        hidden_act="gelu_pytorch_tanh",
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.attention_dropout = attention_dropout
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+        self.num_patches = num_patches
+
+
+class Fgclip2Config(PretrainedConfig):
+    r"""
+    [`Fgclip2Config`] is the configuration class to store the configuration of a [`Fgclip2Model`]. It is used to
+    instantiate a Fgclip2 model according to the specified arguments, defining the text model and vision model configs.
+    Instantiating a configuration with the defaults will yield a similar configuration to that of the Fgclip2
+    [qihoo/fgclip2-base-patch16](https://huggingface.co/qihoo/fgclip2-base-patch16) architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        text_config (`dict`, *optional*):
+            Dictionary of configuration options used to initialize [`Fgclip2TextConfig`].
+        vision_config (`dict`, *optional*):
+            Dictionary of configuration options used to initialize [`Fgclip2VisionConfig`].
+        kwargs (*optional*):
+            Dictionary of keyword arguments.
+
+    Example:
+
+    ```python
+    >>> from transformers import Fgclip2Config, Fgclip2Model
+
+    >>> # Initializing a Fgclip2Config with qihoo/fgclip2-base-patch16 style configuration
+    >>> configuration = Fgclip2Config()
+
+    >>> # Initializing a Fgclip2Model (with random weights) from the qihoo/fgclip2-base-patch16 style configuration
+    >>> model = Fgclip2Model(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # We can also initialize a Fgclip2Config from a Fgclip2TextConfig and a Fgclip2VisionConfig
+    >>> from transformers import Fgclip2TextConfig, Fgclip2VisionConfig
+
+    >>> # Initializing a Fgclip2Text and Fgclip2Vision configuration
+    >>> config_text = Fgclip2TextConfig()
+    >>> config_vision = Fgclip2VisionConfig()
+
+    >>> config = Fgclip2Config.from_text_vision_configs(config_text, config_vision)
+    ```"""
+
+    model_type = "fgclip2"
+    sub_configs = {"text_config": Fgclip2TextConfig, "vision_config": Fgclip2VisionConfig}
+
+    def __init__(self, text_config=None, vision_config=None, **kwargs):
+        super().__init__(**kwargs)
+
+        if text_config is None:
+            text_config = {}
+            logger.info("`text_config` is `None`. Initializing the `Fgclip2TextConfig` with default values.")
+
+        if vision_config is None:
+            vision_config = {}
+            logger.info("`vision_config` is `None`. initializing the `Fgclip2VisionConfig` with default values.")
+
+        self.text_config = Fgclip2TextConfig(**text_config)
+        self.vision_config = Fgclip2VisionConfig(**vision_config)
+
+        self.initializer_factor = 1.0
+
+
+__all__ = ["Fgclip2Config", "Fgclip2TextConfig", "Fgclip2VisionConfig"]

fg-clip2-large/export_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b629685e6dabfc5e195812a4769702089dae5e08dcb51cbbc37036026560f8c
+size 1073

fg-clip2-large/fgclip2_large_image_encoder.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed9961a26494c3d5e51797a96f96d5cffa62b7ed6243f4c1104f94225519855e
+size 341054899

fg-clip2-large/fgclip2_large_image_encoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64f8d087f2e91aa5a4fdeacfbf8e08b8b003cbbab8e918682092e6dffa01b37a
+size 1264302855

fg-clip2-large/fgclip2_large_text_encoder_short.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839099deebbc372bf55cdff8d5bd08a73a572256e3a67c8fe3f0c52a766d431c
+size 852363032

fg-clip2-large/fgclip2_large_text_encoder_short.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52540ff00d2991b1b3b8faa8b388c21ea95c88a7aaaf6bb6e65ca340c7b51759
+size 602037

fg-clip2-large/fgclip2_large_text_encoder_short.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c54658e6f4313f278979f0fcb37d9757a3d526202c896757e271527a642f26a
+size 2262282240

fg-clip2-large/modeling_fgclip2.py

@@ -0,0 +1,1400 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/fgclip2/modular_fgclip2.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_fgclip2.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import warnings
+from dataclasses import dataclass
+from typing import Any, Callable, Optional, Union, List
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.init import _calculate_fan_in_and_fan_out
+from torchvision.ops import roi_align
+
+from transformers.activations import ACT2FN
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import ModelOutput, TransformersKwargs, auto_docstring, can_return_tuple, filter_out_non_signature_kwargs
+from transformers.utils.generic import check_model_inputs
+from .configuration_fgclip2 import Fgclip2Config, Fgclip2TextConfig, Fgclip2VisionConfig
+
+
+@dataclass
+@auto_docstring(
+    custom_intro="""
+    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.
+    """
+)
+class Fgclip2VisionOutput(ModelOutput):
+    r"""
+    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
+        The image embeddings obtained by applying the projection layer to the pooler_output.
+    """
+
+    image_embeds: Optional[torch.FloatTensor] = None
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
+
+
+@dataclass
+@auto_docstring(
+    custom_intro="""
+    Base class for text model's outputs that also contains a pooling of the last hidden states.
+    """
+)
+class Fgclip2TextOutput(ModelOutput):
+    r"""
+    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
+        The text embeddings obtained by applying the projection layer to the pooler_output.
+    """
+
+    text_embeds: Optional[torch.FloatTensor] = None
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
+
+
+@dataclass
+@auto_docstring
+class Fgclip2Output(ModelOutput):
+    r"""
+    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
+        Contrastive loss for image-text similarity.
+    logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
+        The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
+        similarity scores.
+    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
+        The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
+        similarity scores.
+    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
+        The text embeddings obtained by applying the projection layer to the pooled output of [`Fgclip2TextModel`].
+    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
+        The image embeddings obtained by applying the projection layer to the pooled output of [`Fgclip2VisionModel`].
+    text_model_output (`BaseModelOutputWithPooling`):
+        The output of the [`Fgclip2TextModel`].
+    vision_model_output (`BaseModelOutputWithPooling`):
+        The output of the [`Fgclip2VisionModel`].
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    logits_per_image: Optional[torch.FloatTensor] = None
+    logits_per_text: Optional[torch.FloatTensor] = None
+    text_embeds: Optional[torch.FloatTensor] = None
+    image_embeds: Optional[torch.FloatTensor] = None
+    text_model_output: BaseModelOutputWithPooling = None
+    vision_model_output: BaseModelOutputWithPooling = None
+
+    def to_tuple(self) -> tuple[Any]:
+        return tuple(
+            self[k] if k not in ["text_model_output", "vision_model_output"] else getattr(self, k).to_tuple()
+            for k in self.keys()
+        )
+
+
+class Fgclip2VisionEmbeddings(nn.Module):
+    def __init__(self, config: Fgclip2VisionConfig):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.patch_size = config.patch_size
+
+        self.patch_embedding = nn.Linear(
+            in_features=config.num_channels * self.patch_size * self.patch_size,
+            out_features=self.embed_dim,
+        )
+
+        self.num_patches = config.num_patches
+        self.position_embedding_size = int(self.num_patches**0.5)
+        self.position_embedding = nn.Embedding(self.num_patches, self.embed_dim)
+
+    @staticmethod
+    def resize_positional_embeddings(
+        positional_embeddings: torch.Tensor,
+        spatial_shapes: torch.LongTensor,
+        max_length: int,
+    ) -> torch.Tensor:
+        """
+        Resize positional embeddings to image-specific size and pad to a fixed size.
+
+        Args:
+            positional_embeddings (`torch.Tensor`):
+                Position embeddings of shape (height, width, embed_dim)
+            spatial_shapes (`torch.LongTensor`):
+                Spatial shapes of shape (batch_size, 2) to resize the positional embeddings to
+            max_length (`int`):
+                Maximum length of the positional embeddings to pad resized positional embeddings to
+
+        Returns:
+            `torch.Tensor`: Embeddings of shape (batch_size, max_length, embed_dim)
+        """
+        batch_size = spatial_shapes.shape[0]
+        embed_dim = positional_embeddings.shape[-1]
+        source_dtype = positional_embeddings.dtype
+
+        resulted_positional_embeddings = torch.empty(
+            (batch_size, max_length, embed_dim),
+            device=positional_embeddings.device,
+            dtype=source_dtype,
+        )
+
+        # (height, width, embed_dim) -> (1, embed_dim, height, width) for interpolation
+        positional_embeddings = positional_embeddings.permute(2, 0, 1).unsqueeze(0)
+
+        # Upcast to float32 on CPU because antialias is not supported for bfloat16/float16 on CPU
+        if positional_embeddings.device.type == "cpu":
+            positional_embeddings = positional_embeddings.to(torch.float32)
+
+        for i in range(batch_size):
+            # (1, dim, height, width) -> (1, dim, target_height, target_width)
+            height, width = spatial_shapes[i]
+            resized_embeddings = F.interpolate(
+                positional_embeddings,
+                size=(height, width),
+                mode="bilinear",
+                align_corners=False,
+                antialias=True,
+            )
+
+            # (1, dim, target_height, target_width) -> (target_height * target_width, dim)
+            resized_embeddings = resized_embeddings.reshape(embed_dim, height * width).transpose(0, 1)
+
+            # Cast to original dtype
+            resized_embeddings = resized_embeddings.to(source_dtype)
+
+            resulted_positional_embeddings[i, : height * width] = resized_embeddings
+            resulted_positional_embeddings[i, height * width :] = resized_embeddings[0]
+
+        return resulted_positional_embeddings
+
+    def forward(self, pixel_values: torch.FloatTensor, spatial_shapes: torch.LongTensor) -> torch.Tensor:
+        """
+        Args:
+            pixel_values (`torch.FloatTensor`):
+                Pixel values of shape (batch_size, max_num_patches, num_channels * patch_size * patch_size)
+            spatial_shapes (`list[tuple[int, int]]`):
+                Spatial shapes of shape (batch_size, 2) to resize the positional embeddings to
+        """
+
+        # Apply patch embeddings to already patchified pixel values
+        target_dtype = self.patch_embedding.weight.dtype
+        patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype))
+
+        # Get positional resized and padded positional embeddings
+        positional_embeddings = self.position_embedding.weight.reshape(
+            self.position_embedding_size, self.position_embedding_size, -1
+        )
+        resized_positional_embeddings = self.resize_positional_embeddings(
+            positional_embeddings, spatial_shapes, max_length=pixel_values.shape[1]
+        )
+
+        # Add positional embeddings to patch embeddings
+        embeddings = patch_embeds + resized_positional_embeddings
+        return embeddings
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: Optional[torch.Tensor],
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs,
+):
+    attn_weights = torch.matmul(query, key.transpose(-1, -2)) * scaling
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+
+    attn_output = torch.matmul(attn_weights, value)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class Fgclip2Attention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+        self.dropout = config.attention_dropout
+        self.is_causal = False
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
+        """Input shape: Batch x Time x Channel"""
+
+        batch_size, seq_length, embed_dim = hidden_states.shape
+
+        queries = self.q_proj(hidden_states)
+        keys = self.k_proj(hidden_states)
+        values = self.v_proj(hidden_states)
+
+        queries = queries.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2)
+        keys = keys.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2)
+        values = values.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2)
+
+        attention_interface: Callable = eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            queries,
+            keys,
+            values,
+            attention_mask,
+            is_causal=self.is_causal,
+            scaling=self.scale,
+            dropout=0.0 if not self.training else self.dropout,
+        )
+
+        attn_output = attn_output.reshape(batch_size, seq_length, embed_dim).contiguous()
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class Fgclip2MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+class Fgclip2EncoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: Union[Fgclip2VisionConfig, Fgclip2TextConfig]):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+        self.self_attn = Fgclip2Attention(config)
+        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+        self.mlp = Fgclip2MLP(config)
+
+    @auto_docstring
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.FloatTensor:
+        residual = hidden_states
+
+        hidden_states = self.layer_norm1(hidden_states)
+        hidden_states, _ = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        return hidden_states
+
+
+class Fgclip2Encoder(nn.Module):
+    """
+    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+    [`Fgclip2EncoderLayer`].
+
+    Args:
+        config: Fgclip2Config
+    """
+
+    def __init__(self, config: Fgclip2Config):
+        super().__init__()
+        self.config = config
+        self.layers = nn.ModuleList([Fgclip2EncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    # Ignore copy
+    @auto_docstring
+    def forward(
+        self,
+        inputs_embeds,
+        attention_mask: Optional[torch.Tensor] = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutput:
+        hidden_states = inputs_embeds
+        for encoder_layer in self.layers:
+            hidden_states = encoder_layer(
+                hidden_states,
+                attention_mask,
+                **kwargs,
+            )
+
+        return BaseModelOutput(last_hidden_state=hidden_states)
+
+
+class Fgclip2VisionTransformer(nn.Module):
+    def __init__(self, config: Fgclip2VisionConfig):
+        super().__init__()
+        self.config = config
+        embed_dim = config.hidden_size
+
+        self.embeddings = Fgclip2VisionEmbeddings(config)
+        self.encoder = Fgclip2Encoder(config)
+        self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+        self.use_head = True if not hasattr(config, "vision_use_head") else config.vision_use_head
+        if self.use_head:
+            self.head = Fgclip2MultiheadAttentionPoolingHead(config)
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        attention_mask: torch.Tensor,
+        spatial_shapes: torch.LongTensor,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+    ) -> BaseModelOutputWithPooling:
+        r"""
+        spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
+            Tensor containing the spatial dimensions (height, width) of the input images.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+
+        hidden_states = self.embeddings(pixel_values, spatial_shapes)
+
+        if attention_mask is not None and self.config._attn_implementation != "flash_attention_2":
+            # [batch_size, seq_len] -> [batch_size, 1, tgt_seq_len, src_seq_len]
+            encoder_attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype)
+        else:
+            encoder_attention_mask = attention_mask
+
+        encoder_outputs: BaseModelOutput = self.encoder(
+            inputs_embeds=hidden_states,
+            attention_mask=encoder_attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+
+        last_hidden_state = encoder_outputs.last_hidden_state
+        last_hidden_state = self.post_layernorm(last_hidden_state)
+
+        pooler_output = self.head(last_hidden_state, attention_mask) if self.use_head else None
+
+        return BaseModelOutputWithPooling(
+            last_hidden_state=last_hidden_state,
+            pooler_output=pooler_output,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+def _trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn(
+            "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+            "The distribution of values may be incorrect.",
+            stacklevel=2,
+        )
+
+    # Values are generated by using a truncated uniform distribution and
+    # then using the inverse CDF for the normal distribution.
+    # Get upper and lower cdf values
+    l = norm_cdf((a - mean) / std)
+    u = norm_cdf((b - mean) / std)
+
+    # Uniformly fill tensor with values from [l, u], then translate to
+    # [2l-1, 2u-1].
+    tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+    # Use inverse cdf transform for normal distribution to get truncated
+    # standard normal
+    tensor.erfinv_()
+
+    # Transform to proper mean, std
+    tensor.mul_(std * math.sqrt(2.0))
+    tensor.add_(mean)
+
+    # Clamp to ensure it's in the proper range
+    tensor.clamp_(min=a, max=b)
+
+
+def trunc_normal_tf_(
+    tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
+) -> torch.Tensor:
+    """Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are effectively drawn from the
+    normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
+    with values outside :math:`[a, b]` redrawn until they are within
+    the bounds. The method used for generating the random values works
+    best when :math:`a \\leq \text{mean} \\leq b`.
+
+    NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
+    bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
+    and the result is subsequently scaled and shifted by the mean and std args.
+
+    Args:
+        tensor: an n-dimensional `torch.Tensor`
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+        a: the minimum cutoff value
+        b: the maximum cutoff value
+    """
+    with torch.no_grad():
+        _trunc_normal_(tensor, 0, 1.0, a, b)
+        tensor.mul_(std).add_(mean)
+
+
+def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    if mode == "fan_in":
+        denom = fan_in
+    elif mode == "fan_out":
+        denom = fan_out
+    elif mode == "fan_avg":
+        denom = (fan_in + fan_out) / 2
+
+    variance = scale / denom
+
+    if distribution == "truncated_normal":
+        # constant is stddev of standard normal truncated to (-2, 2)
+        trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
+    elif distribution == "normal":
+        with torch.no_grad():
+            tensor.normal_(std=math.sqrt(variance))
+    elif distribution == "uniform":
+        bound = math.sqrt(3 * variance)
+        with torch.no_grad():
+            tensor.uniform_(-bound, bound)
+    else:
+        raise ValueError(f"invalid distribution {distribution}")
+
+
+def lecun_normal_(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
+
+
+def default_flax_embed_init(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="normal")
+
+
+@auto_docstring
+class Fgclip2PreTrainedModel(PreTrainedModel):
+    config: Fgclip2Config
+    base_model_prefix = "fgclip2"
+    supports_gradient_checkpointing = True
+
+    _no_split_modules = [
+        "Fgclip2TextEmbeddings",
+        "Fgclip2VisionEmbeddings",
+        "Fgclip2EncoderLayer",
+        "Fgclip2MultiheadAttentionPoolingHead",
+    ]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+
+    _can_record_outputs = {
+        "hidden_states": Fgclip2EncoderLayer,
+        "attentions": Fgclip2Attention,
+    }
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, Fgclip2VisionEmbeddings):
+            width = (
+                self.config.vision_config.hidden_size
+                if isinstance(self.config, Fgclip2Config)
+                else self.config.hidden_size
+            )
+            nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
+        elif isinstance(module, nn.Embedding):
+            default_flax_embed_init(module.weight)
+        elif isinstance(module, Fgclip2Attention):
+            nn.init.xavier_uniform_(module.q_proj.weight)
+            nn.init.xavier_uniform_(module.k_proj.weight)
+            nn.init.xavier_uniform_(module.v_proj.weight)
+            nn.init.xavier_uniform_(module.out_proj.weight)
+            nn.init.zeros_(module.q_proj.bias)
+            nn.init.zeros_(module.k_proj.bias)
+            nn.init.zeros_(module.v_proj.bias)
+            nn.init.zeros_(module.out_proj.bias)
+        elif isinstance(module, Fgclip2MLP):
+            nn.init.xavier_uniform_(module.fc1.weight)
+            nn.init.xavier_uniform_(module.fc2.weight)
+            nn.init.normal_(module.fc1.bias, std=1e-6)
+            nn.init.normal_(module.fc2.bias, std=1e-6)
+        elif isinstance(module, Fgclip2MultiheadAttentionPoolingHead):
+            nn.init.xavier_uniform_(module.probe.data)
+            nn.init.xavier_uniform_(module.attention.in_proj_weight.data)
+            nn.init.zeros_(module.attention.in_proj_bias.data)
+        elif isinstance(module, Fgclip2Model):
+            logit_scale_init = torch.log(torch.tensor(1.0))
+            module.logit_scale.data.fill_(logit_scale_init)
+            module.logit_bias.data.zero_()
+        elif isinstance(module, (nn.Linear, nn.Conv2d)):
+            lecun_normal_(module.weight)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+
+class Fgclip2TextEmbeddings(nn.Module):
+    def __init__(self, config: Fgclip2TextConfig):
+        super().__init__()
+        embed_dim = config.hidden_size
+
+        self.token_embedding = nn.Embedding(config.vocab_size, embed_dim)
+        self.position_embedding = nn.Embedding(config.max_position_embeddings, embed_dim)
+
+        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
+        self.register_buffer(
+            "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)), persistent=False
+        )
+
+        keep_len = config.keep_len
+        longtext_len = config.longtext_len
+
+        self.position_embedding_res = nn.Embedding(longtext_len, embed_dim)
+        self.position_embedding_ori = nn.Embedding(longtext_len, embed_dim)
+
+        self.mask1 = torch.zeros([longtext_len, 1])
+        self.mask1[:keep_len, :] = 1
+        self.mask2 = torch.zeros([longtext_len, 1])
+        self.mask2[keep_len:, :] = 1
+
+        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
+        self.register_buffer("position_ids", torch.arange(longtext_len).expand((1, -1)), persistent=False)
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_short_position_ids: Optional[bool] = True,
+    ) -> torch.Tensor:
+        r"""
+        Args:
+        use_short_position_ids (`bool`, optional, defaults to `True`):
+            If `True`, applies a positional encoding scheme optimized for **short-text processing** and **local-region description processing**,
+            such as phrases or simple sentences. Corresponds to the `"short"` and `"box"` walk type.
+            Assumes compact semantic structure and local dependency dominance.
+        """
+
+        seq_length = input_ids.shape[-1] if input_ids is not None else inputs_embeds.shape[-2]
+
+        if position_ids is None:
+            position_ids = self.position_ids[:, :seq_length]
+
+        if inputs_embeds is None:
+            inputs_embeds = self.token_embedding(input_ids)
+
+        if use_short_position_ids:
+            position_embeddings = self.position_embedding(position_ids)
+            embeddings = inputs_embeds + position_embeddings
+        else:
+            position_embeddings_res = self.position_embedding_res(position_ids)
+            position_embeddings_ori = self.position_embedding_ori(position_ids)
+            embeddings = (
+                inputs_embeds
+                + (position_embeddings_ori * self.mask1.to(inputs_embeds.device))
+                .type(inputs_embeds.dtype)
+                .to(inputs_embeds.device)
+                + (position_embeddings_res * self.mask2.to(inputs_embeds.device))
+                .type(inputs_embeds.dtype)
+                .to(inputs_embeds.device)
+            )
+
+        return embeddings
+
+
+class Fgclip2TextTransformer(nn.Module):
+    def __init__(self, config: Fgclip2TextConfig):
+        super().__init__()
+        self.config = config
+        embed_dim = config.hidden_size
+        self.embeddings = Fgclip2TextEmbeddings(config)
+        self.encoder = Fgclip2Encoder(config)
+        self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+        self.head = nn.Linear(embed_dim, config.projection_size)
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        walk_type: str = "short",  # Modified: Single parameter
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPooling:
+        r"""
+        Args:
+        walk_type (`str`, optional, defaults to `"short"`):
+            The traversal strategy used during feature extraction. Must be one of
+            `"short"`, `"box"`, or `"long"`. This controls how contextual information
+            is aggregated across the input:
+            - `"short"`: Optimized for short-text understanding, focusing on tight semantic coherence
+            and direct word interactions. Suitable when the input is a phrase or brief sentence.
+            - `"box"`: Designed for local-region description processing, such as grounding in vision-language
+            models or processing localized textual descriptions (e.g., object regions or segments).
+            Emphasizes dense features within bounded semantic units.
+            - `"long"`: Tailored for long-form text processing, enabling modeling of extended dependencies
+            and discourse structure. Uses strategies like chunking or hierarchical attention to handle
+            longer sequences effectively.
+        """
+        if input_ids is None:
+            raise ValueError("You have to specify input_ids")
+
+        # Validate walk_type
+        walk_type = walk_type.lower()
+        if walk_type not in ["short", "box", "long"]:
+            raise ValueError(f"Invalid `walk_type`: {walk_type}. Must be one of 'short', 'box', 'long'.")
+
+        # Convert walk_type to boolean flags for internal logic
+        walk_short = walk_type == "short"
+        walk_box = walk_type == "box"
+        walk_long = walk_type == "long"
+
+        input_shape = input_ids.size()
+        input_ids = input_ids.view(-1, input_shape[-1])
+        hidden_states = self.embeddings(
+            input_ids=input_ids, position_ids=position_ids, use_short_position_ids=(not walk_long)
+        )
+        # note: fgclip2's text model does not use a causal mask, unlike the original CLIP model.
+        # expand attention_mask
+        uses_flash_attention = "flash" in self.config._attn_implementation
+        if uses_flash_attention:
+            attention_mask = None
+        elif attention_mask is not None and not uses_flash_attention:
+            # [batch_size, seq_len] -> [batch_size, 1, tgt_seq_len, src_seq_len]
+            attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype)
+        encoder_outputs: BaseModelOutput = self.encoder(
+            inputs_embeds=hidden_states,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+        last_hidden_state = encoder_outputs.last_hidden_state
+        last_hidden_state = self.final_layer_norm(last_hidden_state)
+        # The model uses the last token's hidden state, which may be padding.
+        pooled_output = last_hidden_state[:, -1, :]
+        if walk_short == True:
+            assert walk_box == False
+            assert walk_long == False
+            temp_pool_out = []
+            for i in range(pooled_output.shape[0]):
+                temp_pool_out.append(self.head(pooled_output[i : i + 1]))
+            pooled_output = torch.cat(temp_pool_out, dim=0)
+            # pooled_output = self.head(pooled_output)
+        if walk_box == True:
+            assert walk_short == False
+            assert walk_long == False
+            pooled_output = pooled_output
+        if walk_long == True:
+            assert walk_short == False
+            assert walk_box == False
+            pooled_output = pooled_output
+        return BaseModelOutputWithPooling(
+            last_hidden_state=last_hidden_state,
+            pooler_output=pooled_output,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    The text model from Fgclip2 without any head or projection on top.
+    """
+)
+class Fgclip2TextModel(Fgclip2PreTrainedModel):
+    config: Fgclip2TextConfig
+
+    def __init__(self, config: Fgclip2TextConfig):
+        super().__init__(config)
+        self.text_model = Fgclip2TextTransformer(config)
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.text_model.embeddings.token_embedding
+
+    def set_input_embeddings(self, value):
+        self.text_model.embeddings.token_embedding = value
+
+    @check_model_inputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        walk_type: str = "short",  # Modified: Single parameter
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPooling:
+        r"""
+        Args:
+        walk_type (`str`, optional, defaults to `"short"`):
+            The traversal strategy used during feature extraction. Must be one of
+            `"short"`, `"box"`, or `"long"`. This controls how contextual information
+            is aggregated across the input:
+            - `"short"`: Optimized for short-text understanding, focusing on tight semantic coherence
+            and direct word interactions. Suitable when the input is a phrase or brief sentence.
+            - `"box"`: Designed for local-region description processing, such as grounding in vision-language
+            models or processing localized textual descriptions (e.g., object regions or segments).
+            Emphasizes dense features within bounded semantic units.
+            - `"long"`: Tailored for long-form text processing, enabling modeling of extended dependencies
+            and discourse structure. Uses strategies like chunking or hierarchical attention to handle
+            longer sequences effectively.
+        """
+        return self.text_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            walk_type=walk_type,  # Modified: Pass single parameter
+            **kwargs,
+        )
+
+
+class Fgclip2MultiheadAttentionPoolingHead(nn.Module):
+    """Multihead Attention Pooling."""
+
+    def __init__(self, config: Fgclip2VisionConfig):
+        super().__init__()
+
+        self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
+        self.attention = torch.nn.MultiheadAttention(config.hidden_size, config.num_attention_heads, batch_first=True)
+        self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.mlp = Fgclip2MLP(config)
+        self.num_heads = config.num_attention_heads
+
+    def forward(self, hidden_state: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+        batch_size = hidden_state.shape[0]
+        probe = self.probe.repeat(batch_size, 1, 1)
+
+        if attention_mask is not None:
+            target_len, source_len = probe.shape[1], hidden_state.shape[1]
+            attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_state.dtype, target_len)
+            attention_mask = attention_mask.repeat(1, self.num_heads, target_len, 1)
+            attention_mask = attention_mask.reshape(-1, target_len, source_len)
+
+        group_size = self.num_heads
+        outputs = []
+        for i in range(batch_size):
+            start_idx = i * group_size
+            end_idx = start_idx + group_size
+            out_i = self.attention(
+                probe[i : i + 1],
+                hidden_state[i : i + 1],
+                hidden_state[i : i + 1],
+                attn_mask=attention_mask[start_idx:end_idx] if attention_mask is not None else None,
+            )[0]
+            outputs.append(out_i)
+
+        hidden_state = torch.cat(outputs, dim=0)
+        residual = hidden_state
+        hidden_state = self.layernorm(hidden_state)
+
+        temp_outs = []
+        for k in range(batch_size):
+            out_k = self.mlp(hidden_state[k : k + 1])
+            temp_outs.append(out_k)
+        hidden_state = residual + torch.cat(temp_outs, dim=0)
+
+        return hidden_state[:, 0]
+
+
+@auto_docstring(
+    custom_intro="""
+    The vision model from Fgclip2 without any head or projection on top.
+    """
+)
+class Fgclip2VisionModel(Fgclip2PreTrainedModel):
+    config: Fgclip2VisionConfig
+    main_input_name = "pixel_values"
+
+    def __init__(self, config: Fgclip2VisionConfig):
+        super().__init__(config)
+
+        self.vision_model = Fgclip2VisionTransformer(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.vision_model.embeddings.patch_embedding
+
+    @check_model_inputs
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        pixel_attention_mask: torch.Tensor,
+        spatial_shapes: torch.LongTensor,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+    ) -> BaseModelOutputWithPooling:
+        r"""
+        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
+            Mask to avoid performing attention on padding pixel indices.
+        spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
+            Tensor containing the spatial dimensions (height, width) of the input images.
+
+        Examples:
+
+        ```python
+        >>> from PIL import Image
+        >>> import requests
+        >>> from transformers import AutoProcessor, Fgclip2VisionModel
+
+        >>> model = Fgclip2VisionModel.from_pretrained("qihoo360/fg-clip2-base")
+        >>> processor = AutoProcessor.from_pretrained("qihoo360/fg-clip2-base")
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = Image.open(requests.get(url, stream=True).raw)
+
+        >>> inputs = processor(images=image, return_tensors="pt")
+
+        >>> outputs = model(**inputs)
+        >>> last_hidden_state = outputs.last_hidden_state
+        >>> pooled_output = outputs.pooler_output  # pooled features
+        ```"""
+        return self.vision_model(
+            pixel_values=pixel_values,
+            attention_mask=pixel_attention_mask,
+            spatial_shapes=spatial_shapes,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+
+
+@auto_docstring
+class Fgclip2Model(Fgclip2PreTrainedModel):
+    config: Fgclip2Config
+
+    def __init__(self, config: Fgclip2Config):
+        super().__init__(config)
+
+        if not isinstance(config.text_config, Fgclip2TextConfig):
+            raise TypeError(
+                "config.text_config is expected to be of type Fgclip2TextConfig but is of type"
+                f" {type(config.text_config)}."
+            )
+
+        if not isinstance(config.vision_config, Fgclip2VisionConfig):
+            raise TypeError(
+                "config.vision_config is expected to be of type Fgclip2VisionConfig but is of type"
+                f" {type(config.vision_config)}."
+            )
+
+        text_config = config.text_config
+        vision_config = config.vision_config
+
+        # First, initialize the text and vision models with proper attention implementation
+        text_model = Fgclip2TextModel._from_config(text_config)
+        vision_model = Fgclip2VisionModel._from_config(vision_config)
+
+        # Second, get the text and vision submodules (for backward compatibility)
+        self.text_model = text_model.text_model
+        self.vision_model = vision_model.vision_model
+
+        self.logit_scale = nn.Parameter(torch.randn(1))
+        self.logit_bias = nn.Parameter(torch.randn(1))
+        self.dense_feature_head = Fgclip2MultiheadAttentionPoolingHead(vision_config)
+        self.embed_dim = text_config.hidden_size
+        self.longtext_head = nn.Linear(self.embed_dim, self.embed_dim)
+        self.boxtext_head = nn.Linear(self.embed_dim, self.embed_dim)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @filter_out_non_signature_kwargs()
+    @auto_docstring
+    def get_text_features(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        walk_type: str = "short",
+    ) -> torch.FloatTensor:
+        r"""
+        Extracts feature representations from the input text.
+
+        Args:
+            input_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`):
+                The token IDs of the input sequence, as generated by the tokenizer.
+            attention_mask (`torch.Tensor`, optional, of shape `(batch_size, sequence_length)`):
+                A mask indicating which tokens are valid (1) and which are padding (0).
+                If not provided, all tokens are assumed to be valid.
+            position_ids (`torch.Tensor`, optional, of shape `(batch_size, sequence_length)`):
+                Position indices for each token in the sequence. If not provided,
+                positions are automatically constructed based on `input_ids`.
+            walk_type (`str`, optional, defaults to `"short"`):
+                The traversal strategy used during feature extraction. Must be one of
+                `"short"`, `"box"`, or `"long"`. This controls how contextual information
+                is aggregated across the input:
+                - `"short"`: Optimized for short-text understanding, focusing on tight semantic coherence
+                and direct word interactions. Suitable when the input is a phrase or brief sentence.
+                - `"box"`: Designed for local-region description processing, such as grounding in vision-language
+                models or processing localized textual descriptions (e.g., object regions or segments).
+                Emphasizes dense features within bounded semantic units.
+                - `"long"`: Tailored for long-form text processing, enabling modeling of extended dependencies
+                and discourse structure. Uses strategies like chunking or hierarchical attention to handle
+                longer sequences effectively.
+
+        Returns:
+            `torch.FloatTensor` of shape `(batch_size, hidden_size)` or `(batch_size, sequence_length, hidden_size)`:
+                The extracted feature tensor representing the input text. The output shape depends on
+                whether a pooled representation or per-token embeddings are returned.
+
+        Examples:
+
+        ```python
+        >>> from transformers import AutoTokenizer, AutoModel
+        >>> import torch
+
+        >>> model = AutoModel.from_pretrained("qihoo360/fg-clip2-base")
+        >>> tokenizer = AutoTokenizer.from_pretrained("qihoo360/fg-clip2-base")
+
+        >>> # important: make sure to set padding="max_length" as that's how the model was trained
+        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding="max_length", return_tensors="pt")
+        >>> with torch.no_grad():
+        ...     text_features = model.get_text_features(**inputs, walk_type="short")
+        ```"""
+
+        walk_type = walk_type.lower()
+
+        if walk_type not in ["short", "box", "long"]:
+            raise ValueError(f"Invalid `walk_type`: {walk_type}. Must be one of 'short', 'box', 'long'.")
+
+        walk_short = walk_type == "short"
+        walk_box = walk_type == "box"
+        walk_long = walk_type == "long"
+
+        text_outputs: BaseModelOutputWithPooling = self.text_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            walk_type=walk_type,
+        )
+
+        if walk_short:
+            pooled_output = text_outputs.pooler_output
+
+        if walk_box:
+            pooled_output = self.boxtext_head(text_outputs.pooler_output)
+
+        if walk_long:
+            pooled_output = self.longtext_head(text_outputs.pooler_output)
+
+        return pooled_output
+
+    @filter_out_non_signature_kwargs()
+    @auto_docstring
+    def get_image_features(
+        self,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_attention_mask: Optional[torch.Tensor] = None,
+        spatial_shapes: Optional[torch.LongTensor] = None,
+    ) -> torch.FloatTensor:
+        r"""
+        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
+            Mask to avoid performing attention on padding pixel indices.
+        spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
+            Tensor containing the spatial dimensions (height, width) of the input images.
+
+        Returns:
+            image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+            applying the projection layer to the pooled output of [`Fgclip2VisionModel`].
+
+        Examples:
+
+        ```python
+        >>> import torch
+        >>> from transformers import AutoProcessor, AutoModel
+        >>> from transformers.image_utils import load_image
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = load_image(url)
+
+        >>> model = AutoModel.from_pretrained("qihoo360/fg-clip2-base")
+        >>> processor = AutoProcessor.from_pretrained("qihoo360/fg-clip2-base")
+
+        >>> inputs = processor(images=image, return_tensors="pt")
+
+        >>> with torch.no_grad():
+        ...     image_features = model.get_image_features(**inputs)
+        ```
+        """
+        vision_outputs: BaseModelOutputWithPooling = self.vision_model(
+            pixel_values=pixel_values,
+            attention_mask=pixel_attention_mask,
+            spatial_shapes=spatial_shapes,
+        )
+        pooled_output = vision_outputs.pooler_output
+
+        return pooled_output
+
+    # NOTE: Fgclip2Model uses Pretrained backbones, so we don't need to add `check_model_inputs` here
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_attention_mask: Optional[torch.Tensor] = None,
+        spatial_shapes: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        return_loss: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        walk_type: str = "short",
+    ) -> Fgclip2Output:
+        r"""
+        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
+            Mask to avoid performing attention on padding pixel indices.
+        spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
+            Tensor containing the spatial dimensions (height, width) of the input images.
+        return_loss (`bool`, *optional*):
+            Whether or not to return the contrastive loss.
+        walk_type (`str`, optional, defaults to `"short"`):
+                The traversal strategy used during feature extraction. Must be one of
+                `"short"`, `"box"`, or `"long"`. This controls how contextual information
+                is aggregated across the input:
+                - `"short"`: Optimized for short-text understanding, focusing on tight semantic coherence
+                and direct word interactions. Suitable when the input is a phrase or brief sentence.
+                - `"box"`: Designed for local-region description processing, such as grounding in vision-language
+                models or processing localized textual descriptions (e.g., object regions or segments).
+                Emphasizes dense features within bounded semantic units.
+                - `"long"`: Tailored for long-form text processing, enabling modeling of extended dependencies
+                and discourse structure. Uses strategies like chunking or hierarchical attention to handle
+                longer sequences effectively.
+        
+
+        Examples:
+
+        ```python
+        >>> from PIL import Image
+        >>> import requests
+        >>> from transformers import AutoProcessor, AutoModel
+        >>> import torch
+
+        >>> model = AutoModel.from_pretrained("qihoo360/fg-clip2-base")
+        >>> processor = AutoProcessor.from_pretrained("qihoo360/fg-clip2-base")
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = Image.open(requests.get(url, stream=True).raw)
+
+        >>> texts = ["a photo of 2 cats", "a photo of 2 dogs"]
+        >>> # important: we pass `padding=max_length` since the model was trained with this
+        >>> inputs = processor(text=texts, images=image, padding="max_length", return_tensors="pt")
+
+        >>> with torch.no_grad():
+        ...     outputs = model(**inputs)
+
+        >>> logits_per_image = outputs.logits_per_image
+        >>> probs = torch.sigmoid(logits_per_image) # these are the probabilities
+        >>> print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'")
+        31.9% that image 0 is 'a photo of 2 cats'
+        ```
+        """
+        walk_type = walk_type.lower()
+
+        if walk_type not in ["short", "box", "long"]:
+            raise ValueError(f"Invalid `walk_type`: {walk_type}. Must be one of 'short', 'box', 'long'.")
+
+        walk_short = walk_type == "short"
+        walk_box = walk_type == "box"
+        walk_long = walk_type == "long"
+
+        # Use Fgclip2 model's config for some fields (if specified) instead of those of vision & text components.
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+
+        vision_outputs: BaseModelOutputWithPooling = self.vision_model(
+            pixel_values=pixel_values,
+            attention_mask=pixel_attention_mask,
+            spatial_shapes=spatial_shapes,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+
+        text_outputs: BaseModelOutputWithPooling = self.text_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            walk_type=walk_type,
+        )
+
+        image_embeds = vision_outputs.pooler_output
+
+        if walk_short:
+            text_embeds = text_outputs.pooler_output
+
+        if walk_box:
+            text_embeds = self.boxtext_head(text_outputs.pooler_output)
+
+        if walk_long:
+            text_embeds = self.longtext_head(text_outputs.pooler_output)
+
+        # normalized features
+        image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
+        text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
+
+        # cosine similarity as logits
+        logits_per_text = torch.matmul(text_embeds, image_embeds.t().to(text_embeds.device))
+
+        logit_scale, logit_bias = self.logit_scale.to(text_embeds.device), self.logit_bias.to(text_embeds.device)
+        logits_per_text = logits_per_text * logit_scale.exp() + logit_bias
+
+        logits_per_image = logits_per_text.t()
+
+        loss = None
+        if return_loss:
+            # Adapted from https://github.com/google-research/big_vision/blob/01edb81a4716f93a48be43b3a4af14e29cdb3a7f/big_vision/trainers/proj/image_text/fgclip2.py#L287
+            eye = torch.eye(logits_per_text.size(0), device=logits_per_text.device)
+            m1_diag1 = -torch.ones_like(logits_per_text) + 2 * eye
+            loglik = torch.nn.functional.logsigmoid(m1_diag1 * logits_per_text)
+            nll = -torch.sum(loglik, dim=-1)
+            loss = nll.mean()
+
+        return Fgclip2Output(
+            loss=loss,
+            logits_per_image=logits_per_image,
+            logits_per_text=logits_per_text,
+            text_embeds=text_embeds,
+            image_embeds=image_embeds,
+            text_model_output=text_outputs,
+            vision_model_output=vision_outputs,
+        )
+
+    # New function: Acquire dense visual features of images with support for dynamic resolution
+    @filter_out_non_signature_kwargs()
+    @auto_docstring
+    def get_image_dense_feature(
+        self,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_attention_mask: Optional[torch.Tensor] = None,
+        spatial_shapes: Optional[torch.LongTensor] = None,
+    ) -> torch.FloatTensor:
+        r"""
+        Extract dense visual features from input images by forwarding through the vision backbone.
+
+        Args:
+            pixel_values (`torch.FloatTensor`):
+                Pixel values of shape (batch_size, max_num_patches, num_channels * patch_size * patch_size)
+            pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
+                Mask to avoid performing attention on padding pixel indices.
+            spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
+                Tensor containing the spatial dimensions (height, width) of the input images.
+
+        Returns:
+            `torch.FloatTensor` of shape  `(batch_size, max_num_patches, hidden_size)`:
+
+        """
+
+        vision_outputs: BaseModelOutputWithPooling = self.vision_model(
+            pixel_values=pixel_values,
+            attention_mask=pixel_attention_mask,
+            spatial_shapes=spatial_shapes,
+        )
+
+        probe = vision_outputs.last_hidden_state
+        hidden_state = vision_outputs.last_hidden_state
+        attention_mask = pixel_attention_mask
+
+        if attention_mask is not None:
+            target_len, source_len = probe.shape[1], hidden_state.shape[1]
+            attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_state.dtype, target_len)
+            attention_mask = attention_mask.repeat(1, self.dense_feature_head.num_heads, 1, 1)
+            attention_mask = attention_mask.reshape(-1, target_len, source_len)
+
+        hidden_state = self.dense_feature_head.attention(probe, hidden_state, hidden_state, attn_mask=attention_mask)[
+            0
+        ]
+        residual = hidden_state
+        hidden_state = self.dense_feature_head.layernorm(hidden_state)
+        hidden_state = residual + self.dense_feature_head.mlp(hidden_state)
+        feature_map = hidden_state
+
+        return feature_map
+
+    # New function: Acquire local features of images, applicable to retrieval, classification, and localization, with support for dynamic resolution
+    @filter_out_non_signature_kwargs()
+    @auto_docstring
+    def get_image_region_features(
+        self,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_attention_mask: Optional[torch.Tensor] = None,
+        spatial_shapes: Optional[torch.LongTensor] = None,
+        image_sizes: Optional[list[tuple]] = None,
+        region_infos: Optional[list[list[list[float]]]] = None,
+    ) -> list[torch.FloatTensor]:
+        r"""
+        Extract region-of-interest (RoI) features from images using RoI Align.
+        This method supports batched processing of variable-sized images and allows feature extraction
+        from user-specified image regions.
+
+        The input can be either a full image with corresponding region coordinates.
+        Features are extracted per region (e.g., bounding boxes), making this function suitable for tasks such as
+        object detection, referring expression grounding, or vision-language alignment.
+
+        Args:
+            pixel_values (`torch.FloatTensor`):
+                Pixel values of shape (batch_size, max_num_patches, num_channels * patch_size * patch_size)
+            pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
+                Mask to avoid performing attention on padding pixel indices.
+            spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
+                Tensor containing the spatial dimensions (height, width) of the input images.
+            image_sizes (`List[tuple]`, optional, each tuple of form `(int, int)`):
+                Original size (height, width) of each image in the batch before padding or resizing.
+                Required for accurate coordinate projection when region_infos are defined in original image space.
+            region_infos (`List[List[List[float]]]`, optional):
+                Bounding box coordinates for regions of interest in each image. Format:
+                - Outer list: length `batch_size`
+                - Middle list: number of regions per image
+                - Inner list: each contains `[x_min, y_min, x_max, y_max]` in **absolute pixel coordinates**
+                relative to the original image size (as specified in `image_sizes`).
+                These boxes are projected to feature map space using `image_sizes` and `spatial_shapes`,
+                then used to pool features via RoI Align or equivalent.
+
+        Returns:
+            `List[torch.FloatTensor]`:
+                A list of length `batch_size`, where each element is a tensor of shape
+                `(num_boxes, hidden_dim)` containing the extracted visual features for each region
+                in the corresponding image.
+        Example::
+            >>> # For a batch of 2 images
+            >>> region_features = model.get_image_region_features(
+            >>>     pixel_values=pixel_values,
+            >>>     image_sizes=[(640, 480), (480, 640)],
+            >>>     region_infos=[
+            >>>         [[100, 100, 200, 200], [300, 300, 400, 400]],  # 2 boxes in first image
+            >>>         [[50, 50, 150, 150]]                          # 1 box in second image
+            >>>     ]
+            >>> )
+            >>> print(region_features[0].shape)  # torch.Size([2, hidden_dim])
+            >>> print(region_features[1].shape)  # torch.Size([1, hidden_dim])
+
+        """
+        if region_infos is None or len(region_infos) == 0:
+            return []
+
+        # Get dense feature maps: (B, N, D)
+        dense_feature_map = self.get_image_dense_feature(
+            pixel_values=pixel_values,
+            pixel_attention_mask=pixel_attention_mask,
+            spatial_shapes=spatial_shapes,
+        )
+        bs, _, hidden_dim = dense_feature_map.shape
+
+        all_region_features = []
+
+        for i in range(bs):
+            h, w = spatial_shapes[i].tolist()
+            img_h, img_w = image_sizes[i]
+            bboxes = region_infos[i]
+
+            if not bboxes:
+                all_region_features.append(torch.empty(0, hidden_dim, device=dense_feature_map.device))
+                continue
+
+            # Reshape to (1, C, H', W')
+            num_valid = h * w
+            feat_seq = dense_feature_map[i, :num_valid]  # (num_valid, D)
+            feat_map = feat_seq.view(h, w, hidden_dim).permute(2, 0, 1).unsqueeze(0)  # (1, D, H', W')
+
+            # Normalize bboxes to feature map coordinates
+            rois = []
+            for x1, y1, x2, y2 in bboxes:
+                nx1 = (x1 / img_w) * w
+                ny1 = (y1 / img_h) * h
+                nx2 = (x2 / img_w) * w
+                ny2 = (y2 / img_h) * h
+                rois.append([0, nx1, ny1, nx2, ny2])  #
+            rois_tensor = torch.tensor(rois, dtype=torch.float32, device=feat_map.device)  # (N, 5)
+
+            # RoI Align on single image
+            pooled = roi_align(
+                input=feat_map,
+                boxes=rois_tensor,
+                output_size=(1, 1),
+                spatial_scale=1.0,
+                sampling_ratio=-1,
+                aligned=True,
+            )  # (N, D, 1, 1)
+            region_feats = pooled.squeeze(-1).squeeze(-1)  # (N, D)
+
+            all_region_features.append(region_feats)
+
+        return all_region_features
+
+
+__all__ = ["Fgclip2Model", "Fgclip2PreTrainedModel", "Fgclip2TextModel", "Fgclip2VisionModel"]

fg-clip2-large/preprocessor_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1125703e5446d5b6ff4d5893a33bac128cdd21dc12e3dad2469a648fb0ae3bf7
+size 393

fg-clip2-large/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baec30ea10906f16adb8c18af7a34023002c1746542612b8b41c9f09e1351351
+size 636

fg-clip2-large/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58a1696e79c9d97937389ed116f552a15c84811d7b8023918b86f4bc5775b1b0
+size 34356304

fg-clip2-large/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

fg-clip2-large/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607049b4766b302a66e31906a6b4729e01f623597c82262fb52e552f0d4f7fb4
+size 40160

src/export_onnx.py

@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+"""Export FG-CLIP 2 Large (HuggingFace remote-code format) to ONNX.
+
+Produces two standalone graphs:
+  * image encoder: pixel_values[1,N,768] -> image_features[1,1024]
+  * text  encoder: input_ids[1,L]       -> text_features[1,1024]
+
+Input contract (matches the official processor / tokenizer):
+  image : Siglip2 patchified input. ``Siglip2ImageProcessor`` returns
+          ``pixel_values`` of shape [1, num_patches, 3*patch_size*patch_size].
+          This export fixes a square grid (default 16x16 = 256 patches, i.e. a
+          256x256 image), so only ``pixel_values`` is exposed as a graph input.
+  text  : int64 token ids padded to a fixed length. 64 for walk_type
+          ``short``/``box``, 196 for ``long``. The official tokenizer does not
+          emit an attention mask, so the graph takes ids only.
+
+walk_type selects which text head is baked into the graph:
+  short -> text_model's own projection head (default, retrieval)
+  long  -> longtext_head (long captions)
+  box   -> boxtext_head  (dense / region descriptions)
+
+Usage:
+  python src/export_onnx.py
+  python src/export_onnx.py --walk-type long --text-length 196
+"""
+
+import argparse
+import json
+import os
+import sys
+
+import numpy as np
+import torch
+import torch.nn as nn
+from transformers import AutoModelForCausalLM
+
+DEFAULT_MODEL_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "fg-clip2-large"
+)
+DEFAULT_OUTPUT_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "models"
+)
+
+
+class ImageEncoder(nn.Module):
+    """Vision tower with a fixed grid.
+
+    ``Fgclip2VisionEmbeddings`` resizes its positional table with
+    ``antialias=True``, which maps to ``aten::_upsample_bilinear2d_aa`` and has
+    no ONNX symbolic. The resize is an identity when the target grid equals the
+    table grid (the default 16x16 case), so the table is baked in directly.
+    For other grids we fall back to bilinear interpolation without antialias;
+    that differs slightly from the PyTorch default, see export_info.json.
+    """
+
+    def __init__(self, model, grid_h: int, grid_w: int):
+        super().__init__()
+        self.vision_model = model.vision_model
+        emb = model.vision_model.embeddings
+        table = emb.position_embedding.weight.reshape(
+            emb.position_embedding_size, emb.position_embedding_size, -1
+        )
+        if (grid_h, grid_w) != (emb.position_embedding_size, emb.position_embedding_size):
+            import torch.nn.functional as F
+
+            resized = F.interpolate(
+                table.permute(2, 0, 1).unsqueeze(0),
+                size=(grid_h, grid_w),
+                mode="bilinear",
+                align_corners=False,
+                antialias=False,
+            )
+            table = resized.reshape(table.shape[-1], -1).transpose(0, 1)
+        self.register_buffer("pos_table", table.reshape(1, -1, table.shape[-1]).clone(), persistent=False)
+
+    def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        vm = self.vision_model
+        hidden = vm.embeddings.patch_embedding(pixel_values) + self.pos_table
+        hidden = vm.encoder(inputs_embeds=hidden, attention_mask=None).last_hidden_state
+        hidden = vm.post_layernorm(hidden)
+        return vm.head(hidden, attention_mask=None)
+
+
+class TextEncoder(nn.Module):
+    def __init__(self, model, walk_type: str):
+        super().__init__()
+        self.text_model = model.text_model
+        self.walk_type = walk_type
+        if walk_type == "long":
+            self.out_head = model.longtext_head
+        elif walk_type == "box":
+            self.out_head = model.boxtext_head
+        else:
+            self.out_head = None
+
+    def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+        pooled = self.text_model(input_ids=input_ids, walk_type=self.walk_type).pooler_output
+        if self.out_head is not None:
+            pooled = self.out_head(pooled)
+        return pooled
+
+
+def build_model(model_dir: str):
+    model = AutoModelForCausalLM.from_pretrained(
+        model_dir,
+        trust_remote_code=True,
+        local_files_only=True,
+    )
+    model.eval()
+    model.requires_grad_(False)
+
+    # The remote config only propagates the top-level attention backend, and
+    # transformers 5 defaults sub-configs to sdpa; force eager everywhere.
+    model.config._attn_implementation = "eager"
+    model.config.vision_config._attn_implementation = "eager"
+    model.config.text_config._attn_implementation = "eager"
+    model.vision_model.config._attn_implementation = "eager"
+    model.text_model.config._attn_implementation = "eager"
+
+    # transformers 5 builds non-persistent buffers / plain attributes under a
+    # meta-device context. position_ids, mask1 and mask2 are never in the
+    # checkpoint, so rebuild them explicitly (same values as __init__).
+    emb = model.text_model.embeddings
+    cfg = model.config.text_config
+    idx = torch.arange(cfg.longtext_len)
+    emb.position_ids = idx.unsqueeze(0)
+    emb.mask1 = (idx.unsqueeze(1) < cfg.keep_len).to(torch.float32)
+    emb.mask2 = (idx.unsqueeze(1) >= cfg.keep_len).to(torch.float32)
+
+    stale = [n for n, b in model.named_buffers() if b.is_meta]
+    stale += [n for n, p in model.named_parameters() if p.is_meta]
+    if stale:
+        raise RuntimeError(f"unresolved meta tensors after load: {stale[:10]}")
+    return model
+
+
+def export_graph(module, args, path, input_names, output_names, opset):
+    print(f"[export] {path}")
+    with torch.no_grad():
+        torch.onnx.export(
+            module,
+            args,
+            path,
+            input_names=input_names,
+            output_names=output_names,
+            opset_version=opset,
+            do_constant_folding=False,
+            export_params=True,
+            dynamo=False,
+        )
+
+
+def consolidate_external_data(onnx_path):
+    """Merge per-tensor side-car weights into a single <model>.onnx.data file."""
+    import onnx
+
+    model = onnx.load(onnx_path, load_external_data=False)
+    locations = set()
+    for tensor in model.graph.initializer:
+        if tensor.data_location != onnx.TensorProto.EXTERNAL:
+            continue
+        for kv in tensor.external_data:
+            if kv.key == "location":
+                locations.add(kv.value)
+    if not locations:
+        return None
+
+    base = os.path.dirname(onnx_path)
+    onnx.external_data_helper.load_external_data_for_model(model, base)
+
+    data_name = os.path.basename(onnx_path) + ".data"
+    onnx.save_model(
+        model,
+        onnx_path,
+        save_as_external_data=True,
+        all_tensors_to_one_file=True,
+        location=data_name,
+        size_threshold=1024,
+    )
+
+    for location in locations:
+        stale = os.path.join(base, location)
+        if os.path.isfile(stale) and os.path.basename(stale) != data_name:
+            os.remove(stale)
+
+    return os.path.join(base, data_name)
+
+
+def verify(onnx_path, module, inputs, output_name, atol, rtol):
+    import onnxruntime as ort
+
+    session = ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])
+    feed = {node.name: x.numpy() for node, x in zip(session.get_inputs(), inputs)}
+    ort_out = session.run([output_name], feed)[0]
+
+    with torch.no_grad():
+        ref = module(*inputs).numpy()
+
+    abs_err = np.abs(ort_out - ref)
+    max_abs = float(abs_err.max())
+    rel = max_abs / max(float(np.abs(ref).max()), 1e-12)
+    ok = np.allclose(ort_out, ref, atol=atol, rtol=rtol)
+    print(
+        f"[verify] {os.path.basename(onnx_path)}: max_abs={max_abs:.3e} "
+        f"rel={rel:.3e} shape={ort_out.shape} -> {'OK' if ok else 'FAIL'}"
+    )
+    return ok
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("--model-dir", default=DEFAULT_MODEL_DIR)
+    parser.add_argument("--output-dir", default=DEFAULT_OUTPUT_DIR)
+    parser.add_argument(
+        "--image-size",
+        type=int,
+        default=256,
+        help="square input size; must be divisible by patch_size (default 256)",
+    )
+    parser.add_argument("--walk-type", choices=["short", "long", "box"], default="short")
+    parser.add_argument(
+        "--text-length",
+        type=int,
+        default=0,
+        help="fixed token length (0 = 64 for short/box, 196 for long)",
+    )
+    parser.add_argument("--opset", type=int, default=14)
+    parser.add_argument("--no-verify", action="store_true")
+    parser.add_argument("--atol", type=float, default=1e-4)
+    parser.add_argument("--rtol", type=float, default=1e-3)
+    args = parser.parse_args()
+
+    model_dir = os.path.abspath(args.model_dir)
+    output_dir = os.path.abspath(args.output_dir)
+    os.makedirs(output_dir, exist_ok=True)
+
+    print(f"[load] {model_dir}")
+    model = build_model(model_dir)
+    vision_cfg = model.config.vision_config
+    text_cfg = model.config.text_config
+
+    patch_size = vision_cfg.patch_size
+    if args.image_size % patch_size != 0:
+        raise SystemExit(f"image size {args.image_size} is not divisible by patch_size {patch_size}")
+    grid = args.image_size // patch_size
+    num_patches = grid * grid
+    if num_patches > vision_cfg.num_patches:
+        raise SystemExit(
+            f"grid {grid}x{grid} needs {num_patches} patches, model supports {vision_cfg.num_patches}"
+        )
+
+    text_length = args.text_length or (text_cfg.longtext_len if args.walk_type == "long" else text_cfg.max_position_embeddings)
+
+    image_encoder = ImageEncoder(model, grid, grid).eval()
+    text_encoder = TextEncoder(model, args.walk_type).eval()
+
+    image_input = torch.randn(
+        1, num_patches, vision_cfg.num_channels * patch_size * patch_size
+    )
+    text_input = torch.randint(0, text_cfg.vocab_size, (1, text_length), dtype=torch.int64)
+
+    image_path = os.path.join(output_dir, "fgclip2_large_image_encoder.onnx")
+    text_path = os.path.join(output_dir, f"fgclip2_large_text_encoder_{args.walk_type}.onnx")
+
+    export_graph(
+        image_encoder,
+        (image_input,),
+        image_path,
+        ["pixel_values"],
+        ["image_features"],
+        args.opset,
+    )
+    export_graph(
+        text_encoder,
+        (text_input,),
+        text_path,
+        ["input_ids"],
+        ["text_features"],
+        args.opset,
+    )
+
+    image_data = consolidate_external_data(image_path)
+    text_data = consolidate_external_data(text_path)
+
+    feature_dim = text_cfg.projection_size
+    info = {
+        "model_dir": model_dir,
+        "opset": args.opset,
+        "walk_type": args.walk_type,
+        "image_encoder": {
+            "file": os.path.basename(image_path),
+            "external_data": os.path.basename(image_data) if image_data else None,
+            "input": {
+                "name": "pixel_values",
+                "shape": [1, num_patches, vision_cfg.num_channels * patch_size * patch_size],
+            },
+            "output": {"name": "image_features", "shape": [1, feature_dim]},
+            "preprocess": {
+                "image_size": args.image_size,
+                "patch_size": patch_size,
+                "grid": [grid, grid],
+                "mean": [0.5, 0.5, 0.5],
+                "std": [0.5, 0.5, 0.5],
+            },
+        },
+        "text_encoder": {
+            "file": os.path.basename(text_path),
+            "external_data": os.path.basename(text_data) if text_data else None,
+            "input": {"name": "input_ids", "shape": [1, text_length], "dtype": "int64"},
+            "output": {"name": "text_features", "shape": [1, feature_dim]},
+            "walk_type": args.walk_type,
+        },
+    }
+    info_path = os.path.join(output_dir, "export_info.json")
+    with open(info_path, "w", encoding="utf-8") as f:
+        json.dump(info, f, indent=2)
+    print(f"[info] {info_path}")
+
+    if not args.no_verify:
+        ok_image = verify(image_path, image_encoder, [image_input], "image_features", args.atol, args.rtol)
+        ok_text = verify(text_path, text_encoder, [text_input], "text_features", args.atol, args.rtol)
+        if not (ok_image and ok_text):
+            print("[error] verification failed", file=sys.stderr)
+            return 1
+
+    for path, data in ((image_path, image_data), (text_path, text_data)):
+        total = os.path.getsize(path) + (os.path.getsize(data) if data else 0)
+        suffix = f" + {os.path.basename(data)}" if data else ""
+        print(f"[done] {path}{suffix} ({total / 1024**2:.1f} MiB)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/gen_image_calibration_data.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Generate Pulsar2 "Numpy" calibration data for the FG-CLIP 2 image encoder.
+
+The image encoder ONNX takes ``pixel_values[1, 256, 768]`` — a patchified
+Siglip2 input, not an NCHW image. Pulsar2's ``Image`` calibration path cannot
+produce it (its input processors only resize/normalize/layout-convert), so the
+samples are preprocessed here and saved as ``.npy`` files.
+
+Preprocess matches Siglip2ImageProcessor: RGB, BILINEAR resize to a square
+``image_size``, /255, (x-0.5)/0.5, then 16x16 patches in (nh, nw, p, p, C)
+order.
+"""
+
+import argparse
+import os
+import zipfile
+
+import numpy as np
+from PIL import Image
+
+PATCH_SIZE = 16
+DEFAULT_IMAGES = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "datasets",
+    "coco2017val_500",
+)
+DEFAULT_OUTPUT_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "datasets",
+    "fgclip2_image_calibration",
+)
+IMAGE_SUFFIXES = (".jpg", ".jpeg", ".png", ".bmp", ".webp")
+
+
+def list_images(directory, limit):
+    names = sorted(n for n in os.listdir(directory) if n.lower().endswith(IMAGE_SUFFIXES))
+    if limit:
+        names = names[:limit]
+    if not names:
+        raise SystemExit(f"no images found in {directory}")
+    return names
+
+
+def preprocess(path, image_size):
+    image = Image.open(path).convert("RGB")
+    resample = getattr(Image, "Resampling", Image).BILINEAR
+    image = image.resize((image_size, image_size), resample)
+    x = np.asarray(image, dtype=np.float32) / 255.0
+    x = (x - 0.5) / 0.5
+    grid = image_size // PATCH_SIZE
+    patches = x.reshape(grid, PATCH_SIZE, grid, PATCH_SIZE, 3)
+    patches = patches.transpose(0, 2, 1, 3, 4).reshape(1, grid * grid, PATCH_SIZE * PATCH_SIZE * 3)
+    return np.ascontiguousarray(patches)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--images", default=DEFAULT_IMAGES)
+    parser.add_argument("--output-dir", default=DEFAULT_OUTPUT_DIR)
+    parser.add_argument("--zip", dest="zip_path",
+                        default=os.path.join(os.path.dirname(DEFAULT_OUTPUT_DIR),
+                                             "fgclip2_image_calibration.zip"))
+    parser.add_argument("--image-size", type=int, default=256,
+                        help="must be a multiple of 16 and match the exported grid")
+    parser.add_argument("--limit", type=int, default=64, help="max images (0 = all)")
+    parser.add_argument("--no-zip", action="store_true")
+    args = parser.parse_args()
+
+    images_dir = os.path.abspath(args.images)
+    output_dir = os.path.abspath(args.output_dir)
+    zip_path = os.path.abspath(args.zip_path)
+    if args.image_size % PATCH_SIZE != 0:
+        raise SystemExit(f"image size {args.image_size} is not divisible by {PATCH_SIZE}")
+
+    names = list_images(images_dir, args.limit)
+    os.makedirs(output_dir, exist_ok=True)
+    for idx, name in enumerate(names):
+        arr = preprocess(os.path.join(images_dir, name), args.image_size)
+        np.save(os.path.join(output_dir, f"{idx:06d}.npy"), arr)
+    print(f"[write] {len(names)} npy files -> {output_dir}")
+
+    manifest = output_dir.rstrip(os.sep) + "_manifest.txt"
+    with open(manifest, "w", encoding="utf-8") as f:
+        for idx, name in enumerate(names):
+            f.write(f"{idx:06d}.npy\t{name}\n")
+    print(f"[write] manifest -> {manifest}")
+
+    if not args.no_zip:
+        folder = os.path.basename(output_dir.rstrip(os.sep))
+        os.makedirs(os.path.dirname(zip_path), exist_ok=True)
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for name in sorted(os.listdir(output_dir)):
+                if name.endswith(".npy"):
+                    zf.write(os.path.join(output_dir, name), f"{folder}/{name}")
+        print(f"[zip] {zip_path} ({os.path.getsize(zip_path) / 1024 ** 2:.2f} MiB)")
+
+    sample = preprocess(os.path.join(images_dir, names[0]), args.image_size)
+    print(f"[info] sample shape={sample.shape} dtype={sample.dtype} "
+          f"min={sample.min():.4f} max={sample.max():.4f}")
+
+
+if __name__ == "__main__":
+    main()

src/gen_text_calibration_data.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Generate Pulsar2 "Numpy" calibration data for the FG-CLIP 2 text encoder.
+
+The text encoder ONNX takes ``input_ids[1, 64]`` int64 token ids (Gemma BPE),
+so calibration samples are tokenized captions saved as ``.npy`` files and
+packed into a zip archive (``calibration_format: "Numpy"``).
+
+Do not reuse the Chinese-CLIP calibration zip: it is BertTokenizer output with
+shape [1, 52] and will fail with a shape mismatch.
+
+Default corpus: the human-written raw Chinese captions from coco-cn
+(``imageid.human-written-caption.txt``), read directly from the dataset tar.gz.
+"""
+
+import argparse
+import io
+import os
+import tarfile
+import zipfile
+
+import numpy as np
+from transformers import AutoTokenizer
+
+
+DEFAULT_CAPTIONS = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    "..", "..", "..", "datasets", "coco-cn-version1805v1.1.tar.gz",
+)
+DEFAULT_MEMBER = "coco-cn-version1805v1.1/imageid.human-written-caption.txt"
+DEFAULT_MODEL_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "fg-clip2-large"
+)
+DEFAULT_OUTPUT_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "datasets",
+    "fgclip2_text_calibration",
+)
+
+
+def read_lines(path, member=None):
+    """Yield text lines from a plain file or a tar/tar.gz member."""
+    if tarfile.is_tarfile(path):
+        with tarfile.open(path, "r:*") as tar:
+            names = [n for n in tar.getnames() if not os.path.basename(n).startswith("._")]
+            if member is None:
+                member = next(n for n in names if n.endswith(".txt") and "caption" in n)
+            if member not in names:
+                raise SystemExit(f"member not found in {path}: {member}")
+            f = tar.extractfile(member)
+            for raw in io.TextIOWrapper(f, encoding="utf-8"):
+                yield raw.rstrip("\n")
+    else:
+        with open(path, encoding="utf-8") as f:
+            for raw in f:
+                yield raw.rstrip("\n")
+
+
+def parse_caption(line):
+    """Split ``COCO_val2014_000000043734#0<tab>两只黑色的狗...`` into (key, text)."""
+    line = line.strip()
+    if not line:
+        return None
+    if "\t" in line:
+        key, text = line.split("\t", 1)
+    else:
+        parts = line.split(None, 1)
+        if len(parts) != 2:
+            return None
+        key, text = parts
+    text = text.strip()
+    return (key, text) if text else None
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--captions", default=DEFAULT_CAPTIONS,
+                        help="plain caption .txt or dataset .tar/.tar.gz")
+    parser.add_argument("--member", default=DEFAULT_MEMBER,
+                        help="member path inside the archive")
+    parser.add_argument("--model-dir", default=DEFAULT_MODEL_DIR,
+                        help="directory holding tokenizer.json")
+    parser.add_argument("--output-dir", default=DEFAULT_OUTPUT_DIR)
+    parser.add_argument("--zip", dest="zip_path",
+                        default=os.path.join(os.path.dirname(DEFAULT_OUTPUT_DIR),
+                                             "fgclip2_text_calibration.zip"))
+    parser.add_argument("--text-length", type=int, default=64)
+    parser.add_argument("--limit", type=int, default=512,
+                        help="max samples to write, 0 means all")
+    parser.add_argument("--no-zip", action="store_true")
+    args = parser.parse_args()
+
+    captions_path = os.path.abspath(args.captions)
+    model_dir = os.path.abspath(args.model_dir)
+    output_dir = os.path.abspath(args.output_dir)
+    zip_path = os.path.abspath(args.zip_path)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True)
+
+    records = []
+    for line in read_lines(captions_path, args.member):
+        item = parse_caption(line)
+        if item is not None:
+            records.append(item)
+        if args.limit and len(records) >= args.limit:
+            break
+    if not records:
+        raise SystemExit(f"no captions parsed from {captions_path}")
+    print(f"[read] {len(records)} captions from {captions_path}")
+
+    os.makedirs(output_dir, exist_ok=True)
+    _, texts = zip(*records)
+    enc = tokenizer(
+        list(texts),
+        padding="max_length",
+        max_length=args.text_length,
+        truncation=True,
+        return_tensors="np",
+    )
+    input_ids = enc["input_ids"]
+    assert input_ids.shape == (len(records), args.text_length), input_ids.shape
+
+    for idx in range(len(records)):
+        arr = input_ids[idx:idx + 1].astype(np.int64)
+        np.save(os.path.join(output_dir, f"{idx:06d}.npy"), arr)
+
+    manifest = output_dir.rstrip(os.sep) + "_manifest.txt"
+    with open(manifest, "w", encoding="utf-8") as f:
+        for idx, (key, text) in enumerate(records):
+            f.write(f"{idx:06d}.npy\t{key}\t{text}\n")
+    print(f"[write] {len(records)} npy files -> {output_dir}")
+    print(f"[write] manifest -> {manifest}")
+
+    if not args.no_zip:
+        folder = os.path.basename(output_dir.rstrip(os.sep))
+        os.makedirs(os.path.dirname(zip_path), exist_ok=True)
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for name in sorted(os.listdir(output_dir)):
+                if name.endswith(".npy"):
+                    zf.write(os.path.join(output_dir, name), f"{folder}/{name}")
+        print(f"[zip] {zip_path} ({os.path.getsize(zip_path) / 1024 ** 2:.2f} MiB)")
+
+    print(f"[info] sample shape={input_ids[0:1].shape} dtype=int64 "
+          f"ids[:8]={input_ids[0][:8].tolist()}")
+
+
+if __name__ == "__main__":
+    main()

src/infer/fgclip2_infer_axmodel.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""Run the FG-CLIP 2 Large encoders on AX650 (axmodel).
+
+This script is meant for the BOARD. It only needs axengine + numpy + Pillow;
+it never imports onnxruntime, torch or transformers.
+
+Two modes:
+
+  single  --image PATH --texts ...      quick sanity check, prints similarity
+  dataset --dataset DIR                 batch inference, writes raw features
+
+Dataset layout:
+  <dataset>/images/*.jpg
+  <dataset>/pairs.jsonl     # one JSON object per line: {"image": "...", "text": "..."}
+
+Input contract (verified against the compiled graph):
+  image : patchified Siglip2 input [1, num_patches, 3*patch*patch].
+          resize(RGB, BILINEAR) to image-size -> /255 -> (x-0.5)/0.5 -> 16x16 patches.
+  text  : int64 token ids [1,L] (L=64 for walk_type "short"). Input dtype is
+          read from the runtime, i.e. it may become int32 after pulsar2
+          rewrites the ONNX int64 input.
+
+Usage:
+  # single image
+  python fgclip2_infer_axmodel.py \
+      --image-model fgclip2_large_image_encoder.axmodel \
+      --text-model  fgclip2_large_text_encoder_short.axmodel \
+      --tokenizer tokenizer.json --image bedroom.jpg \
+      --texts 一个简约风格的卧室角落 一张铺着白色床单的床
+
+  # dataset
+  python fgclip2_infer_axmodel.py \
+      --image-model fgclip2_large_image_encoder.axmodel \
+      --text-model  fgclip2_large_text_encoder_short.axmodel \
+      --tokenizer tokenizer.json \
+      --dataset fgclip_eval_64 --output results/axmodel.npz
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+import numpy as np
+from PIL import Image
+
+try:
+    from tqdm import tqdm
+except ImportError:  # board image may not ship tqdm
+    tqdm = None
+
+
+PATCH_SIZE = 16
+IMAGE_MEAN = 0.5
+IMAGE_STD = 0.5
+DEFAULT_TEXTS = ["一个简约风格的卧室角落，黑色金属衣架上挂着衣物。", "一只猫在窗台上晒太阳。"]
+BACKEND = "axmodel"
+
+
+# ---------------------------------------------------------------------------
+# Gemma BPE tokenizer (reads tokenizer.json; mirrors the HF fast tokenizer)
+# ---------------------------------------------------------------------------
+
+class GemmaBpeTokenizer:
+    def __init__(self, tokenizer_file, pad_id=0, eos_id=1, unk_id=3):
+        with open(tokenizer_file, encoding="utf-8") as f:
+            data = json.load(f)
+        self.vocab = data["model"]["vocab"]
+        self.rank = {}
+        for rank, pair in enumerate(data["model"]["merges"]):
+            self.rank[(pair[0], pair[1])] = rank
+        added = [(a["content"], a["id"]) for a in data.get("added_tokens", [])]
+        added.sort(key=lambda x: -len(x[0]))
+        self.added = added
+        self.pad_id = pad_id
+        self.eos_id = eos_id
+        self.unk_id = unk_id
+
+    def _byte_fallback(self, ch):
+        out = []
+        for b in ch.encode("utf-8"):
+            token = f"<0x{b:02X}>"
+            out.append(token if token in self.vocab else None)
+        return out
+
+    def _bpe(self, segment):
+        symbols = []
+        for ch in segment:
+            if ch in self.vocab:
+                symbols.append(ch)
+            else:
+                for token in self._byte_fallback(ch):
+                    symbols.append(token if token is not None else "<unk>")
+
+        while len(symbols) > 1:
+            best_rank, best_i = None, -1
+            for i in range(len(symbols) - 1):
+                rank = self.rank.get((symbols[i], symbols[i + 1]))
+                if rank is not None and (best_rank is None or rank < best_rank):
+                    best_rank, best_i = rank, i
+            if best_rank is None:
+                break
+
+            pair = (symbols[best_i], symbols[best_i + 1])
+            merged, i = [], 0
+            while i < len(symbols):
+                if i + 1 < len(symbols) and (symbols[i], symbols[i + 1]) == pair:
+                    merged.append(symbols[i] + symbols[i + 1])
+                    i += 2
+                else:
+                    merged.append(symbols[i])
+                    i += 1
+            symbols = merged
+        return symbols
+
+    def encode(self, text, max_length, dtype=np.int64):
+        text = text.replace(" ", "▁")
+        ids, buf, i = [], "", 0
+        while i < len(text):
+            hit = None
+            for content, token_id in self.added:
+                if text.startswith(content, i):
+                    hit = (content, token_id)
+                    break
+            if hit is not None:
+                if buf:
+                    ids.extend(self.vocab.get(t, self.unk_id) for t in self._bpe(buf))
+                    buf = ""
+                ids.append(hit[1])
+                i += len(hit[0])
+            else:
+                buf += text[i]
+                i += 1
+        if buf:
+            ids.extend(self.vocab.get(t, self.unk_id) for t in self._bpe(buf))
+
+        ids = (ids + [self.eos_id])[:max_length]
+        ids += [self.pad_id] * (max_length - len(ids))
+        return np.asarray(ids, dtype=dtype).reshape(1, max_length)
+
+
+# ---------------------------------------------------------------------------
+# Runtime helpers
+# ---------------------------------------------------------------------------
+
+def create_session(path):
+    import axengine
+    return axengine.InferenceSession(path)
+
+
+def input_name_and_dtype(sess):
+    node = sess.get_inputs()[0]
+    dtype = getattr(node, "dtype", None)
+    if not isinstance(dtype, np.dtype):
+        text = str(getattr(node, "type", ""))
+        name = text.split("(")[-1].rstrip(")")
+        dtype = np.dtype({
+            "float": np.float32, "double": np.float64, "float16": np.float16,
+            "int8": np.int8, "uint8": np.uint8, "int16": np.int16,
+            "uint16": np.uint16, "int32": np.int32, "uint32": np.uint32,
+            "int64": np.int64,
+        }.get(name, name))
+    return node.name, dtype
+
+
+def run_feature(sess, x):
+    name, dtype = input_name_and_dtype(sess)
+    x = np.ascontiguousarray(x, dtype=dtype)
+    start = time.perf_counter()
+    out = sess.run(None, {name: x})[0]
+    elapsed = (time.perf_counter() - start) * 1000.0
+    return np.asarray(out, dtype=np.float32).reshape(-1), elapsed
+
+
+def preprocess_image(path, image_size):
+    image = Image.open(path).convert("RGB")
+    resample = getattr(Image, "Resampling", Image).BILINEAR
+    image = image.resize((image_size, image_size), resample)
+    x = np.asarray(image, dtype=np.float32) / 255.0
+    x = (x - IMAGE_MEAN) / IMAGE_STD
+    grid = image_size // PATCH_SIZE
+    patches = x.reshape(grid, PATCH_SIZE, grid, PATCH_SIZE, 3)
+    patches = patches.transpose(0, 2, 1, 3, 4).reshape(1, grid * grid, PATCH_SIZE * PATCH_SIZE * 3)
+    return np.ascontiguousarray(patches)
+
+
+def normalize(x):
+    return x / (np.linalg.norm(x, axis=-1, keepdims=True) + 1e-12)
+
+
+def softmax(x):
+    x = x - np.max(x)
+    e = np.exp(x)
+    return e / np.sum(e)
+
+
+# ---------------------------------------------------------------------------
+# Dataset
+# ---------------------------------------------------------------------------
+
+def load_pairs(dataset_dir, pairs_file, limit):
+    pairs_path = pairs_file or os.path.join(dataset_dir, "pairs.jsonl")
+    if not os.path.isfile(pairs_path):
+        raise SystemExit(f"pairs file not found: {pairs_path}")
+    pairs = []
+    with open(pairs_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            item = json.loads(line)
+            pairs.append((item["image"], item["text"]))
+            if limit and len(pairs) >= limit:
+                break
+    if not pairs:
+        raise SystemExit(f"no pairs in {pairs_path}")
+    return pairs
+
+
+def group_pairs(pairs):
+    """Deduplicate images, keep the first-seen order, map each text to an image."""
+    image_names = []
+    image_index = {}
+    text_captions = []
+    text_image_index = []
+    for image_name, text in pairs:
+        if image_name not in image_index:
+            image_index[image_name] = len(image_names)
+            image_names.append(image_name)
+        text_captions.append(text)
+        text_image_index.append(image_index[image_name])
+    return image_names, text_captions, np.asarray(text_image_index, dtype=np.int32)
+
+
+def save_dump(path, backend, image_model, text_model, image_names, text_captions,
+              text_image_index, image_features, text_features,
+              image_ms, text_ms, image_pre_ms, text_pre_ms):
+    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+    np.savez(
+        path,
+        backend=np.asarray(backend),
+        image_model=np.asarray(os.path.basename(image_model)),
+        text_model=np.asarray(os.path.basename(text_model)),
+        image_names=np.asarray(image_names),
+        text_captions=np.asarray(text_captions),
+        text_image_index=np.asarray(text_image_index, dtype=np.int32),
+        image_features=image_features,
+        text_features=text_features,
+        image_ms=np.asarray(image_ms, dtype=np.float32),
+        text_ms=np.asarray(text_ms, dtype=np.float32),
+        image_preprocess_ms=np.asarray(image_pre_ms, dtype=np.float32),
+        text_tokenize_ms=np.asarray(text_pre_ms, dtype=np.float32),
+    )
+    print(f"[save] {path}")
+
+
+# ---------------------------------------------------------------------------
+# Modes
+# ---------------------------------------------------------------------------
+
+def run_single(args, tokenizer, image_sess, text_sess):
+    start = time.perf_counter()
+    pre_start = time.perf_counter()
+    x = preprocess_image(args.image, args.image_size)
+    pre_ms = (time.perf_counter() - pre_start) * 1000.0
+    image_feature, image_ms = run_feature(image_sess, x)
+
+    text_features = []
+    text_ms = []
+    for text in args.texts:
+        tok_start = time.perf_counter()
+        ids = tokenizer.encode(text, args.text_length)
+        tok_ms = (time.perf_counter() - tok_start) * 1000.0
+        feat, ms = run_feature(text_sess, ids)
+        text_features.append(feat)
+        text_ms.append(ms)
+        if args.verbose:
+            print(f"[time] text tokenize={tok_ms:.2f}ms infer={ms:.2f}ms")
+    text_features = np.stack(text_features)
+    total_ms = (time.perf_counter() - start) * 1000.0
+
+    print(f"[time] preprocess={pre_ms:.2f}ms image_infer={image_ms:.2f}ms "
+          f"text_infer(avg)={np.mean(text_ms):.2f}ms total={total_ms:.2f}ms")
+
+    sim = normalize(image_feature[None])[0] @ normalize(text_features).T
+    probs = softmax(sim * 100.0)
+    best = int(np.argmax(probs))
+    print("similarity matrix (cosine):")
+    for i, text in enumerate(args.texts):
+        print(f"  {text:<40} sim={sim[i]:.4f} prob={probs[i]:.4f}")
+    print(f"[result] top-1: {args.texts[best]} (prob={probs[best]:.4f})")
+
+    if args.output:
+        save_dump(
+            args.output, BACKEND, args.image_model, args.text_model,
+            [os.path.basename(args.image)], args.texts, np.zeros(len(args.texts)),
+            image_feature[None], text_features,
+            [image_ms], text_ms, [pre_ms], [0.0] * len(args.texts),
+        )
+
+
+def run_dataset(args, tokenizer, image_sess, text_sess):
+    dataset_dir = os.path.abspath(args.dataset)
+    pairs = load_pairs(dataset_dir, args.pairs, args.limit)
+    image_names, text_captions, text_image_index = group_pairs(pairs)
+    print(f"[data] {len(image_names)} images, {len(text_captions)} captions")
+
+    image_iter = tqdm(image_names, desc="image") if tqdm else image_names
+    image_features = None
+    image_ms = []
+    image_pre_ms = []
+    for i, name in enumerate(image_iter):
+        pre_start = time.perf_counter()
+        x = preprocess_image(os.path.join(dataset_dir, "images", name), args.image_size)
+        image_pre_ms.append((time.perf_counter() - pre_start) * 1000.0)
+        feat, ms = run_feature(image_sess, x)
+        if image_features is None:
+            image_features = np.zeros((len(image_names), feat.shape[0]), dtype=np.float32)
+        image_features[i] = feat
+        image_ms.append(ms)
+
+    text_iter = tqdm(text_captions, desc="text") if tqdm else text_captions
+    text_features = np.zeros((len(text_captions), image_features.shape[1]), dtype=np.float32)
+    text_ms = []
+    text_pre_ms = []
+    for i, caption in enumerate(text_iter):
+        pre_start = time.perf_counter()
+        ids = tokenizer.encode(caption, args.text_length)
+        text_pre_ms.append((time.perf_counter() - pre_start) * 1000.0)
+        feat, ms = run_feature(text_sess, ids)
+        if feat.shape[0] != text_features.shape[1]:
+            raise SystemExit(f"feature size mismatch: "
+                             f"image={text_features.shape[1]} text={feat.shape[0]}")
+        text_features[i] = feat
+        text_ms.append(ms)
+
+    print(f"[time] image preprocess={np.mean(image_pre_ms):.2f}ms "
+          f"infer={np.mean(image_ms):.2f}ms/img; "
+          f"text tokenize={np.mean(text_pre_ms):.2f}ms "
+          f"infer={np.mean(text_ms):.2f}ms/text")
+    print(f"[time] image p50={np.percentile(image_ms, 50):.2f}ms "
+          f"p90={np.percentile(image_ms, 90):.2f}ms "
+          f"p99={np.percentile(image_ms, 99):.2f}ms")
+    total_ms = float(np.sum(image_pre_ms) + np.sum(image_ms) + np.sum(text_pre_ms) + np.sum(text_ms))
+    print(f"[time] total={total_ms:.2f}ms fps={len(text_captions) / (total_ms / 1000.0):.2f}/s")
+
+    save_dump(
+        args.output, BACKEND, args.image_model, args.text_model,
+        image_names, text_captions, text_image_index,
+        image_features, text_features,
+        image_ms, text_ms, image_pre_ms, text_pre_ms,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter,
+        allow_abbrev=False)
+    parser.add_argument("--image-model", required=True, help="image encoder .axmodel")
+    parser.add_argument("--text-model", required=True, help="text encoder .axmodel")
+    parser.add_argument("--tokenizer", required=True, help="tokenizer.json path")
+    parser.add_argument("--image", default=None, help="single image path (single mode)")
+    parser.add_argument("--texts", nargs="+", default=None,
+                        help=f"candidate texts for single mode (default: {DEFAULT_TEXTS})")
+    parser.add_argument("--dataset", default=None, help="dataset directory (dataset mode)")
+    parser.add_argument("--pairs", default=None,
+                        help="pairs.jsonl (default: <dataset>/pairs.jsonl)")
+    parser.add_argument("--output", default=None,
+                        help="output .npz (required in dataset mode, optional in single mode)")
+    parser.add_argument("--image-size", type=int, default=256,
+                        help="square input size, must be a multiple of 16 (default 256)")
+    parser.add_argument("--text-length", type=int, default=64,
+                        help="fixed token length (64 for short/box, 196 for long)")
+    parser.add_argument("--limit", type=int, default=0,
+                        help="only use the first N pairs (dataset mode, 0 = all)")
+    parser.add_argument("--verbose", action="store_true",
+                        help="print tokenize/infer time per text in single mode")
+    args = parser.parse_args()
+
+    if bool(args.image) == bool(args.dataset):
+        parser.error("exactly one of --image (single mode) / --dataset (dataset mode) is required")
+    if args.dataset and not args.output:
+        parser.error("--output is required in dataset mode")
+    if args.image and not args.texts:
+        args.texts = DEFAULT_TEXTS
+
+    tokenizer = GemmaBpeTokenizer(args.tokenizer)
+    image_sess = create_session(args.image_model)
+    text_sess = create_session(args.text_model)
+
+    if args.image:
+        run_single(args, tokenizer, image_sess, text_sess)
+    else:
+        run_dataset(args, tokenizer, image_sess, text_sess)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/infer/fgclip2_infer_onnx.py

@@ -0,0 +1,398 @@
+#!/usr/bin/env python3
+"""Run the FG-CLIP 2 Large encoders in FP32 ONNX.
+
+This script is meant for the DEVELOPMENT HOST. It uses onnxruntime and writes
+the same .npz schema as ``fgclip2_infer_axmodel.py`` so one evaluator can
+compare the two backends.
+
+Two modes:
+
+  single  --image PATH --texts ...      quick sanity check, prints similarity
+  dataset --dataset DIR                 batch inference, writes raw features
+
+Dataset layout:
+  <dataset>/images/*.jpg
+  <dataset>/pairs.jsonl     # one JSON object per line: {"image": "...", "text": "..."}
+
+Input contract:
+  image : patchified Siglip2 input [1, num_patches, 3*patch*patch].
+          resize(RGB, BILINEAR) to image-size -> /255 -> (x-0.5)/0.5 -> 16x16 patches.
+  text  : int64 token ids [1,L] (L=64 for walk_type "short"). Tokenizer is a
+          pure-Python Gemma BPE reader (tokenizer.json); it matches the
+          HuggingFace tokenizer and needs no transformers on the board.
+
+Usage:
+  # single image
+  python src/infer/fgclip2_infer_onnx.py \
+      --image-model models/fgclip2_large_image_encoder.onnx \
+      --text-model  models/fgclip2_large_text_encoder_short.onnx \
+      --tokenizer fg-clip2-large/tokenizer.json --image bedroom.jpg \
+      --texts 一个简约风格的卧室角落 一张铺着白色床单的床
+
+  # dataset
+  python src/infer/fgclip2_infer_onnx.py \
+      --image-model models/fgclip2_large_image_encoder.onnx \
+      --text-model  models/fgclip2_large_text_encoder_short.onnx \
+      --tokenizer fg-clip2-large/tokenizer.json \
+      --dataset fgclip_eval_64 --output results/onnx.npz
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+import numpy as np
+from PIL import Image
+
+try:
+    from tqdm import tqdm
+except ImportError:  # board image may not ship tqdm
+    tqdm = None
+
+
+PATCH_SIZE = 16
+IMAGE_MEAN = 0.5
+IMAGE_STD = 0.5
+DEFAULT_TEXTS = ["一个简约风格的卧室角落，黑色金属衣架上挂着衣物。", "一只猫在窗台上晒太阳。"]
+BACKEND = "onnx"
+
+
+# ---------------------------------------------------------------------------
+# Gemma BPE tokenizer (reads tokenizer.json; mirrors the HF fast tokenizer)
+# ---------------------------------------------------------------------------
+
+class GemmaBpeTokenizer:
+    def __init__(self, tokenizer_file, pad_id=0, eos_id=1, unk_id=3):
+        with open(tokenizer_file, encoding="utf-8") as f:
+            data = json.load(f)
+        self.vocab = data["model"]["vocab"]
+        self.rank = {}
+        for rank, pair in enumerate(data["model"]["merges"]):
+            self.rank[(pair[0], pair[1])] = rank
+        added = [(a["content"], a["id"]) for a in data.get("added_tokens", [])]
+        added.sort(key=lambda x: -len(x[0]))
+        self.added = added
+        self.pad_id = pad_id
+        self.eos_id = eos_id
+        self.unk_id = unk_id
+
+    def _byte_fallback(self, ch):
+        out = []
+        for b in ch.encode("utf-8"):
+            token = f"<0x{b:02X}>"
+            out.append(token if token in self.vocab else None)
+        return out
+
+    def _bpe(self, segment):
+        symbols = []
+        for ch in segment:
+            if ch in self.vocab:
+                symbols.append(ch)
+            else:
+                for token in self._byte_fallback(ch):
+                    symbols.append(token if token is not None else "<unk>")
+
+        while len(symbols) > 1:
+            best_rank, best_i = None, -1
+            for i in range(len(symbols) - 1):
+                rank = self.rank.get((symbols[i], symbols[i + 1]))
+                if rank is not None and (best_rank is None or rank < best_rank):
+                    best_rank, best_i = rank, i
+            if best_rank is None:
+                break
+
+            pair = (symbols[best_i], symbols[best_i + 1])
+            merged, i = [], 0
+            while i < len(symbols):
+                if i + 1 < len(symbols) and (symbols[i], symbols[i + 1]) == pair:
+                    merged.append(symbols[i] + symbols[i + 1])
+                    i += 2
+                else:
+                    merged.append(symbols[i])
+                    i += 1
+            symbols = merged
+        return symbols
+
+    def encode(self, text, max_length, dtype=np.int64):
+        text = text.replace(" ", "▁")
+        ids, buf, i = [], "", 0
+        while i < len(text):
+            hit = None
+            for content, token_id in self.added:
+                if text.startswith(content, i):
+                    hit = (content, token_id)
+                    break
+            if hit is not None:
+                if buf:
+                    ids.extend(self.vocab.get(t, self.unk_id) for t in self._bpe(buf))
+                    buf = ""
+                ids.append(hit[1])
+                i += len(hit[0])
+            else:
+                buf += text[i]
+                i += 1
+        if buf:
+            ids.extend(self.vocab.get(t, self.unk_id) for t in self._bpe(buf))
+
+        ids = (ids + [self.eos_id])[:max_length]
+        ids += [self.pad_id] * (max_length - len(ids))
+        return np.asarray(ids, dtype=dtype).reshape(1, max_length)
+
+
+# ---------------------------------------------------------------------------
+# Runtime helpers
+# ---------------------------------------------------------------------------
+
+def create_session(path):
+    import onnxruntime as ort
+    return ort.InferenceSession(path, providers=["CPUExecutionProvider"])
+
+
+def input_name_and_dtype(sess):
+    node = sess.get_inputs()[0]
+    dtype = getattr(node, "dtype", None)
+    if not isinstance(dtype, np.dtype):
+        text = str(getattr(node, "type", ""))
+        name = text.split("(")[-1].rstrip(")")
+        dtype = np.dtype({
+            "float": np.float32, "double": np.float64, "float16": np.float16,
+            "int8": np.int8, "uint8": np.uint8, "int16": np.int16,
+            "uint16": np.uint16, "int32": np.int32, "uint32": np.uint32,
+            "int64": np.int64,
+        }.get(name, name))
+    return node.name, dtype
+
+
+def run_feature(sess, x):
+    name, dtype = input_name_and_dtype(sess)
+    x = np.ascontiguousarray(x, dtype=dtype)
+    start = time.perf_counter()
+    out = sess.run(None, {name: x})[0]
+    elapsed = (time.perf_counter() - start) * 1000.0
+    return np.asarray(out, dtype=np.float32).reshape(-1), elapsed
+
+
+def preprocess_image(path, image_size):
+    image = Image.open(path).convert("RGB")
+    resample = getattr(Image, "Resampling", Image).BILINEAR
+    image = image.resize((image_size, image_size), resample)
+    x = np.asarray(image, dtype=np.float32) / 255.0
+    x = (x - IMAGE_MEAN) / IMAGE_STD
+    grid = image_size // PATCH_SIZE
+    patches = x.reshape(grid, PATCH_SIZE, grid, PATCH_SIZE, 3)
+    patches = patches.transpose(0, 2, 1, 3, 4).reshape(1, grid * grid, PATCH_SIZE * PATCH_SIZE * 3)
+    return np.ascontiguousarray(patches)
+
+
+def normalize(x):
+    return x / (np.linalg.norm(x, axis=-1, keepdims=True) + 1e-12)
+
+
+def softmax(x):
+    x = x - np.max(x)
+    e = np.exp(x)
+    return e / np.sum(e)
+
+
+# ---------------------------------------------------------------------------
+# Dataset
+# ---------------------------------------------------------------------------
+
+def load_pairs(dataset_dir, pairs_file, limit):
+    pairs_path = pairs_file or os.path.join(dataset_dir, "pairs.jsonl")
+    if not os.path.isfile(pairs_path):
+        raise SystemExit(f"pairs file not found: {pairs_path}")
+    pairs = []
+    with open(pairs_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            item = json.loads(line)
+            pairs.append((item["image"], item["text"]))
+            if limit and len(pairs) >= limit:
+                break
+    if not pairs:
+        raise SystemExit(f"no pairs in {pairs_path}")
+    return pairs
+
+
+def group_pairs(pairs):
+    """Deduplicate images, keep the first-seen order, map each text to an image."""
+    image_names = []
+    image_index = {}
+    text_captions = []
+    text_image_index = []
+    for image_name, text in pairs:
+        if image_name not in image_index:
+            image_index[image_name] = len(image_names)
+            image_names.append(image_name)
+        text_captions.append(text)
+        text_image_index.append(image_index[image_name])
+    return image_names, text_captions, np.asarray(text_image_index, dtype=np.int32)
+
+
+def save_dump(path, backend, image_model, text_model, image_names, text_captions,
+              text_image_index, image_features, text_features,
+              image_ms, text_ms, image_pre_ms, text_pre_ms):
+    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+    np.savez(
+        path,
+        backend=np.asarray(backend),
+        image_model=np.asarray(os.path.basename(image_model)),
+        text_model=np.asarray(os.path.basename(text_model)),
+        image_names=np.asarray(image_names),
+        text_captions=np.asarray(text_captions),
+        text_image_index=np.asarray(text_image_index, dtype=np.int32),
+        image_features=image_features,
+        text_features=text_features,
+        image_ms=np.asarray(image_ms, dtype=np.float32),
+        text_ms=np.asarray(text_ms, dtype=np.float32),
+        image_preprocess_ms=np.asarray(image_pre_ms, dtype=np.float32),
+        text_tokenize_ms=np.asarray(text_pre_ms, dtype=np.float32),
+    )
+    print(f"[save] {path}")
+
+
+# ---------------------------------------------------------------------------
+# Modes
+# ---------------------------------------------------------------------------
+
+def run_single(args, tokenizer, image_sess, text_sess):
+    start = time.perf_counter()
+    pre_start = time.perf_counter()
+    x = preprocess_image(args.image, args.image_size)
+    pre_ms = (time.perf_counter() - pre_start) * 1000.0
+    image_feature, image_ms = run_feature(image_sess, x)
+
+    text_features = []
+    text_ms = []
+    for text in args.texts:
+        ids = tokenizer.encode(text, args.text_length)
+        feat, ms = run_feature(text_sess, ids)
+        text_features.append(feat)
+        text_ms.append(ms)
+    text_features = np.stack(text_features)
+    total_ms = (time.perf_counter() - start) * 1000.0
+
+    print(f"[time] preprocess={pre_ms:.2f}ms image_infer={image_ms:.2f}ms "
+          f"text_infer(avg)={np.mean(text_ms):.2f}ms total={total_ms:.2f}ms")
+
+    sim = normalize(image_feature[None])[0] @ normalize(text_features).T
+    probs = softmax(sim * 100.0)
+    best = int(np.argmax(probs))
+    print("similarity matrix (cosine):")
+    for i, text in enumerate(args.texts):
+        print(f"  {text:<40} sim={sim[i]:.4f} prob={probs[i]:.4f}")
+    print(f"[result] top-1: {args.texts[best]} (prob={probs[best]:.4f})")
+
+    if args.output:
+        save_dump(
+            args.output, BACKEND, args.image_model, args.text_model,
+            [os.path.basename(args.image)], args.texts, np.zeros(len(args.texts)),
+            image_feature[None], text_features,
+            [image_ms], text_ms, [pre_ms], [0.0] * len(args.texts),
+        )
+
+
+def run_dataset(args, tokenizer, image_sess, text_sess):
+    dataset_dir = os.path.abspath(args.dataset)
+    pairs = load_pairs(dataset_dir, args.pairs, args.limit)
+    image_names, text_captions, text_image_index = group_pairs(pairs)
+    print(f"[data] {len(image_names)} images, {len(text_captions)} captions")
+
+    image_iter = tqdm(image_names, desc="image") if tqdm else image_names
+    image_features = None
+    image_ms = []
+    image_pre_ms = []
+    for i, name in enumerate(image_iter):
+        pre_start = time.perf_counter()
+        x = preprocess_image(os.path.join(dataset_dir, "images", name), args.image_size)
+        image_pre_ms.append((time.perf_counter() - pre_start) * 1000.0)
+        feat, ms = run_feature(image_sess, x)
+        if image_features is None:
+            image_features = np.zeros((len(image_names), feat.shape[0]), dtype=np.float32)
+        image_features[i] = feat
+        image_ms.append(ms)
+
+    text_iter = tqdm(text_captions, desc="text") if tqdm else text_captions
+    text_features = np.zeros((len(text_captions), image_features.shape[1]), dtype=np.float32)
+    text_ms = []
+    text_pre_ms = []
+    for i, caption in enumerate(text_iter):
+        pre_start = time.perf_counter()
+        ids = tokenizer.encode(caption, args.text_length)
+        text_pre_ms.append((time.perf_counter() - pre_start) * 1000.0)
+        feat, ms = run_feature(text_sess, ids)
+        if feat.shape[0] != text_features.shape[1]:
+            raise SystemExit(f"feature size mismatch: "
+                             f"image={text_features.shape[1]} text={feat.shape[0]}")
+        text_features[i] = feat
+        text_ms.append(ms)
+
+    print(f"[time] image preprocess={np.mean(image_pre_ms):.2f}ms "
+          f"infer={np.mean(image_ms):.2f}ms/img; "
+          f"text tokenize={np.mean(text_pre_ms):.2f}ms "
+          f"infer={np.mean(text_ms):.2f}ms/text")
+    print(f"[time] image p50={np.percentile(image_ms, 50):.2f}ms "
+          f"p90={np.percentile(image_ms, 90):.2f}ms "
+          f"p99={np.percentile(image_ms, 99):.2f}ms")
+    total_ms = float(np.sum(image_pre_ms) + np.sum(image_ms) + np.sum(text_pre_ms) + np.sum(text_ms))
+    print(f"[time] total={total_ms:.2f}ms fps={len(text_captions) / (total_ms / 1000.0):.2f}/s")
+
+    save_dump(
+        args.output, BACKEND, args.image_model, args.text_model,
+        image_names, text_captions, text_image_index,
+        image_features, text_features,
+        image_ms, text_ms, image_pre_ms, text_pre_ms,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter,
+        allow_abbrev=False)
+    parser.add_argument("--image-model", required=True, help="image encoder .onnx")
+    parser.add_argument("--text-model", required=True, help="text encoder .onnx")
+    parser.add_argument("--tokenizer", required=True, help="tokenizer.json path")
+    parser.add_argument("--image", default=None, help="single image path (single mode)")
+    parser.add_argument("--texts", nargs="+", default=None,
+                        help=f"candidate texts for single mode (default: {DEFAULT_TEXTS})")
+    parser.add_argument("--dataset", default=None, help="dataset directory (dataset mode)")
+    parser.add_argument("--pairs", default=None,
+                        help="pairs.jsonl (default: <dataset>/pairs.jsonl)")
+    parser.add_argument("--output", default=None,
+                        help="output .npz (required in dataset mode, optional in single mode)")
+    parser.add_argument("--image-size", type=int, default=256,
+                        help="square input size, must be a multiple of 16 (default 256)")
+    parser.add_argument("--text-length", type=int, default=64,
+                        help="fixed token length (64 for short/box, 196 for long)")
+    parser.add_argument("--limit", type=int, default=0,
+                        help="only use the first N pairs (dataset mode, 0 = all)")
+    args = parser.parse_args()
+
+    if bool(args.image) == bool(args.dataset):
+        parser.error("exactly one of --image (single mode) / --dataset (dataset mode) is required")
+    if args.dataset and not args.output:
+        parser.error("--output is required in dataset mode")
+    if args.image and not args.texts:
+        args.texts = DEFAULT_TEXTS
+
+    tokenizer = GemmaBpeTokenizer(args.tokenizer)
+    image_sess = create_session(args.image_model)
+    text_sess = create_session(args.text_model)
+
+    if args.image:
+        run_single(args, tokenizer, image_sess, text_sess)
+    else:
+        run_dataset(args, tokenizer, image_sess, text_sess)
+
+
+if __name__ == "__main__":
+    sys.exit(main())