Export GraphTokenLM to Hugging Face Hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "GraphTokenLM"
+  ],
+  "base_model": "Qwen/Qwen3-4B-Base",
+  "bos_token_id": 151643,
+  "dtype": "float32",
+  "enable_lora": false,
+  "eos_token_id": 151643,
+  "freeze_llm": true,
+  "gnn_hidden": 64,
+  "gnn_hidden_dim": 64,
+  "gnn_out": 64,
+  "gnn_out_dim": 64,
+  "gnn_type": "GIN",
+  "graph_pooling": [
+    "mean"
+  ],
+  "hidden_size": 2560,
+  "llm_name": "Qwen/Qwen3-4B-Base",
+  "lpe_dim": 8,
+  "model_type": "graph_token_lm",
+  "node_feat_dim": 8,
+  "node_pos_emb_dim": 8,
+  "num_attention_heads": 32,
+  "num_gnn_layers": 3,
+  "num_graph_tokens": 4,
+  "num_hidden_layers": 36,
+  "num_max_nodes": 20,
+  "num_proj_layers": 2,
+  "pos_emb_dim": 8,
+  "transformers_version": "4.57.1",
+  "use_degree_emb": false,
+  "vocab_size": 151936,
+  "auto_map": {
+    "AutoConfig": "glm.GraphTokenLMConfig",
+    "AutoModelForCausalLM": "glm.GraphTokenLM"
+  }
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [
+    151643
+  ],
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.1"
+}

glm.py

@@ -0,0 +1,553 @@
+from typing import Literal
+
+import torch
+import torch.nn as nn
+from torch_geometric.nn import (
+    GATConv,
+    GCNConv,
+    GINConv,
+    GraphSAGE,
+    TransformerConv,
+    global_add_pool,
+    global_max_pool,
+    global_mean_pool,
+)
+from torch_geometric.utils import to_dense_batch
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    PretrainedConfig,
+    PreTrainedModel,
+)
+from transformers.generation.utils import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+VALID_GRAPH_POOLING = ["mean", "sum", "max"]
+
+
+class GraphTokenLMConfig(PretrainedConfig):
+    model_type = "graph_token_lm"
+
+    def __init__(
+        self,
+        base_model="Qwen/Qwen3-4B-Base",
+        gnn_type: Literal["GCN", "GAT", "GIN", "GraphSAGE", "GraphTransformer"] = "GCN",
+        node_feat_dim=8,
+        lpe_dim: int | None = None,
+        use_degree_emb: bool = False,
+        pos_emb_dim=8,
+        gnn_hidden_dim=256,
+        gnn_out_dim=512,
+        num_gnn_layers=2,
+        graph_pooling: list[Literal["mean", "sum", "max"]] = ["mean"],
+        num_proj_layers=1,
+        num_graph_tokens=4,
+        num_max_nodes=20,  # maximum number of nodes per batch
+        freeze_llm=True,
+        enable_lora: bool = False,
+        tie_word_embeddings=True,
+        **kwargs,
+    ):
+        """Initialize a GraphToken language model configuration.
+
+        Parameters
+        ----------
+        base_model : str, default="Qwen/Qwen3-4B-Base"
+            Hugging Face model name or path for the underlying LLM.
+        gnn_type : {"GCN", "GAT", "GIN", "GraphSAGE", "GraphTransformer"}, default="GCN"
+            Type of GNN layer to use for encoding graph nodes.
+        node_feat_dim : int, default=8
+            Dimensionality of the raw node features.
+        lpe_dim : int or None, default=None
+            Dimensionality for Laplacian positional encodings; defaults to ``node_feat_dim``.
+        use_degree_emb : bool, default=False
+            Whether to add degree-based embeddings to node features.
+        pos_emb_dim : int, default=8
+            Dimensionality of learned positional embeddings for nodes.
+        gnn_hidden_dim : int, default=256
+            Hidden dimensionality for the GNN stack.
+        gnn_out_dim : int, default=512
+            Output dimensionality of the GNN encoder.
+        num_gnn_layers : int, default=2
+            Number of GNN layers to apply.
+        num_proj_layers : int, default=1
+            Number of projection layers mapping graph reps to tokens.
+        num_graph_tokens : int, default=4
+            Number of graph tokens to prepend to the LLM.
+        num_max_nodes : int, default=20
+            Maximum number of nodes per graph in a batch.
+        graph_pooling : {"mean", "sum", "max"} list, default=["mean"]
+            Pooling strategy for graph-level aggregation. When multiple values are
+            provided, pooled vectors are concatenated.
+        freeze_llm : bool, default=True
+            Whether to freeze the underlying LLM parameters.
+        enable_lora : bool, default=False
+            Whether LoRA adapters are expected to be active, in which case the
+            base LLM should remain in training mode unless explicitly set
+            elsewhere.
+        tie_word_embeddings : bool, default=True
+            Whether to tie input/output embeddings in the LLM config.
+        **kwargs
+            Additional arguments forwarded to ``PretrainedConfig``.
+        """
+        self.base_model = base_model
+        self.llm_name = base_model  # backward compatibility
+        self.gnn_type = gnn_type
+
+        self.node_feat_dim = node_feat_dim
+        self.lpe_dim = lpe_dim if lpe_dim is not None else node_feat_dim
+        self.use_degree_emb = bool(use_degree_emb)
+        self.pos_emb_dim = pos_emb_dim
+        self.node_pos_emb_dim = pos_emb_dim  # backward compatibility
+        self.gnn_hidden_dim = gnn_hidden_dim
+        self.gnn_hidden = gnn_hidden_dim  # backward compatibility
+        self.gnn_out_dim = gnn_out_dim
+        self.gnn_out = gnn_out_dim  # backward compatibility
+        self.num_gnn_layers = num_gnn_layers
+        self.num_proj_layers = num_proj_layers
+        self.num_graph_tokens = num_graph_tokens
+        self.num_max_nodes = num_max_nodes
+        self.graph_pooling = graph_pooling
+        self.freeze_llm = freeze_llm
+        self.enable_lora = enable_lora
+
+        # Keep generation-related fields for compatibility (updated later).
+        self.vocab_size = kwargs.get("vocab_size", None)
+        self.pad_token_id = kwargs.get("pad_token_id", None)
+        self.bos_token_id = kwargs.get("bos_token_id", None)
+        self.eos_token_id = kwargs.get("eos_token_id", None)
+
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+    # Maintain compatibility with transformers.GenerationConfig.
+    def get_text_config(self, decoder: bool | None = None, **kwargs):
+        return self
+
+
+class GNNEncoder(nn.Module):
+
+    def __init__(
+        self,
+        in_dim: int,
+        hid_dim: int,
+        out_dim: int,
+        max_nodes: int,
+        num_layers: int = 2,
+        node_pos_emb_dim: int = 8,
+        dropout: float = 0.1,
+        gnn_type: Literal["GCN", "GAT", "GIN", "GraphSAGE", "GraphTransformer"] = "GCN",
+    ):
+        """Encode graph node features with a configurable GNN stack.
+
+        Parameters
+        ----------
+        in_dim : int
+            Dimensionality of the input node features.
+        hid_dim : int
+            Hidden dimensionality used for intermediate layers.
+        out_dim : int
+            Dimensionality of the output node representations.
+        max_nodes : int
+            Maximum number of nodes per graph in a batch.
+        num_layers : int, default=2
+            Number of graph convolution layers.
+        node_pos_emb_dim : int, default=8
+            Dimensionality of the optional learned positional embeddings.
+        dropout : float, default=0.1
+            Dropout probability applied between hidden layers.
+        gnn_type : {"GCN", "GAT", "GIN", "GraphSAGE", "GraphTransformer"}, default="GCN"
+            Type of graph convolution layer to build.
+        """
+        super().__init__()
+        self.max_nodes = max_nodes
+        self.pos_emb = nn.Embedding(max_nodes, node_pos_emb_dim) if node_pos_emb_dim > 0 else None
+
+        in_channels = in_dim + (node_pos_emb_dim if node_pos_emb_dim > 0 else 0)
+        if in_channels <= 0:
+            raise ValueError("GNNEncoder requires a positive input feature dimension.")
+
+        hidden_dims = [hid_dim] * max(num_layers - 1, 0)
+        dims = [in_channels, *hidden_dims, out_dim]
+        self.convs = nn.ModuleList()
+        for i in range(len(dims) - 1):
+            match gnn_type:
+                case "GCN":
+                    self.convs.append(GCNConv(dims[i], dims[i + 1]))
+                case "GAT":
+                    self.convs.append(GATConv(dims[i], dims[i + 1]))
+                case "GIN":
+                    self.convs.append(GINConv(nn.Linear(dims[i], dims[i + 1])))
+                case "GraphSAGE":
+                    self.convs.append(GraphSAGE(dims[i], dims[i + 1], 1))
+                case "GraphTransformer":
+                    # Multi-head attention with concat disabled to keep the output dim aligned.
+                    self.convs.append(TransformerConv(dims[i], dims[i + 1], heads=4, concat=False, dropout=dropout))
+                case _:
+                    raise ValueError(f"Unsupported gnn_type: {gnn_type}")
+        self.act = nn.ReLU()
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x, edge_index, batch):
+        if self.pos_emb is not None:
+            # Assign positional indices per graph starting from zero within the batch.
+            _, mask = to_dense_batch(x, batch, max_num_nodes=self.max_nodes)
+            pos_idx = torch.arange(self.max_nodes, device=x.device).unsqueeze(0).expand(mask.size(0), -1)
+            pos_idx = pos_idx[mask]
+            x = torch.cat([x, self.pos_emb(pos_idx)], dim=-1)
+        for i, conv in enumerate(self.convs):
+            x = conv(x, edge_index)
+            if i < len(self.convs) - 1:
+                x = self.act(x)
+                x = self.dropout(x)
+        return x  # [num_nodes, out_dim]
+
+
+def _normalize_graph_pooling(graph_pooling: list[str]) -> list[str]:
+    """Normalize and validate graph pooling values into a canonical list."""
+    poolings = list(graph_pooling)
+    if not (1 <= len(poolings) <= 3):
+        raise ValueError(f"Unsupported graph_pooling length: {len(poolings)}")
+    if len(set(poolings)) != len(poolings):
+        raise ValueError(f"Duplicate graph_pooling values: {poolings}")
+    if not set(poolings).issubset(set(VALID_GRAPH_POOLING)):
+        raise ValueError(f"Unsupported graph_pooling values: {poolings}")
+    return poolings
+
+
+class DomainProjector(nn.Module):
+
+    def __init__(
+        self,
+        gnn_out_dim,
+        llm_hidden_size,
+        num_graph_tokens=4,
+        num_layers=1,
+        graph_pooling: list[str] = ["mean"],
+    ):
+        """Project graph-level representations into graph tokens.
+
+        The projector first pools node embeddings into a graph representation and
+        then maps it into ``k`` graph tokens that match the language model's hidden
+        dimension.
+
+        Parameters
+        ----------
+        gnn_out_dim : int
+            Dimensionality of the encoder output to project from.
+        llm_hidden_size : int
+            Target dimensionality matching the language model embeddings.
+        num_graph_tokens : int, default=4
+            Number of graph tokens to produce.
+        graph_pooling : {"mean", "sum", "max"} list, default=["mean"]
+            Pooling strategy used to aggregate node embeddings. When multiple values are
+            provided, pooled vectors are concatenated.
+        num_layers : int, default=1
+            Number of linear/GELU projection layers.
+        """
+        super().__init__()
+        if num_layers < 1:
+            raise ValueError("DomainProjector requires at least one projection layer.")
+        poolings = _normalize_graph_pooling(graph_pooling)
+        self.num_graph_tokens = num_graph_tokens
+        self.graph_pooling = poolings
+        layers = []
+        in_dim = gnn_out_dim * len(poolings)
+        final_dim = llm_hidden_size * num_graph_tokens
+        if num_layers == 1:
+            layer_dims = [final_dim]
+        else:
+            ratio = final_dim / in_dim
+            layer_dims = []
+            prev_dim = in_dim
+            for layer_idx in range(num_layers):
+                t = (layer_idx + 1) / num_layers
+                dim = max(1, int(round(in_dim * (ratio ** t))))
+                if dim % 2 != 0:
+                    dim += 1
+                dim = max(dim, prev_dim)
+                layer_dims.append(dim)
+                prev_dim = dim
+            layer_dims[-1] = final_dim
+        for layer_idx, out_dim in enumerate(layer_dims):
+            layers.append(nn.Linear(in_dim, out_dim))
+            if layer_idx < num_layers - 1:
+                layers.append(nn.GELU())
+            in_dim = out_dim
+        self.project = nn.Sequential(*layers)
+
+        # Optional learned positional embeddings for graph tokens.
+        self.graph_pos = nn.Embedding(num_graph_tokens, llm_hidden_size)
+
+    def forward(self, node_repr, batch_index):
+        """Aggregate node embeddings into graph tokens.
+
+        Parameters
+        ----------
+        node_repr : torch.Tensor
+            Node representations of shape ``(num_nodes_total, gnn_out_dim)``.
+        batch_index : torch.Tensor
+            Batch indices identifying the graph for each node. Shape
+            ``(num_nodes_total,)``.
+
+        Returns
+        -------
+        torch.Tensor
+            Graph token tensor of shape ``(batch_size, num_graph_tokens, hidden)``.
+        """
+        # Pool node representations into a graph-level vector.
+        pooled_list = []
+        if "mean" in self.graph_pooling:
+            pooled_list.append(global_mean_pool(node_repr, batch_index))  # [B, gnn_out_dim]
+        if "sum" in self.graph_pooling:
+            pooled_list.append(global_add_pool(node_repr, batch_index))  # [B, gnn_out_dim]
+        if "max" in self.graph_pooling:
+            pooled_list.append(global_max_pool(node_repr, batch_index))  # [B, gnn_out_dim]
+        pooled = (
+            torch.cat(pooled_list, dim=-1) if len(pooled_list) > 1 else pooled_list[0]
+        )  # [B, gnn_out_dim * num_poolings]
+        B = pooled.size(0)
+
+        # Expand into k tokens via the projection stack.
+        tokens = self.project(pooled)  # [B, k * hidden]
+        Hk = tokens.view(B, self.num_graph_tokens, -1)  # [B, k, hidden]
+
+        # Add learned positional embeddings.
+        pos = self.graph_pos.weight.unsqueeze(0).expand(B, -1, -1)  # [B, k, hidden]
+        Hk = Hk + pos
+        return Hk  # [B, k, hidden]
+
+
+class GraphTokenLM(PreTrainedModel, GenerationMixin):
+    """Language model that prepends graph tokens to textual inputs.
+
+    A graph neural network encodes node features, pools them, and projects the
+    result into learned graph tokens that are concatenated with language model
+    embeddings before decoding.
+
+    Parameters
+    ----------
+    config : GraphTokenLMConfig
+        Model configuration describing the graph encoder and base LLM.
+    load_llm_weights : bool, default=True
+        Whether to load pretrained weights for the base language model.
+    """
+
+    _tied_weights_keys = ["llm.lm_head.weight"]
+    _keys_to_ignore_on_load_missing = [r"^llm\.lm_head\.weight$"]
+
+    config_class = GraphTokenLMConfig
+    base_model_prefix = "llm"
+
+    def __init__(self, config: GraphTokenLMConfig, load_llm_weights: bool = True):
+
+        super().__init__(config)
+
+        # LLM
+        if load_llm_weights:
+            self.llm = AutoModelForCausalLM.from_pretrained(
+                config.base_model, trust_remote_code=True, tie_word_embeddings=True
+            )
+        else:
+            llm_cfg = AutoConfig.from_pretrained(config.base_model, dtype=torch.float32)
+            self.llm = AutoModelForCausalLM.from_config(llm_cfg)
+
+        self.num_graph_tokens = config.num_graph_tokens
+
+        # GNN + Domain Projector
+        self.gnn = GNNEncoder(
+            gnn_type=config.gnn_type,
+            node_pos_emb_dim=config.pos_emb_dim,
+            in_dim=config.node_feat_dim,
+            hid_dim=config.gnn_hidden_dim,
+            out_dim=config.gnn_out_dim,
+            num_layers=config.num_gnn_layers,
+            max_nodes=config.num_max_nodes,
+        )
+        self.tokenizer_head = DomainProjector(
+            gnn_out_dim=config.gnn_out_dim,
+            llm_hidden_size=self.llm.config.hidden_size,
+            num_graph_tokens=config.num_graph_tokens,
+            num_layers=config.num_proj_layers,
+            graph_pooling=config.graph_pooling,
+        )
+
+        if config.freeze_llm:
+            for p in self.llm.parameters():
+                p.requires_grad = False
+            if not config.enable_lora:
+                self.llm.eval()
+
+        # --- sync basic generation fields so GenerationMixin works cleanly ---
+        mirror_keys = [
+            "vocab_size",
+            "pad_token_id",
+            "bos_token_id",
+            "eos_token_id",
+            "hidden_size",
+            "num_hidden_layers",
+            "num_attention_heads",
+        ]
+        for k in mirror_keys:
+            if hasattr(self.llm.config, k):
+                setattr(self.config, k, getattr(self.llm.config, k))
+
+        # make sure tying is done once at init (harmless if already tied)
+        if getattr(self.config, "tie_word_embeddings", False):
+            self.tie_weights()
+
+    @property
+    def device(self):
+        return next(self.parameters()).device
+
+    def _concat_graph_tokens(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        labels=None,
+        inputs_embeds=None,
+        graph=None,
+    ):
+        """Prepend graph tokens to language model embeddings.
+
+        Parameters
+        ----------
+        input_ids : torch.Tensor, optional
+            Token IDs used to derive embeddings if ``inputs_embeds`` is not provided.
+        attention_mask : torch.Tensor, optional
+            Attention mask aligned with ``input_ids``.
+        labels : torch.Tensor, optional
+            Label tensor passed through unchanged.
+        inputs_embeds : torch.Tensor, optional
+            Precomputed language model embeddings.
+        graph : Mapping[str, torch.Tensor], optional
+            Graph structure containing ``x``, ``edge_index``, and ``batch`` as
+            produced by the collator.
+
+        Returns
+        -------
+        tuple of torch.Tensor
+            Tuple ``(inputs_embeds, attention_mask, labels)`` with graph tokens
+            concatenated at the front of the sequence.
+        """
+        # Obtain embeddings from the base language model if necessary.
+        if inputs_embeds is None:
+            inputs_embeds = self.llm.get_input_embeddings()(input_ids)
+
+        B, T, H = inputs_embeds.size()
+
+        # ---- Graph to tokens ----
+        graph_device = next(self.gnn.parameters()).device
+        if hasattr(graph, "to"):
+            graph = graph.to(graph_device)
+
+        x = graph["x"]  # [N_nodes, node_feat_dim]
+        edge_index = graph["edge_index"]  # [2, N_edges]
+        batch = graph["batch"]  # [N_nodes]
+        node_repr = self.gnn(x, edge_index, batch)  # [N_nodes, gnn_out]
+        graph_tokens = self.tokenizer_head(node_repr, batch)  # [B, k, H]
+        if inputs_embeds is not None:
+            graph_tokens = graph_tokens.to(inputs_embeds.device)
+
+        # ---- Concatenate by prepending graph tokens ----
+        new_inputs = torch.cat([graph_tokens, inputs_embeds], dim=1)  # [B, k+T, H]
+
+        # Prepend ones to the attention mask for the graph tokens.
+        if attention_mask is None:
+            attention_mask = input_ids.ne(self.llm.config.pad_token_id).long()
+        new_attention = torch.cat(
+            [torch.ones((B, self.num_graph_tokens), dtype=attention_mask.dtype, device=self.device), attention_mask],
+            dim=1,
+        )
+
+        return new_inputs, new_attention, labels
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        graph=None,
+        labels=None,
+        **generate_kwargs,
+    ) -> CausalLMOutputWithPast:
+        if (input_ids is None) and (inputs_embeds is None):
+            raise ValueError("Either input_ids or inputs_embeds must be provided")
+
+        if graph is not None:
+            inputs_embeds, attention_mask, labels = self._concat_graph_tokens(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                labels=labels,
+                inputs_embeds=inputs_embeds,
+                graph=graph,
+            )
+        else:
+            # For generation, inputs may already include concatenated graph tokens.
+            if inputs_embeds is None:
+                inputs_embeds = self.llm.get_input_embeddings()(input_ids)
+            if attention_mask is None:
+                if input_ids is None:
+                    raise ValueError("When attention_mask is not provided, input_ids must also be provided")
+                attention_mask = input_ids.ne(self.llm.config.pad_token_id).long()
+
+        # Exclude auxiliary keys that may interfere with Qwen3 loss computation.
+        blocked = {
+            "num_items_in_batch",
+            "label_smoothing",  # Added by TRL/transformers in some setups.
+            "labels_shifted",  # Same as above.
+        }
+        passdown = {
+            k: v
+            for k, v in generate_kwargs.items()
+            if k not in blocked and k in {"use_cache", "output_attentions", "output_hidden_states", "past_key_values"}
+        }
+
+        out = self.llm(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            labels=labels,
+            **passdown,
+        )
+        return out
+
+    def prepare_inputs_for_generation(
+        self, input_ids=None, inputs_embeds=None, attention_mask=None, graph=None, **kwargs
+    ):
+        if input_ids is None and inputs_embeds is None:
+            raise ValueError("Either input_ids or inputs_embeds must be provided")
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("Both input_ids and inputs_embeds cannot be provided at the same time")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.llm.get_input_embeddings()(input_ids)
+        if graph is not None:
+            inputs_embeds, attention_mask, _ = self._concat_graph_tokens(
+                input_ids=None,
+                attention_mask=attention_mask,
+                labels=None,
+                inputs_embeds=inputs_embeds,
+                graph=graph,
+            )
+
+        return {"inputs_embeds": inputs_embeds, "attention_mask": attention_mask, "graph": None}
+
+    # delegate embeddings to inner LLM so HF can tie weights correctly
+    def get_input_embeddings(self):
+        return self.llm.get_input_embeddings()
+
+    def set_input_embeddings(self, new_embeddings):
+        self.llm.set_input_embeddings(new_embeddings)
+
+    def get_output_embeddings(self):
+        return self.llm.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.llm.set_output_embeddings(new_embeddings)
+
+    def tie_weights(self):
+        # honor config.tie_word_embeddings and delegate
+        if getattr(self.config, "tie_word_embeddings", False):
+            # inner LLM handles actual tying (lm_head <-> embeddings)
+            self.llm.tie_weights()
+        # keep parent behavior (no-op for most models)
+        return super().tie_weights()

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3650ff7c5d121c035778ff7f6999694c9e8bf7ca4869851db5f24ff6e8fe08dc
+size 4990095416

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7811b66d29f1713a08a2c6f7b799109bb71f8a8eb960751c05fb9f717a87d9b7
+size 4944309624

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:122c095ab16b76ac3c26ec172d81d80a54d7522ac35de09a5e8a1f7f2c3c0f65
+size 4996759392

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0210b6073eeabe32ac32d50b19b4a99acc82b35dea9b27996d07365cb8bf9a16
+size 1192264692

model.safetensors.index.json

@@ -0,0 +1,421 @@
+{
+  "metadata": {
+    "total_parameters": 4030845194,
+    "total_size": 16123380788
+  },
+  "weight_map": {
+    "gnn.convs.0.eps": "model-00004-of-00004.safetensors",
+    "gnn.convs.0.nn.bias": "model-00004-of-00004.safetensors",
+    "gnn.convs.0.nn.weight": "model-00004-of-00004.safetensors",
+    "gnn.convs.1.eps": "model-00004-of-00004.safetensors",
+    "gnn.convs.1.nn.bias": "model-00004-of-00004.safetensors",
+    "gnn.convs.1.nn.weight": "model-00004-of-00004.safetensors",
+    "gnn.convs.2.eps": "model-00004-of-00004.safetensors",
+    "gnn.convs.2.nn.bias": "model-00004-of-00004.safetensors",
+    "gnn.convs.2.nn.weight": "model-00004-of-00004.safetensors",
+    "gnn.pos_emb.weight": "model-00004-of-00004.safetensors",
+    "llm.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "llm.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.norm.weight": "model-00004-of-00004.safetensors",
+    "tokenizer_head.graph_pos.weight": "model-00004-of-00004.safetensors",
+    "tokenizer_head.project.0.bias": "model-00004-of-00004.safetensors",
+    "tokenizer_head.project.0.weight": "model-00004-of-00004.safetensors",
+    "tokenizer_head.project.2.bias": "model-00004-of-00004.safetensors",
+    "tokenizer_head.project.2.weight": "model-00004-of-00004.safetensors"
+  }
+}

requirements.txt

@@ -0,0 +1,5 @@
+torch
+transformers
+huggingface_hub
+safetensors
+torch-geometric

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}