Unify modeling/configuration/tokenizer with the 0708 standard (transformers 4.57 compatible, includes query RoPE fix)

configuration_moss_vl.py

@@ -69,8 +69,6 @@ class MossVLTextConfig(PretrainedConfig):
 
     model_type = "moss_vl_text"
     base_config_key = "text_config"
-    default_theta = 5000000.0
-    ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
 
     def __init__(
         self,
@@ -88,11 +86,9 @@ class MossVLTextConfig(PretrainedConfig):
         use_cache=True,
         tie_word_embeddings=False,
         rope_theta=5000000.0,
-        rope_parameters=None,
         rope_scaling=None,
         attention_bias=False,
         attention_dropout=0.0,
-        pad_token_id=None,
         # Cross attention specific
         cross_attention_layers=None,  # List of layer indices to insert cross attention
         **kwargs,
@@ -116,31 +112,11 @@ class MossVLTextConfig(PretrainedConfig):
         self.rms_norm_eps = rms_norm_eps
         self.use_cache = use_cache
         self.rope_theta = rope_theta
-        if rope_parameters is None:
-            if rope_scaling is not None:
-                rope_parameters = dict(rope_scaling)
-            else:
-                rope_parameters = {"rope_type": "default"}
-        else:
-            rope_parameters = dict(rope_parameters)
-
-        if "type" in rope_parameters and "rope_type" not in rope_parameters:
-            rope_parameters["rope_type"] = rope_parameters.pop("type")
-        rope_parameters.setdefault("rope_type", "default")
-        rope_parameters.setdefault("rope_theta", rope_theta)
-
-        self.rope_parameters = rope_parameters
-        self.rope_scaling = rope_scaling if rope_scaling is not None else dict(rope_parameters)
+        self.rope_scaling = rope_scaling
         self.attention_bias = attention_bias
         self.attention_dropout = attention_dropout
-        self.pad_token_id = pad_token_id
-
-        if hasattr(self, "standardize_rope_params"):
-            self.standardize_rope_params()
-        if hasattr(self, "validate_rope"):
-            self.validate_rope()
-        else:
-            rope_config_validation(self, ignore_keys=self.ignore_keys_at_rope_validation)
+
+        rope_config_validation(self, ignore_keys={"mrope_section", "mrope_interleaved"})
         self.cross_attention_layers = cross_attention_layers or [2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46]
         super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
 
@@ -185,4 +161,4 @@ class MossVLConfig(PretrainedConfig):
         super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
         
 
-__all__ = ["MossVLConfig", "MossVLTextConfig"]
+__all__ = ["MossVLConfig", "MossVLTextConfig"]

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52d44d7e09e05fb10f9ec5dc913bf1d62ff37ac249cb9ec47d891935149f5e3e
-size 11423034
+oid sha256:e7bbd0f9784004df51aca562befc3c7a8f294b4045aa8685536c35804c9aa493
+size 11423411

tokenizer_config.json

@@ -225,6 +225,22 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "151671": {
+      "content": "<|silence|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|response|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
@@ -242,7 +258,9 @@
     "<|image_pad|>",
     "<|video_pad|>",
     "<|time_start|>",
-    "<|time_end|>"
+    "<|time_end|>",
+    "<|silence|>",
+    "<|response|>"
   ],
   "bos_token": null,
   "clean_up_tokenization_spaces": false,