gijs commited on
Commit
ae7e1d0
·
verified ·
1 Parent(s): 361141a

Replace the unmerged training snapshot with the merged weights, add 1_Pooling

Browse files
1_Pooling/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "embedding_dimension": 3584,
3
+ "pooling_mode": "lasttoken",
4
+ "include_prompt": true
5
+ }
README.md CHANGED
@@ -17,6 +17,8 @@ tags:
17
 
18
  # VoiceCLAP-Large
19
 
 
 
20
  Voice-text contrastive embedding model — the larger of the two anchors
21
  released with [VoiceNet](https://huggingface.co/VoiceNet).
22
 
 
17
 
18
  # VoiceCLAP-Large
19
 
20
+ > **Update 2026-11-11.** `model.safetensors` now holds the merged weights, and the `1_Pooling` config was added. The earlier file was an unmerged training snapshot: it loaded with randomly initialised linear layers.
21
+
22
  Voice-text contrastive embedding model — the larger of the two anchors
23
  released with [VoiceNet](https://huggingface.co/VoiceNet).
24
 
added_tokens.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</tool_call>": 151658,
3
+ "<tool_call>": 151657,
4
+ "<|AUDIO|>": 151646,
5
+ "<|IMAGE|>": 151655,
6
+ "<|VIDEO|>": 151656,
7
+ "<|audio_bos|>": 151647,
8
+ "<|audio_eos|>": 151648,
9
+ "<|box_end|>": 151649,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|quad_end|>": 151651,
19
+ "<|quad_start|>": 151650,
20
+ "<|repo_name|>": 151663,
21
+ "<|vision_bos|>": 151652,
22
+ "<|vision_eos|>": 151653,
23
+ "<|vision_pad|>": 151654
24
+ }
additional_chat_templates/sentence_transformers.jinja ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- set audio_count = namespace(value=0) -%}
2
+ {%- set image_count = namespace(value=0) -%}
3
+ {%- set video_count = namespace(value=0) -%}
4
+ <|im_start|>system
5
+ You are a helpful assistant.<|im_end|>
6
+ {% for message in messages -%}
7
+ {%- if message['role'] == 'system' -%}
8
+ {#- skip: the fixed system prompt above was already emitted. Any input system
9
+ message is only present to silence Qwen2.5-Omni's default-prompt warning. -#}
10
+ {%- else -%}
11
+ <|im_start|>{{ message['role'] }}
12
+ {% if message['content'] is string -%}
13
+ {{- message['content'] -}}<|im_end|>
14
+ {% else -%}
15
+ {%- set seen = namespace(image=false, audio=false, video=false) -%}
16
+ {%- for content in message['content'] -%}
17
+ {%- if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
18
+ {%- set image_count.value = image_count.value + 1 -%}
19
+ {%- set seen.image = true -%}
20
+ {%- if add_vision_id -%}Picture {{ image_count.value }}: {% endif -%}
21
+ <|vision_bos|><|IMAGE|><|vision_eos|>
22
+ {%- elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content -%}
23
+ {%- set audio_count.value = audio_count.value + 1 -%}
24
+ {%- set seen.audio = true -%}
25
+ {%- if add_audio_id -%}Audio {{ audio_count.value }}: {% endif -%}
26
+ <|audio_bos|><|AUDIO|><|audio_eos|>
27
+ {%- elif content['type'] == 'video' or 'video' in content -%}
28
+ {%- set video_count.value = video_count.value + 1 -%}
29
+ {%- set seen.video = true -%}
30
+ {%- if add_vision_id -%}Video {{ video_count.value }}: {% endif -%}
31
+ <|vision_bos|><|VIDEO|><|vision_eos|>
32
+ {%- elif 'text' in content -%}
33
+ {{- content['text'] -}}
34
+ {%- endif -%}
35
+ {%- endfor -%}
36
+ {%- if seen.image -%}
37
+ {{ '\n' }}Summarize the above image in one word:
38
+ {%- elif seen.video -%}
39
+ {{ '\n' }}Summarize the above video in one word:
40
+ {%- elif seen.audio -%}
41
+ {{ '\n' }}Summarize the above audio in one word:
42
+ {%- else -%}
43
+ {{ '\n' }}Summarize the above text in one word:
44
+ {%- endif -%}
45
+ <|im_end|>
46
+ {% endif -%}
47
+ {%- endif -%}
48
+ {%- endfor -%}
49
+ {%- if add_generation_prompt -%}
50
+ <|im_start|>assistant
51
+ {% endif -%}
config.json CHANGED
@@ -10,7 +10,6 @@
10
  "attention_dropout": 0.0,
11
  "d_model": 1280,
12
  "dropout": 0.0,
13
- "dtype": "bfloat16",
14
  "encoder_attention_heads": 20,
15
  "encoder_ffn_dim": 5120,
16
  "encoder_layerdrop": 0.0,
@@ -43,9 +42,6 @@
43
  "seconds_per_chunk": 2,
44
  "text_config": {
45
  "attention_dropout": 0.0,
46
- "bos_token_id": null,
47
- "dtype": "bfloat16",
48
- "eos_token_id": null,
49
  "hidden_act": "silu",
50
  "hidden_size": 3584,
51
  "init_std": 0.02,
@@ -87,31 +83,28 @@
87
  "num_attention_heads": 28,
88
  "num_hidden_layers": 28,
89
  "num_key_value_heads": 4,
90
- "pad_token_id": null,
91
  "rms_norm_eps": 1e-06,
92
- "rope_parameters": {
93
  "mrope_section": [
94
  16,
95
  24,
96
  24
97
  ],
98
- "rope_theta": 1000000.0,
99
  "rope_type": "default",
100
  "type": "default"
101
  },
 
102
  "sliding_window": null,
103
  "use_cache": true,
104
  "use_sliding_window": false,
105
  "vocab_size": 152064
106
  },
107
- "tie_word_embeddings": false,
108
- "transformers_version": "5.1.0",
109
  "user_token_id": 872,
110
  "video_token_index": 151656,
111
  "vision_config": {
112
  "_attn_implementation_autoset": true,
113
  "depth": 32,
114
- "dtype": "bfloat16",
115
  "embed_dim": 1280,
116
  "fullatt_block_indexes": [
117
  7,
 
10
  "attention_dropout": 0.0,
11
  "d_model": 1280,
12
  "dropout": 0.0,
 
13
  "encoder_attention_heads": 20,
14
  "encoder_ffn_dim": 5120,
15
  "encoder_layerdrop": 0.0,
 
42
  "seconds_per_chunk": 2,
43
  "text_config": {
44
  "attention_dropout": 0.0,
 
 
 
45
  "hidden_act": "silu",
46
  "hidden_size": 3584,
47
  "init_std": 0.02,
 
83
  "num_attention_heads": 28,
84
  "num_hidden_layers": 28,
85
  "num_key_value_heads": 4,
 
86
  "rms_norm_eps": 1e-06,
87
+ "rope_scaling": {
88
  "mrope_section": [
89
  16,
90
  24,
91
  24
92
  ],
 
93
  "rope_type": "default",
94
  "type": "default"
95
  },
96
+ "rope_theta": 1000000.0,
97
  "sliding_window": null,
98
  "use_cache": true,
99
  "use_sliding_window": false,
100
  "vocab_size": 152064
101
  },
102
+ "transformers_version": "4.57.0.dev0",
 
103
  "user_token_id": 872,
104
  "video_token_index": 151656,
105
  "vision_config": {
106
  "_attn_implementation_autoset": true,
107
  "depth": 32,
 
108
  "embed_dim": 1280,
109
  "fullatt_block_indexes": [
110
  7,
config_sentence_transformers.json CHANGED
@@ -1,15 +1,13 @@
1
  {
2
  "__version__": {
3
  "pytorch": "2.10.0+cu128",
4
- "sentence_transformers": "5.4.1",
5
- "transformers": "5.1.0"
6
  },
7
  "default_prompt_name": "default",
8
  "model_type": "SentenceTransformer",
9
  "prompts": {
10
- "default": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
11
- "document": "",
12
- "query": ""
13
  },
14
  "similarity_fn_name": "cosine"
15
- }
 
1
  {
2
  "__version__": {
3
  "pytorch": "2.10.0+cu128",
4
+ "sentence_transformers": "5.4.0",
5
+ "transformers": "5.6.0"
6
  },
7
  "default_prompt_name": "default",
8
  "model_type": "SentenceTransformer",
9
  "prompts": {
10
+ "default": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
 
 
11
  },
12
  "similarity_fn_name": "cosine"
13
+ }
generation_config.json CHANGED
@@ -3,5 +3,5 @@
3
  "bos_token_id": 151644,
4
  "eos_token_id": 151645,
5
  "pad_token_id": 151643,
6
- "transformers_version": "5.1.0"
7
  }
 
3
  "bos_token_id": 151644,
4
  "eos_token_id": 151645,
5
  "pad_token_id": 151643,
6
+ "transformers_version": "4.57.0.dev0"
7
  }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5cb1c7c101cd5c360775bde52742bc7369bb8906d1991c4ab3cbeab836d81bf0
3
- size 18133945952
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32e7052c8fe5e569d3333a72379711f6038d02b6dffbff3c345de1548d79f4b2
3
+ size 17863779976
processor_config.json DELETED
@@ -1,116 +0,0 @@
1
- {
2
- "feature_extractor": {
3
- "chunk_length": 300,
4
- "dither": 0.0,
5
- "feature_extractor_type": "WhisperFeatureExtractor",
6
- "feature_size": 128,
7
- "hop_length": 160,
8
- "image_mean": [
9
- 0.48145466,
10
- 0.4578275,
11
- 0.40821073
12
- ],
13
- "image_processor_type": "Qwen2VLImageProcessor",
14
- "image_std": [
15
- 0.26862954,
16
- 0.26130258,
17
- 0.27577711
18
- ],
19
- "max_pixels": 12845056,
20
- "merge_size": 2,
21
- "min_pixels": 3136,
22
- "n_fft": 400,
23
- "n_samples": 4800000,
24
- "nb_max_frames": 30000,
25
- "padding_side": "right",
26
- "padding_value": 0.0,
27
- "patch_size": 14,
28
- "return_attention_mask": true,
29
- "sampling_rate": 16000,
30
- "temporal_patch_size": 2
31
- },
32
- "image_processor": {
33
- "chunk_length": 300,
34
- "data_format": "channels_first",
35
- "dither": 0.0,
36
- "do_convert_rgb": true,
37
- "do_normalize": true,
38
- "do_rescale": true,
39
- "do_resize": true,
40
- "feature_size": 128,
41
- "hop_length": 160,
42
- "image_mean": [
43
- 0.48145466,
44
- 0.4578275,
45
- 0.40821073
46
- ],
47
- "image_processor_type": "Qwen2VLImageProcessorFast",
48
- "image_std": [
49
- 0.26862954,
50
- 0.26130258,
51
- 0.27577711
52
- ],
53
- "merge_size": 2,
54
- "n_fft": 400,
55
- "n_samples": 4800000,
56
- "nb_max_frames": 30000,
57
- "padding_side": "right",
58
- "padding_value": 0.0,
59
- "patch_size": 14,
60
- "resample": 3,
61
- "rescale_factor": 0.00392156862745098,
62
- "return_attention_mask": true,
63
- "sampling_rate": 16000,
64
- "size": {
65
- "longest_edge": 12845056,
66
- "shortest_edge": 3136
67
- },
68
- "temporal_patch_size": 2
69
- },
70
- "processor_class": "Qwen2_5OmniProcessor",
71
- "video_processor": {
72
- "chunk_length": 300,
73
- "data_format": "channels_first",
74
- "default_to_square": true,
75
- "dither": 0.0,
76
- "do_convert_rgb": true,
77
- "do_normalize": true,
78
- "do_rescale": true,
79
- "do_resize": true,
80
- "do_sample_frames": false,
81
- "feature_extractor_type": "WhisperFeatureExtractor",
82
- "feature_size": 128,
83
- "hop_length": 160,
84
- "image_mean": [
85
- 0.48145466,
86
- 0.4578275,
87
- 0.40821073
88
- ],
89
- "image_processor_type": "Qwen2VLImageProcessor",
90
- "image_std": [
91
- 0.26862954,
92
- 0.26130258,
93
- 0.27577711
94
- ],
95
- "max_frames": 768,
96
- "merge_size": 2,
97
- "min_frames": 4,
98
- "n_fft": 400,
99
- "n_samples": 4800000,
100
- "nb_max_frames": 30000,
101
- "padding_side": "right",
102
- "padding_value": 0.0,
103
- "patch_size": 14,
104
- "resample": 3,
105
- "rescale_factor": 0.00392156862745098,
106
- "return_attention_mask": true,
107
- "return_metadata": false,
108
- "sampling_rate": 16000,
109
- "size": {
110
- "longest_edge": 12845056,
111
- "shortest_edge": 3136
112
- },
113
- "temporal_patch_size": 2,
114
- "video_processor_type": "Qwen2VLVideoProcessor"
115
- }
116
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
special_tokens_map.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|AUDIO|>",
6
+ "<|audio_bos|>",
7
+ "<|audio_eos|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_bos|>",
12
+ "<|vision_eos|>",
13
+ "<|vision_pad|>",
14
+ "<|IMAGE|>",
15
+ "<|VIDEO|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ },
31
+ "image_token": "<|IMAGE|>",
32
+ "audio_token": "<|AUDIO|>",
33
+ "video_token": "<|VIDEO|>",
34
+ "vision_bos_token": "<|vision_bos|>",
35
+ "vision_eos_token": "<|vision_eos|>",
36
+ "audio_bos_token": "<|audio_bos|>",
37
+ "audio_eos_token": "<|audio_eos|>"
38
+ }
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0c75e6d39795d574e5ec741e767ca690ea08a33bfa024ef2d372b4e4c72db191
3
- size 11422137
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8441917e39ae0244e06d704b95b3124795cec478e297f9afac39ba670d7e9d99
3
+ size 11421870
tokenizer_config.json CHANGED
@@ -1,31 +1,223 @@
1
  {
2
  "add_prefix_space": false,
3
- "audio_bos_token": "<|audio_bos|>",
4
- "audio_eos_token": "<|audio_eos|>",
5
- "audio_token": "<|AUDIO|>",
6
- "backend": "tokenizers",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  "bos_token": null,
 
8
  "clean_up_tokenization_spaces": false,
9
  "eos_token": "<|im_end|>",
10
  "errors": "replace",
11
- "image_token": "<|IMAGE|>",
12
- "is_local": true,
13
- "model_max_length": 32768,
14
- "model_specific_special_tokens": {
15
- "audio_bos_token": "<|audio_bos|>",
16
- "audio_eos_token": "<|audio_eos|>",
17
- "audio_token": "<|AUDIO|>",
18
  "image_token": "<|IMAGE|>",
 
19
  "video_token": "<|VIDEO|>",
20
  "vision_bos_token": "<|vision_bos|>",
21
- "vision_eos_token": "<|vision_eos|>"
 
 
22
  },
 
23
  "pad_token": "<|endoftext|>",
24
  "processor_class": "Qwen2_5OmniProcessor",
25
  "split_special_tokens": false,
26
- "tokenizer_class": "TokenizersBackend",
27
  "unk_token": null,
 
 
28
  "video_token": "<|VIDEO|>",
29
  "vision_bos_token": "<|vision_bos|>",
30
- "vision_eos_token": "<|vision_eos|>"
31
- }
 
 
 
1
  {
2
  "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "151643": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "151644": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "151645": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "151646": {
29
+ "content": "<|AUDIO|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "151647": {
37
+ "content": "<|audio_bos|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "151648": {
45
+ "content": "<|audio_eos|>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "151649": {
53
+ "content": "<|box_end|>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "151650": {
61
+ "content": "<|quad_start|>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "151651": {
69
+ "content": "<|quad_end|>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "151652": {
77
+ "content": "<|vision_bos|>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "151653": {
85
+ "content": "<|vision_eos|>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "151654": {
93
+ "content": "<|vision_pad|>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "151655": {
101
+ "content": "<|IMAGE|>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "151656": {
109
+ "content": "<|VIDEO|>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "151657": {
117
+ "content": "<tool_call>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": false
123
+ },
124
+ "151658": {
125
+ "content": "</tool_call>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": false
131
+ },
132
+ "151659": {
133
+ "content": "<|fim_prefix|>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": false
139
+ },
140
+ "151660": {
141
+ "content": "<|fim_middle|>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": false
147
+ },
148
+ "151661": {
149
+ "content": "<|fim_suffix|>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": false
155
+ },
156
+ "151662": {
157
+ "content": "<|fim_pad|>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": false
163
+ },
164
+ "151663": {
165
+ "content": "<|repo_name|>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": false
171
+ },
172
+ "151664": {
173
+ "content": "<|file_sep|>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": false
179
+ }
180
+ },
181
+ "additional_special_tokens": [
182
+ "<|im_start|>",
183
+ "<|im_end|>",
184
+ "<|AUDIO|>",
185
+ "<|audio_bos|>",
186
+ "<|audio_eos|>",
187
+ "<|box_end|>",
188
+ "<|quad_start|>",
189
+ "<|quad_end|>",
190
+ "<|vision_bos|>",
191
+ "<|vision_eos|>",
192
+ "<|vision_pad|>",
193
+ "<|IMAGE|>",
194
+ "<|VIDEO|>"
195
+ ],
196
  "bos_token": null,
197
+ "chat_template": "{% set audio_count = namespace(value=0) %}{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_bos|><|IMAGE|><|vision_eos|>{% elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content %}{% set audio_count.value = audio_count.value + 1 %}{% if add_audio_id %}Audio {{ audio_count.value }}: {% endif %}<|audio_bos|><|AUDIO|><|audio_eos|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_bos|><|VIDEO|><|vision_eos|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
198
  "clean_up_tokenization_spaces": false,
199
  "eos_token": "<|im_end|>",
200
  "errors": "replace",
201
+ "extra_special_tokens": {
 
 
 
 
 
 
202
  "image_token": "<|IMAGE|>",
203
+ "audio_token": "<|AUDIO|>",
204
  "video_token": "<|VIDEO|>",
205
  "vision_bos_token": "<|vision_bos|>",
206
+ "vision_eos_token": "<|vision_eos|>",
207
+ "audio_bos_token": "<|audio_bos|>",
208
+ "audio_eos_token": "<|audio_eos|>"
209
  },
210
+ "model_max_length": 32768,
211
  "pad_token": "<|endoftext|>",
212
  "processor_class": "Qwen2_5OmniProcessor",
213
  "split_special_tokens": false,
214
+ "tokenizer_class": "Qwen2Tokenizer",
215
  "unk_token": null,
216
+ "image_token": "<|IMAGE|>",
217
+ "audio_token": "<|AUDIO|>",
218
  "video_token": "<|VIDEO|>",
219
  "vision_bos_token": "<|vision_bos|>",
220
+ "vision_eos_token": "<|vision_eos|>",
221
+ "audio_bos_token": "<|audio_bos|>",
222
+ "audio_eos_token": "<|audio_eos|>"
223
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff