File size: 3,502 Bytes
25169c2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | {
"add_prefix_space": false,
"audio_bos_token": "<|audio_start|>",
"audio_eos_token": "<|audio_end|>",
"audio_token": "<|audio_pad|>",
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": [
"<|image|>",
"<|video|>",
"<|sound|>",
"<|voice|>",
"<|listening|>",
"<|vision|>",
"<|mood_happy|>",
"<|mood_sad|>",
"<|mood_angry|>",
"<|mood_neutral|>",
"<fim_prefix>",
"<fim_middle>",
"<fim_suffix>",
"<|action_start|>",
"<|action_end|>",
"<|trajectory_start|>",
"<|trajectory_end|>",
"<|joint_start|>",
"<|joint_end|>",
"<|sensor_start|>",
"<|sensor_end|>",
"<|command_start|>",
"<|command_end|>",
"<|state_start|>",
"<|state_end|>",
"<|pose|>",
"<|velocity|>",
"<|force|>",
"<|torque|>",
"<|gripper|>",
"<|navigation|>",
"<|obstacle|>",
"<|task_start|>",
"<|task_end|>",
"<|plan_start|>",
"<|plan_end|>",
"<|behavior_start|>",
"<|behavior_end|>",
"<|skill_start|>",
"<|skill_end|>",
"<|motor|>",
"<|servo|>",
"<|imu|>",
"<|lidar|>",
"<|camera|>",
"<|depth|>",
"<|waypoint|>",
"<|path|>",
"<|collision|>",
"<|grasp|>",
"<|release|>",
"<|homing|>",
"<|emergency_stop|>",
"<|calibration|>",
"<|manipulation|>",
"<|locomotion|>",
"<|feedback|>",
"<|control_loop|>",
"<|language|>",
"<|tool_call_start|>",
"<|tool_call_end|>",
"<|tool_result_start|>",
"<|tool_result_end|>",
"__SCIENCE__",
"__CODING__",
"__STOCK_EXCHANGE__",
"__MEDICINE__",
"__GOVERNMENT__",
"__NEWS__",
"__GENERAL__",
"__MATERIAL_SCIENCE__",
"__ELECTRONICS__",
"__MICROELECTRONICS__",
"__ENGINEERING__",
"__ROBOTICS__",
"__ENERGY__",
"__AUTOMOTIVE__",
"__AVIATION__",
"__MATH__",
"__PYTHON__",
"__C__",
"__CPP__",
"__C_SHARP__",
"__JAVA__",
"__JAVASCRIPT__",
"__TYPESCRIPT__",
"__RUST__",
"__GO__",
"__RUBY__",
"__PHP__",
"__SWIFT__",
"__KOTLIN__",
"__BASH__",
"__SQL__",
"__ASSEMBLY__",
"__PHILOSOPHY__",
"__LITERATURE__",
"__SOCIOLOGY__",
"__PSYCHOLOGY__",
"__POLITICAL_SCIENCE__",
"__CULTURAL_STUDIES__",
"__ETHNOGRAPHY__",
"__HUMAN_RIGHTS__",
"__COMPLIANCE__",
"__MILITARY__",
"__BANKING__",
"__OIL_INDUSTRY__",
"__LIGHT_INDUSTRY__",
"__NATURE__",
"__OCEAN__",
"__SPORT__",
"__CULINARY__",
"__TRAVEL__",
"__HOBBY__"
],
"image_token": "<|image_pad|>",
"is_local": false,
"local_files_only": false,
"model_max_length": 262144,
"model_specific_special_tokens": {
"audio_bos_token": "<|audio_start|>",
"audio_eos_token": "<|audio_end|>",
"audio_token": "<|audio_pad|>",
"image_token": "<|image_pad|>",
"video_token": "<|video_pad|>",
"vision_bos_token": "<|vision_start|>",
"vision_eos_token": "<|vision_end|>"
},
"pad_token": "<|endoftext|>",
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null,
"video_token": "<|video_pad|>",
"vision_bos_token": "<|vision_start|>",
"vision_eos_token": "<|vision_end|>"
}
|