| {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}, "additional_special_tokens": ["string_content", "global_variable", "constant", "field_identifier", "none", "instance_variable", "string_fragment", "float_literal", "simple_symbol", "class_variable", "decimal_floating_point_literal", "shorthand_property_identifier", "comment", "escape_sequence", "text", "boolean_type", "\"", "label_name", "keyword", "property_identifier", "null_literal", "hash_key_symbol", "raw_string_literal", "shorthand_property_identifier_pattern", "type_identifier", "heredoc_beginning", "name", "identifier", "php_tag", "package_identifier", "namespace", "separators", "ERROR", "number", "hex_integer_literal", "boolean", "heredoc_end", "character_literal", "operator", "regex_pattern", "extends", "string", "integer", "statement_identifier", "string_literal", "decimal_integer_literal", "class", "regex_flags", "void_type", "rune_literal", "int_literal", "heredoc_content"]} |