CodeGPT-small-java1 / tokenizer_config.json
yanghuattt's picture
Upload tokenizer
0ae63bb verified
Raw History Blame
54.5 kB
{ "add_bos_token": false, "add_prefix_space": false, "added_tokens_decoder": { "0": { "content": "<s>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<pad>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "</s>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "50000": { "content": "<|UNKNOWN|>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "50001": { "content": "<EOL>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50002": { "content": "<STR_LIT>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50003": { "content": "<NUM_LIT>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50004": { "content": "<CHAR_LIT>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50005": { "content": "<STR_LIT:U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50006": { "content": "<STR_LIT:\\n>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50007": { "content": "<STR_LIT:/>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50008": { "content": "<STR_LIT:unchecked>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50009": { "content": "<STR_LIT:U+002C>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50010": { "content": "<STR_LIT:.>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50011": { "content": "<STR_LIT:name>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50012": { "content": "<STR_LIT:)>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50013": { "content": "<STR_LIT:1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50014": { "content": "<STR_LIT:U+002CU+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50015": { "content": "<STR_LIT:id>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50016": { "content": "<STR_LIT:a>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50017": { "content": "<STR_LIT::>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50018": { "content": "<STR_LIT:foo>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50019": { "content": "<STR_LIT:test>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50020": { "content": "<STR_LIT:]>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50021": { "content": "<STR_LIT:}>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50022": { "content": "<STR_LIT:0>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50023": { "content": "<STR_LIT:true>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50024": { "content": "<STR_LIT:'>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50025": { "content": "<STR_LIT:\\\">", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50026": { "content": "<STR_LIT:b>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50027": { "content": "<STR_LIT:value>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50028": { "content": "<STR_LIT:->", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50029": { "content": "<STR_LIT:UTF-8>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50030": { "content": "<STR_LIT:type>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50031": { "content": "<STR_LIT::U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50032": { "content": "<STR_LIT:unused>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50033": { "content": "<STR_LIT:2>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50034": { "content": "<STR_LIT:null>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50035": { "content": "<STR_LIT:_>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50036": { "content": "<STR_LIT:false>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50037": { "content": "<STR_LIT:A>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50038": { "content": "<STR_LIT:(>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50039": { "content": "<STR_LIT:bar>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50040": { "content": "<STR_LIT:=>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50041": { "content": "<STR_LIT:[>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50042": { "content": "<STR_LIT:x>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50043": { "content": "<STR_LIT:c>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50044": { "content": "<STR_LIT:;>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50045": { "content": "<STR_LIT:*>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50046": { "content": "<STR_LIT:2.0>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50047": { "content": "<STR_LIT:\\t>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50048": { "content": "<STR_LIT:3>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50049": { "content": "<STR_LIT:Doppelganger>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50050": { "content": "<STR_LIT:rawtypes>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50051": { "content": "<STR_LIT:serial>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50052": { "content": "<STR_LIT:{>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50053": { "content": "<STR_LIT:key>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50054": { "content": "<STR_LIT:>>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50055": { "content": "<STR_LIT:B>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50056": { "content": "<STR_LIT:deprecation>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50057": { "content": "<STR_LIT:C>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50058": { "content": "<STR_LIT:msg=>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50059": { "content": "<STR_LIT:%>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50060": { "content": "<STR_LIT:%s>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50061": { "content": "<STR_LIT:#>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50062": { "content": "<STR_LIT:}\\n>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50063": { "content": "<STR_LIT:title>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50064": { "content": "<STR_LIT:NotU+0020supportedU+0020yet.>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50065": { "content": "<STR_LIT:U+0020(>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50066": { "content": "<STR_LIT:description>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50067": { "content": "<STR_LIT:all>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50068": { "content": "<STR_LIT:d>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50069": { "content": "<STR_LIT:U+0020U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50070": { "content": "<STR_LIT:abc>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50071": { "content": "<STR_LIT:password>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50072": { "content": "<STR_LIT:data>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50073": { "content": "<STR_LIT:string>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50074": { "content": "<STR_LIT:user>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50075": { "content": "<STR_LIT:4>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50076": { "content": "<STR_LIT:localhost>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50077": { "content": "<STR_LIT:message>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50078": { "content": "<STR_LIT:text>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50079": { "content": "<STR_LIT:hello>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50080": { "content": "<STR_LIT:status>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50081": { "content": "<STR_LIT:default>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50082": { "content": "<STR_LIT:GET>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50083": { "content": "<STR_LIT:application/json>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50084": { "content": "<STR_LIT:?>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50085": { "content": "<STR_LIT:{}>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50086": { "content": "<STR_LIT:class>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50087": { "content": "<STR_LIT:y>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50088": { "content": "<STR_LIT:\\r\\n>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50089": { "content": "<STR_LIT:int>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50090": { "content": "<STR_LIT:_cvp_Core.PatientShort>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50091": { "content": "<STR_LIT:<>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50092": { "content": "<STR_LIT:version>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50093": { "content": "<STR_LIT:s>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50094": { "content": "<STR_LIT:error>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50095": { "content": "<STR_LIT:5>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50096": { "content": "<STR_LIT:'{0}'>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50097": { "content": "<STR_LIT:|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50098": { "content": "<STR_LIT:file>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50099": { "content": "<STR_LIT:one>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50100": { "content": "<STR_LIT:@>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50101": { "content": "<STR_LIT:url>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50102": { "content": "<STR_LIT:OK>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50103": { "content": "<STR_LIT:f>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50104": { "content": "<STR_LIT:U+0020U+0020U+0020U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50105": { "content": "<STR_LIT:en>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50106": { "content": "<STR_LIT:username>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50107": { "content": "<STR_LIT:U+0020-U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50108": { "content": "<STR_LIT:U+0020:U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50109": { "content": "<STR_LIT:POST>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50110": { "content": "<STR_LIT:&>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50111": { "content": "<STR_LIT:10>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50112": { "content": "<STR_LIT:none>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50113": { "content": "<STR_LIT:1.0>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50114": { "content": "<STR_LIT:D>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50115": { "content": "<STR_LIT:p>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50116": { "content": "<STR_LIT:e>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50117": { "content": "<STR_LIT:U+0020=U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50118": { "content": "<STR_LIT:Content-Type>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50119": { "content": "<STR_LIT:;U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50120": { "content": "<STR_LIT:content>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50121": { "content": "<STR_LIT:+>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50122": { "content": "<STR_LIT:m>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50123": { "content": "<STR_LIT:Test>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50124": { "content": "<STR_LIT:T>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50125": { "content": "<STR_LIT:http>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50126": { "content": "<STR_LIT:text/plain>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50127": { "content": "<STR_LIT:123>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50128": { "content": "<STR_LIT:127.0.0.1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50129": { "content": "<STR_LIT:U+0020toU+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50130": { "content": "<STR_LIT:path>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50131": { "content": "<STR_LIT:get>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50132": { "content": "<STR_LIT:ID>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50133": { "content": "<STR_LIT:Y>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50134": { "content": "<STR_LIT:code>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50135": { "content": "<STR_LIT:_cvp_Naes.Referral>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50136": { "content": "<STR_LIT:Name>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50137": { "content": "<STR_LIT:two>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50138": { "content": "<STR_LIT:U+0020U+0020}\\n>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50139": { "content": "<STR_LIT:count>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50140": { "content": "<STR_LIT:U+0020U+0020U+0020U+0020}\\n>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50141": { "content": "<STR_LIT:double>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50142": { "content": "<STR_LIT:...>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50143": { "content": "<STR_LIT:E>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50144": { "content": "<STR_LIT:X>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50145": { "content": "<STR_LIT:$>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50146": { "content": "<STR_LIT:long>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50147": { "content": "<STR_LIT:i>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50148": { "content": "<STR_LIT:list>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50149": { "content": "<STR_LIT:admin>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50150": { "content": "<STR_LIT:line.separator>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50151": { "content": "<STR_LIT:date>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50152": { "content": "<STR_LIT:!>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50153": { "content": "<STR_LIT:utf-8>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50154": { "content": "<STR_LIT:-1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50155": { "content": "<STR_LIT:field>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50156": { "content": "<STR_LIT:boolean>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50157": { "content": "<STR_LIT:success>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50158": { "content": "<STR_LIT:unknown>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50159": { "content": "<STR_LIT:\\\\>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50160": { "content": "<STR_LIT:r>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50161": { "content": "<STR_LIT:Z>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50162": { "content": "<STR_LIT:6>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50163": { "content": "<STR_LIT:result>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50164": { "content": "<STR_LIT:index>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50165": { "content": "<STR_LIT:S>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50166": { "content": "<STR_LIT:label>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50167": { "content": "<STR_LIT:root>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50168": { "content": "<STR_LIT:F>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50169": { "content": "<STR_LIT:start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50170": { "content": "<STR_LIT:number>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50171": { "content": "<STR_LIT:http://>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50172": { "content": "<STR_LIT:source>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50173": { "content": "<STR_LIT:I>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50174": { "content": "<STR_LIT:src>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50175": { "content": "<STR_LIT:.class>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50176": { "content": "<STR_LIT:state>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50177": { "content": "<STR_LIT:address>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50178": { "content": "<STR_LIT:target>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50179": { "content": "<STR_LIT:body>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50180": { "content": "<STR_LIT:\\\\.>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50181": { "content": "<STR_LIT:email>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50182": { "content": "<STR_LIT:U+0020andU+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50183": { "content": "<STR_LIT:N>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50184": { "content": "<STR_LIT:t>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50185": { "content": "<STR_LIT:value1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50186": { "content": "<STR_LIT:-->", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50187": { "content": "<STR_LIT:String>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50188": { "content": "<STR_LIT:Hello>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50189": { "content": "<STR_LIT:yes>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50190": { "content": "<STR_LIT:baz>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50191": { "content": "<STR_LIT:text/html>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50192": { "content": "<STR_LIT:q>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50193": { "content": "<STR_LIT:size>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50194": { "content": "<STR_LIT:n>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50195": { "content": "<STR_LIT:U+0020U+0020}>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50196": { "content": "<STR_LIT:M>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50197": { "content": "<STR_LIT:key1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50198": { "content": "<STR_LIT:userId>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50199": { "content": "<STR_LIT:cast>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50200": { "content": "<STR_LIT:test1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50201": { "content": "<STR_LIT:[]>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50202": { "content": "<STR_LIT:float>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50203": { "content": "<STR_LIT:query>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50204": { "content": "<STR_LIT:GMT>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50205": { "content": "<NUM_LIT:0>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50206": { "content": "<NUM_LIT:1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50207": { "content": "<NUM_LIT:2>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50208": { "content": "<NUM_LIT:3>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50209": { "content": "<NUM_LIT:4>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50210": { "content": "<NUM_LIT:5>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50211": { "content": "<NUM_LIT:10>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50212": { "content": "<NUM_LIT:8>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50213": { "content": "<NUM_LIT:6>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50214": { "content": "<NUM_LIT:7>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50215": { "content": "<NUM_LIT:16>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50216": { "content": "<NUM_LIT:100>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50217": { "content": "<NUM_LIT:9>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50218": { "content": "<NUM_LIT:1000>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50219": { "content": "<NUM_LIT:12>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50220": { "content": "<NUM_LIT:1L>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50221": { "content": "<NUM_LIT:20>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50222": { "content": "<NUM_LIT:0F>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50223": { "content": "<NUM_LIT:11>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50224": { "content": "<NUM_LIT:0.0>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50225": { "content": "<NUM_LIT:32>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50226": { "content": "<NUM_LIT:0.0f>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50227": { "content": "<NUM_LIT:15>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50228": { "content": "<NUM_LIT:255>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50229": { "content": "<NUM_LIT:0x00>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50230": { "content": "<NUM_LIT:1.0>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50231": { "content": "<NUM_LIT:30>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50232": { "content": "<NUM_LIT:31>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50233": { "content": "<NUM_LIT:24>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50234": { "content": "<NUM_LIT:1.0f>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50235": { "content": "<CHAR_LIT:U+0020>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50236": { "content": "<CHAR_LIT:.>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50237": { "content": "<CHAR_LIT:\\n>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50238": { "content": "<CHAR_LIT:/>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50239": { "content": "<CHAR_LIT:0>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50240": { "content": "<CHAR_LIT:\\'>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50241": { "content": "<CHAR_LIT:a>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50242": { "content": "<CHAR_LIT::>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50243": { "content": "<CHAR_LIT:}>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50244": { "content": "<CHAR_LIT:U+002C>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50245": { "content": "<CHAR_LIT:->", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50246": { "content": "<CHAR_LIT:A>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50247": { "content": "<CHAR_LIT:\\\\>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50248": { "content": "<CHAR_LIT:\">", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50249": { "content": "<CHAR_LIT:)>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50250": { "content": "<CHAR_LIT:=>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50251": { "content": "<CHAR_LIT:9>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50252": { "content": "<CHAR_LIT:\\r>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50253": { "content": "<CHAR_LIT:[>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50254": { "content": "<CHAR_LIT:]>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50255": { "content": "<CHAR_LIT:1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50256": { "content": "<CHAR_LIT:(>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50257": { "content": "<CHAR_LIT:;>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50258": { "content": "<CHAR_LIT:_>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50259": { "content": "<CHAR_LIT:\\t>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50260": { "content": "<CHAR_LIT:e>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50261": { "content": "<CHAR_LIT:c>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50262": { "content": "<CHAR_LIT:>>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50263": { "content": "<CHAR_LIT:Z>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50264": { "content": "<CHAR_LIT:b>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<STR_LIT>", "<NUM_LIT>", "<CHAR_LIT>", "<STR_LIT:U+0020>", "<STR_LIT:\\n>", "<STR_LIT:/>", "<STR_LIT:unchecked>", "<STR_LIT:U+002C>", "<STR_LIT:.>", "<STR_LIT:name>", "<STR_LIT:)>", "<STR_LIT:1>", "<STR_LIT:U+002CU+0020>", "<STR_LIT:id>", "<STR_LIT:a>", "<STR_LIT::>", "<STR_LIT:foo>", "<STR_LIT:test>", "<STR_LIT:]>", "<STR_LIT:}>", "<STR_LIT:0>", "<STR_LIT:true>", "<STR_LIT:'>", "<STR_LIT:\\\">", "<STR_LIT:b>", "<STR_LIT:value>", "<STR_LIT:->", "<STR_LIT:UTF-8>", "<STR_LIT:type>", "<STR_LIT::U+0020>", "<STR_LIT:unused>", "<STR_LIT:2>", "<STR_LIT:null>", "<STR_LIT:_>", "<STR_LIT:false>", "<STR_LIT:A>", "<STR_LIT:(>", "<STR_LIT:bar>", "<STR_LIT:=>", "<STR_LIT:[>", "<STR_LIT:x>", "<STR_LIT:c>", "<STR_LIT:;>", "<STR_LIT:*>", "<STR_LIT:2.0>", "<STR_LIT:\\t>", "<STR_LIT:3>", "<STR_LIT:Doppelganger>", "<STR_LIT:rawtypes>", "<STR_LIT:serial>", "<STR_LIT:{>", "<STR_LIT:key>", "<STR_LIT:>>", "<STR_LIT:B>", "<STR_LIT:deprecation>", "<STR_LIT:C>", "<STR_LIT:msg=>", "<STR_LIT:%>", "<STR_LIT:%s>", "<STR_LIT:#>", "<STR_LIT:}\\n>", "<STR_LIT:title>", "<STR_LIT:NotU+0020supportedU+0020yet.>", "<STR_LIT:U+0020(>", "<STR_LIT:description>", "<STR_LIT:all>", "<STR_LIT:d>", "<STR_LIT:U+0020U+0020>", "<STR_LIT:abc>", "<STR_LIT:password>", "<STR_LIT:data>", "<STR_LIT:string>", "<STR_LIT:user>", "<STR_LIT:4>", "<STR_LIT:localhost>", "<STR_LIT:message>", "<STR_LIT:text>", "<STR_LIT:hello>", "<STR_LIT:status>", "<STR_LIT:default>", "<STR_LIT:GET>", "<STR_LIT:application/json>", "<STR_LIT:?>", "<STR_LIT:{}>", "<STR_LIT:class>", "<STR_LIT:y>", "<STR_LIT:\\r\\n>", "<STR_LIT:int>", "<STR_LIT:_cvp_Core.PatientShort>", "<STR_LIT:<>", "<STR_LIT:version>", "<STR_LIT:s>", "<STR_LIT:error>", "<STR_LIT:5>", "<STR_LIT:'{0}'>", "<STR_LIT:|>", "<STR_LIT:file>", "<STR_LIT:one>", "<STR_LIT:@>", "<STR_LIT:url>", "<STR_LIT:OK>", "<STR_LIT:f>", "<STR_LIT:U+0020U+0020U+0020U+0020>", "<STR_LIT:en>", "<STR_LIT:username>", "<STR_LIT:U+0020-U+0020>", "<STR_LIT:U+0020:U+0020>", "<STR_LIT:POST>", "<STR_LIT:&>", "<STR_LIT:10>", "<STR_LIT:none>", "<STR_LIT:1.0>", "<STR_LIT:D>", "<STR_LIT:p>", "<STR_LIT:e>", "<STR_LIT:U+0020=U+0020>", "<STR_LIT:Content-Type>", "<STR_LIT:;U+0020>", "<STR_LIT:content>", "<STR_LIT:+>", "<STR_LIT:m>", "<STR_LIT:Test>", "<STR_LIT:T>", "<STR_LIT:http>", "<STR_LIT:text/plain>", "<STR_LIT:123>", "<STR_LIT:127.0.0.1>", "<STR_LIT:U+0020toU+0020>", "<STR_LIT:path>", "<STR_LIT:get>", "<STR_LIT:ID>", "<STR_LIT:Y>", "<STR_LIT:code>", "<STR_LIT:_cvp_Naes.Referral>", "<STR_LIT:Name>", "<STR_LIT:two>", "<STR_LIT:U+0020U+0020}\\n>", "<STR_LIT:count>", "<STR_LIT:U+0020U+0020U+0020U+0020}\\n>", "<STR_LIT:double>", "<STR_LIT:...>", "<STR_LIT:E>", "<STR_LIT:X>", "<STR_LIT:$>", "<STR_LIT:long>", "<STR_LIT:i>", "<STR_LIT:list>", "<STR_LIT:admin>", "<STR_LIT:line.separator>", "<STR_LIT:date>", "<STR_LIT:!>", "<STR_LIT:utf-8>", "<STR_LIT:-1>", "<STR_LIT:field>", "<STR_LIT:boolean>", "<STR_LIT:success>", "<STR_LIT:unknown>", "<STR_LIT:\\\\>", "<STR_LIT:r>", "<STR_LIT:Z>", "<STR_LIT:6>", "<STR_LIT:result>", "<STR_LIT:index>", "<STR_LIT:S>", "<STR_LIT:label>", "<STR_LIT:root>", "<STR_LIT:F>", "<STR_LIT:start>", "<STR_LIT:number>", "<STR_LIT:http://>", "<STR_LIT:source>", "<STR_LIT:I>", "<STR_LIT:src>", "<STR_LIT:.class>", "<STR_LIT:state>", "<STR_LIT:address>", "<STR_LIT:target>", "<STR_LIT:body>", "<STR_LIT:\\\\.>", "<STR_LIT:email>", "<STR_LIT:U+0020andU+0020>", "<STR_LIT:N>", "<STR_LIT:t>", "<STR_LIT:value1>", "<STR_LIT:-->", "<STR_LIT:String>", "<STR_LIT:Hello>", "<STR_LIT:yes>", "<STR_LIT:baz>", "<STR_LIT:text/html>", "<STR_LIT:q>", "<STR_LIT:size>", "<STR_LIT:n>", "<STR_LIT:U+0020U+0020}>", "<STR_LIT:M>", "<STR_LIT:key1>", "<STR_LIT:userId>", "<STR_LIT:cast>", "<STR_LIT:test1>", "<STR_LIT:[]>", "<STR_LIT:float>", "<STR_LIT:query>", "<STR_LIT:GMT>", "<NUM_LIT:0>", "<NUM_LIT:1>", "<NUM_LIT:2>", "<NUM_LIT:3>", "<NUM_LIT:4>", "<NUM_LIT:5>", "<NUM_LIT:10>", "<NUM_LIT:8>", "<NUM_LIT:6>", "<NUM_LIT:7>", "<NUM_LIT:16>", "<NUM_LIT:100>", "<NUM_LIT:9>", "<NUM_LIT:1000>", "<NUM_LIT:12>", "<NUM_LIT:1L>", "<NUM_LIT:20>", "<NUM_LIT:0F>", "<NUM_LIT:11>", "<NUM_LIT:0.0>", "<NUM_LIT:32>", "<NUM_LIT:0.0f>", "<NUM_LIT:15>", "<NUM_LIT:255>", "<NUM_LIT:0x00>", "<NUM_LIT:1.0>", "<NUM_LIT:30>", "<NUM_LIT:31>", "<NUM_LIT:24>", "<NUM_LIT:1.0f>", "<CHAR_LIT:U+0020>", "<CHAR_LIT:.>", "<CHAR_LIT:\\n>", "<CHAR_LIT:/>", "<CHAR_LIT:0>", "<CHAR_LIT:\\'>", "<CHAR_LIT:a>", "<CHAR_LIT::>", "<CHAR_LIT:}>", "<CHAR_LIT:U+002C>", "<CHAR_LIT:->", "<CHAR_LIT:A>", "<CHAR_LIT:\\\\>", "<CHAR_LIT:\">", "<CHAR_LIT:)>", "<CHAR_LIT:=>", "<CHAR_LIT:9>", "<CHAR_LIT:\\r>", "<CHAR_LIT:[>", "<CHAR_LIT:]>", "<CHAR_LIT:1>", "<CHAR_LIT:(>", "<CHAR_LIT:;>", "<CHAR_LIT:_>", "<CHAR_LIT:\\t>", "<CHAR_LIT:e>", "<CHAR_LIT:c>", "<CHAR_LIT:>>", "<CHAR_LIT:Z>", "<CHAR_LIT:b>" ], "bos_token": "<s>", "clean_up_tokenization_spaces": false, "do_lower_case": false, "eos_token": "</s>", "errors": "replace", "extra_special_tokens": {}, "full_tokenizer_file": null, "model_max_length": 1000000000000000019884624838656, "pad_token": "<pad>", "sep_token": "<EOL>", "tokenizer_class": "GPT2Tokenizer", "unk_token": "<|UNKNOWN|>" }
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"0": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<pad>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"50000": {
"content": "<|UNKNOWN|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"50001": {
"content": "<EOL>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50002": {
"content": "<STR_LIT>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50003": {
"content": "<NUM_LIT>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50004": {
"content": "<CHAR_LIT>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50005": {
"content": "<STR_LIT:U+0020>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50006": {
"content": "<STR_LIT:\\n>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50007": {
"content": "<STR_LIT:/>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50008": {
"content": "<STR_LIT:unchecked>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},