Hugging Face's logo

ocolegro
/
custom-tokenizer

Model card Files
xet
 Community
custom-tokenizer / added_tokens.json
ocolegro's picture
ocolegro
Upload tokenizer
753d073
Raw History Blame Contribute Delete
1.93 kB
{
" ": 257,
"00": 258,
"01": 259,
"02": 260,
"03": 261,
"04": 262,
"05": 263,
"06": 264,
"07": 265,
"08": 266,
"09": 267,
"10": 268,
"11": 269,
"12": 270,
"13": 271,
"14": 272,
"15": 273,
"16": 274,
"17": 275,
"18": 276,
"19": 277,
"20": 278,
"21": 279,
"22": 280,
"23": 281,
"24": 282,
"25": 283,
"26": 284,
"27": 285,
"28": 286,
"29": 287,
"30": 288,
"31": 289,
"32": 290,
"33": 291,
"34": 292,
"35": 293,
"36": 294,
"37": 295,
"38": 296,
"39": 297,
"40": 298,
"41": 299,
"42": 300,
"43": 301,
"44": 302,
"45": 303,
"46": 304,
"47": 305,
"48": 306,
"49": 307,
"50": 308,
"51": 309,
"52": 310,
"53": 311,
"54": 312,
"55": 313,
"56": 314,
"57": 315,
"58": 316,
"59": 317,
"60": 318,
"61": 319,
"62": 320,
"63": 321,
"64": 322,
"65": 323,
"66": 324,
"67": 325,
"68": 326,
"69": 327,
"70": 328,
"71": 329,
"72": 330,
"73": 331,
"74": 332,
"75": 333,
"76": 334,
"77": 335,
"78": 336,
"79": 337,
"80": 338,
"81": 339,
"82": 340,
"83": 341,
"84": 342,
"85": 343,
"86": 344,
"87": 345,
"88": 346,
"89": 347,
"90": 348,
"91": 349,
"92": 350,
"93": 351,
"94": 352,
"95": 353,
"96": 354,
"97": 355,
"98": 356,
"99": 357,
"[PAD]": 398,
"e+00": 358,
"e+01": 360,
"e+02": 362,
"e+03": 364,
"e+04": 366,
"e+05": 368,
"e+06": 370,
"e+07": 372,
"e+08": 374,
"e+10": 378,
"e+11": 380,
"e+12": 382,
"e+13": 384,
"e+14": 386,
"e+15": 388,
"e+16": 390,
"e+17": 392,
"e+18": 394,
"e+19": 396,
"e+9": 376,
"e-00": 359,
"e-01": 361,
"e-02": 363,
"e-03": 365,
"e-04": 367,
"e-05": 369,
"e-06": 371,
"e-07": 373,
"e-08": 375,
"e-10": 379,
"e-11": 381,
"e-12": 383,
"e-13": 385,
"e-14": 387,
"e-15": 389,
"e-16": 391,
"e-17": 393,
"e-18": 395,
"e-19": 397,
"e-9": 377
}