tokenizer.json

{
  "version": 1,
  "type": "ByteLevelBPE",
  "config": {
    "vocab_size": 512,
    "bos_token": "<|beginoftext|>",
    "eos_token": "<|endoftext|>",
    "pad_token": "<|pad|>",
    "unk_token": "<|unk|>",
    "extra_special_tokens": [],
    "reserved_tokens": [],
    "pre_tokenize": null,
    "split_special": false
  },
  "merges": [
    [
      100,
      101
    ],
    [
      104,
      105
    ],
    [
      256,
      102
    ],
    [
      116,
      117
    ],
    [
      114,
      115
    ],
    [
      111,
      112
    ],
    [
      97,
      98
    ],
    [
      262,
      99
    ],
    [
      263,
      258
    ],
    [
      264,
      103
    ],
    [
      265,
      257
    ],
    [
      266,
      106
    ],
    [
      267,
      107
    ],
    [
      268,
      108
    ],
    [
      269,
      109
    ],
    [
      270,
      110
    ],
    [
      271,
      261
    ],
    [
      272,
      113
    ],
    [
      273,
      260
    ],
    [
      274,
      259
    ],
    [
      275,
      118
    ],
    [
      276,
      119
    ],
    [
      277,
      120
    ],
    [
      278,
      121
    ],
    [
      279,
      122
    ],
    [
      280,
      280
    ],
    [
      281,
      281
    ],
    [
      282,
      282
    ],
    [
      283,
      283
    ],
    [
      224,
      164
    ],
    [
      101,
      32
    ],
    [
      284,
      284
    ],
    [
      105,
      110
    ],
    [
      110,
      32
    ],
    [
      32,
      226
    ],
    [
      114,
      101
    ],
    [
      32,
      32
    ],
    [
      224,
      165
    ],
    [
      217,
      132
    ],
    [
      102,
      111
    ],
    [
      101,
      114
    ],
    [
      101,
      110
    ],
    [
      32,
      98
    ],
    [
      297,
      116
    ],
    [
      290,
      136
    ],
    [
      208,
      184
    ],
    [
      116,
      104
    ],
    [
      115,
      32
    ],
    [
      114,
      97
    ],
    [
      108,
      32
    ],
    [
      103,
      32
    ],
    [
      95,
      95
    ],
    [
      302,
      286
    ],
    [
      296,
      32
    ],
    [
      294,
      216
    ],
    [
      290,
      137
    ],
    [
      287,
      287
    ],
    [
      217,
      138
    ],
    [
      216,
      169
    ],
    [
      314,
      32
    ],
    [
      216,
      167
    ],
    [
      208,
      187
    ],
    [
      121,
      116
    ],
    [
      121,
      32
    ],
    [
      120,
      32
    ],
    [
      117,
      105
    ],
    [
      116,
      304
    ],
    [
      115,
      111
    ],
    [
      113,
      321
    ],
    [
      111,
      289
    ],
    [
      110,
      101
    ],
    [
      109,
      111
    ],
    [
      108,
      97
    ],
    [
      105,
      325
    ],
    [
      97,
      108
    ],
    [
      32,
      194
    ],
    [
      328,
      122
    ],
    [
      332,
      319
    ],
    [
      333,
      100
    ],
    [
      334,
      111
    ],
    [
      335,
      306
    ],
    [
      327,
      256
    ],
    [
      337,
      305
    ],
    [
      324,
      99
    ],
    [
      339,
      107
    ],
    [
      340,
      298
    ],
    [
      341,
      114
    ],
    [
      342,
      111
    ],
    [
      343,
      119
    ],
    [
      344,
      289
    ],
    [
      345,
      295
    ],
    [
      346,
      320
    ],
    [
      347,
      106
    ],
    [
      348,
      117
    ],
    [
      349,
      109
    ],
    [
      350,
      112
    ],
    [
      351,
      303
    ],
    [
      352,
      111
    ],
    [
      353,
      118
    ],
    [
      354,
      309
    ],
    [
      322,
      288
    ],
    [
      318,
      101
    ],
    [
      316,
      310
    ],
    [
      358,
      185
    ],
    [
      359,
      216
    ],
    [
      360,
      177
    ],
    [
      361,
      216
    ],
    [
      362,
      168
    ],
    [
      363,
      313
    ],
    [
      364,
      315
    ],
    [
      303,
      330
    ],
    [
      301,
      209
    ],
    [
      299,
      329
    ],
    [
      293,
      141
    ],
    [
      369,
      285
    ],
    [
      292,
      292
    ],
    [
      288,
      306
    ],
    [
      285,
      168
    ],
    [
      231,
      187
    ],
    [
      209,
      129
    ],
    [
      208,
      186
    ],
    [
      178,
      32
    ],
    [
      120,
      41
    ],
    [
      116,
      368
    ],
    [
      116,
      379
    ],
    [
      115,
      101
    ],
    [
      115,
      99
    ],
    [
      114,
      99
    ],
    [
      110,
      100
    ],
    [
      109,
      97
    ],
    [
      105,
      366
    ],
    [
      103,
      286
    ],
    [
      58,
      10
    ],
    [
      388,
      371
    ],
    [
      40,
      378
    ],
    [
      32,
      61
    ]
  ]
}