input_tokenizer/tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "<unk>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "<s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "</s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 53,
      "content": "<pad>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "Split",
    "pattern": {
      "String": ""
    },
    "behavior": "Isolated",
    "invert": false
  },
  "post_processor": null,
  "decoder": null,
  "model": {
    "type": "Unigram",
    "unk_id": null,
    "vocab": [
      [
        "<unk>",
        0.0
      ],
      [
        "<s>",
        0.0
      ],
      [
        "</s>",
        0.0
      ],
      [
        " ",
        -1.5286495954173631
      ],
      [
        "n",
        -2.4611217054515855
      ],
      [
        "t",
        -2.540307941605734
      ],
      [
        "i",
        -2.605613338169867
      ],
      [
        "h",
        -2.640926653183934
      ],
      [
        "s",
        -2.838120763773432
      ],
      [
        "k",
        -3.1211051609419993
      ],
      [
        "u",
        -3.2249282875507816
      ],
      [
        "g",
        -3.3411392475472077
      ],
      [
        "a",
        -3.444651584822539
      ],
      [
        "l",
        -3.841640698090009
      ],
      [
        "á",
        -3.8848449071201774
      ],
      [
        "o",
        -3.935398567362226
      ],
      [
        "̍",
        -4.132458522274371
      ],
      [
        "â",
        -4.212791854068779
      ],
      [
        "ā",
        -4.259196026935101
      ],
      [
        "p",
        -4.299807936667646
      ],
      [
        "b",
        -4.3836364827691074
      ],
      [
        "à",
        -4.419137992356868
      ],
      [
        "m",
        -4.511037313050792
      ],
      [
        "í",
        -4.53632471778387
      ],
      [
        "e",
        -4.707223106194123
      ],
      [
        "ó",
        -4.720895027230478
      ],
      [
        "ī",
        -4.804104578134297
      ],
      [
        "ì",
        -4.812366760121055
      ],
      [
        "ê",
        -4.839119095873702
      ],
      [
        "ē",
        -4.885699057727079
      ],
      [
        "ō",
        -4.97565684240295
      ],
      [
        "î",
        -5.060400335789719
      ],
      [
        "ū",
        -5.274313929865132
      ],
      [
        "ô",
        -5.322438446627812
      ],
      [
        "ò",
        -5.405901976707965
      ],
      [
        "ú",
        -5.618054771719086
      ],
      [
        "̄",
        -5.7788746101671045
      ],
      [
        "è",
        -5.7873365818392
      ],
      [
        "j",
        -5.927136012185764
      ],
      [
        "é",
        -6.051226621328267
      ],
      [
        "û",
        -6.330160852254361
      ],
      [
        "ù",
        -6.357189522929709
      ],
      [
        "̂",
        -7.126043441251755
      ],
      [
        "ǹ",
        -7.373551868781087
      ],
      [
        "ń",
        -7.57884578056758
      ],
      [
        "ḿ",
        -11.882371352168914
      ],
      [
        "0",
        -12.465704685502246
      ],
      [
        "1",
        -12.965704685502246
      ],
      [
        "2",
        -12.965704685502246
      ],
      [
        "9",
        -12.965704685502246
      ],
      [
        "̋",
        -12.965704685502246
      ],
      [
        "4",
        -13.965704685502246
      ],
      [
        "3",
        -13.965704685502246
      ]
    ],
    "byte_fallback": false
  }
}