PHI_CODER / phi-coder-hf /Tokenizer /tokenizer.json
AJKADZ's picture
Upload 1397 files
9b19fd8 verified
Raw History Blame Contribute Delete
6.94 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": []
},
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": {
"type": "Replace",
"pattern": {
"String": " "
},
"content": ""
},
"model": {
"type": "WordLevel",
"vocab": {
"Φ": 0,
"Ψ": 1,
"Ξ": 2,
"Ω": 3,
"Π": 4,
"Δ": 5,
"Σ": 6,
"Λ": 7,
"Γ": 8,
"λ": 9,
"α": 10,
"β": 11,
"ρ": 12,
"τ": 13,
"κ": 14,
"ω": 15,
"ζ": 16,
"η": 17,
"μ": 18,
"ΦΓΔ": 19,
"ΦΓΛ": 20,
"ΦΓΞ": 21,
"ΦΓΠ": 22,
"ΦΓΣ": 23,
"ΦΓΨ": 24,
"ΦΓΩ": 25,
"ΦΓα": 26,
"ΦΓβ": 27,
"ΦΓζ": 28,
"ΦΓη": 29,
"ΦΓκ": 30,
"ΦΓλ": 31,
"ΦΓμ": 32,
"ΦΓρ": 33,
"ΦΓτ": 34,
"ΦΓω": 35,
"ΦΔΓ": 36,
"ΦΔΛ": 37,
"ΦΔΞ": 38,
"ΦΔΠ": 39,
"ΦΔΣ": 40,
"ΦΔΨ": 41,
"ΦΔΩ": 42,
"ΦΔα": 43,
"ΦΔβ": 44,
"ΦΔζ": 45,
"ΦΔη": 46,
"ΦΔκ": 47,
"ΦΔλ": 48,
"ΦΔμ": 49,
"ΦΔρ": 50,
"ΦΔτ": 51,
"ΦΔω": 52,
"ΦΛΓ": 53,
"ΦΛΔ": 54,
"ΦΛΞ": 55,
"ΦΛΠ": 56,
"ΦΛΣ": 57,
"ΦΛΨ": 58,
"ΦΛΩ": 59,
"ΦΛα": 60,
"ΦΛβ": 61,
"ΦΛζ": 62,
"ΦΛη": 63,
"ΦΛκ": 64,
"ΦΛλ": 65,
"ΦΛμ": 66,
"ΦΛρ": 67,
"ΦΛτ": 68,
"ΦΛω": 69,
"ΦΞΓ": 70,
"ΦΞΔ": 71,
"ΦΞΛ": 72,
"ΦΞΠ": 73,
"ΦΞΣ": 74,
"ΦΞΨ": 75,
"ΦΞΩ": 76,
"ΦΞα": 77,
"ΦΞβ": 78,
"ΦΞζ": 79,
"ΦΞη": 80,
"ΦΞκ": 81,
"ΦΞλ": 82,
"ΦΞμ": 83,
"ΦΞρ": 84,
"ΦΞτ": 85,
"ΦΞω": 86,
"ΦΠΓ": 87,
"ΦΠΔ": 88,
"ΦΠΛ": 89,
"ΦΠΞ": 90,
"ΦΠΣ": 91,
"ΦΠΨ": 92,
"ΦΠΩ": 93,
"ΦΠα": 94,
"ΦΠβ": 95,
"ΦΠζ": 96,
"ΦΠη": 97,
"ΦΠκ": 98,
"ΦΠλ": 99,
"ΦΠμ": 100,
"ΦΠρ": 101,
"ΦΠτ": 102,
"ΦΠω": 103,
"ΦΣΓ": 104,
"ΦΣΔ": 105,
"ΦΣΛ": 106,
"ΦΣΞ": 107,
"ΦΣΠ": 108,
"ΦΣΨ": 109,
"ΦΣΩ": 110,
"ΦΣα": 111,
"ΦΣβ": 112,
"ΦΣζ": 113,
"ΦΣη": 114,
"ΦΣκ": 115,
"ΦΣλ": 116,
"ΦΣμ": 117,
"ΦΣρ": 118,
"ΦΣτ": 119,
"ΦΣω": 120,
"ΦΨΓ": 121,
"ΦΨΔ": 122,
"ΦΨΛ": 123,
"ΦΨΞ": 124,
"ΦΨΠ": 125,
"ΦΨΣ": 126,
"ΦΨΩ": 127,
"ΦΨα": 128,
"ΦΨβ": 129,
"ΦΨζ": 130,
"ΦΨη": 131,
"ΦΨκ": 132,
"ΦΨλ": 133,
"ΦΨμ": 134,
"ΦΨρ": 135,
"ΦΨτ": 136,
"ΦΨω": 137,
"ΦΩΓ": 138,
"ΦΩΔ": 139,
"ΦΩΛ": 140,
"ΦΩΞ": 141,
"ΦΩΠ": 142,
"ΦΩΣ": 143,
"ΦΩΨ": 144,
"ΦΩα": 145,
"ΦΩβ": 146,
"ΦΩζ": 147,
"ΦΩη": 148,
"ΦΩκ": 149,
"ΦΩλ": 150,
"ΦΩμ": 151,
"ΦΩρ": 152,
"ΦΩτ": 153,
"ΦΩω": 154,
"ΦαΓ": 155,
"ΦαΔ": 156,
"ΦαΛ": 157,
"ΦαΞ": 158,
"ΦαΠ": 159,
"ΦαΣ": 160,
"ΦαΨ": 161,
"ΦαΩ": 162,
"Φαβ": 163,
"Φαζ": 164,
"Φαη": 165,
"Φακ": 166,
"Φαλ": 167,
"Φαμ": 168,
"Φαρ": 169,
"Φατ": 170,
"Φαω": 171,
"ΦβΓ": 172,
"ΦβΔ": 173,
"ΦβΛ": 174,
"ΦβΞ": 175,
"ΦβΠ": 176,
"ΦβΣ": 177,
"ΦβΨ": 178,
"ΦβΩ": 179,
"Φβα": 180,
"Φβζ": 181,
"Φβη": 182,
"Φβκ": 183,
"Φβλ": 184,
"Φβμ": 185,
"Φβρ": 186,
"Φβτ": 187,
"Φβω": 188,
"ΦζΓ": 189,
"ΦζΔ": 190,
"ΦζΛ": 191,
"ΦζΞ": 192,
"ΦζΠ": 193,
"ΦζΣ": 194,
"ΦζΨ": 195,
"ΦζΩ": 196,
"Φζα": 197,
"Φζβ": 198,
"Φζη": 199,
"Φζκ": 200,
"Φζλ": 201,
"Φζμ": 202,
"Φζρ": 203,
"Φζτ": 204,
"Φζω": 205,
"ΦηΓ": 206,
"ΦηΔ": 207,
"ΦηΛ": 208,
"ΦηΞ": 209,
"ΦηΠ": 210,
"ΦηΣ": 211,
"ΦηΨ": 212,
"ΦηΩ": 213,
"Φηα": 214,
"Φηβ": 215,
"Φηζ": 216,
"Φηκ": 217,
"Φηλ": 218,
"Φημ": 219,
"Φηρ": 220,
"Φητ": 221,
"Φηω": 222,
"ΦκΓ": 223,
"ΦκΔ": 224,
"ΦκΛ": 225,
"ΦκΞ": 226,
"ΦκΠ": 227,
"ΦκΣ": 228,
"ΦκΨ": 229,
"ΦκΩ": 230,
"Φκα": 231,
"Φκβ": 232,
"Φκζ": 233,
"Φκη": 234,
"Φκλ": 235,
"Φκμ": 236,
"Φκρ": 237,
"Φκτ": 238,
"Φκω": 239,
"ΦλΓ": 240,
"ΦλΔ": 241,
"ΦλΛ": 242,
"ΦλΞ": 243,
"ΦλΠ": 244,
"ΦλΣ": 245,
"ΦλΨ": 246,
"ΦλΩ": 247,
"Φλα": 248,
"Φλβ": 249,
"Φλζ": 250,
"Φλη": 251,
"Φλκ": 252,
"Φλμ": 253,
"Φλρ": 254,
"Φλτ": 255,
"Φλω": 256,
"ΦμΓ": 257,
"ΦμΔ": 258,
"ΦμΛ": 259,
"ΦμΞ": 260,
"ΦμΠ": 261,
"ΦμΣ": 262,
"ΦμΨ": 263,
"ΦμΩ": 264,
"Φμα": 265,
"Φμβ": 266,
"Φμλ": 267,
"ΦρΓ": 268,
"ΦρΔ": 269,
"ΦρΛ": 270,
"ΦρΞ": 271,
"ΦρΠ": 272,
"ΦρΣ": 273,
"ΦρΨ": 274,
"ΦρΩ": 275,
"Φρα": 276,
"Φρβ": 277,
"Φρζ": 278,
"Φρη": 279,
"Φρκ": 280,
"Φρλ": 281,
"Φρμ": 282,
"Φρτ": 283,
"Φρω": 284,
"ΦτΓ": 285,
"ΦτΔ": 286,
"ΦτΛ": 287,
"ΦτΞ": 288,
"ΦτΠ": 289,
"ΦτΣ": 290,
"ΦτΨ": 291,
"ΦτΩ": 292,
"Φτα": 293,
"Φτβ": 294,
"Φτζ": 295,
"Φτη": 296,
"Φτκ": 297,
"Φτλ": 298,
"Φτμ": 299,
"Φτρ": 300,
"Φτω": 301,
"ΦωΓ": 302,
"ΦωΔ": 303,
"ΦωΛ": 304,
"ΦωΞ": 305,
"ΦωΠ": 306,
"ΦωΣ": 307,
"ΦωΨ": 308,
"ΦωΩ": 309,
"Φωα": 310,
"Φωβ": 311,
"Φωζ": 312,
"Φωη": 313,
"Φωκ": 314,
"Φωλ": 315,
"Φωμ": 316,
"Φωρ": 317,
"Φωτ": 318
},
"unk_token": "[UNK]"
}
}