vytre_core / tokenizer.json
itstalmeez's picture
Upload 8 files
0037241 verified
Raw History Blame Contribute Delete
10.7 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 128,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 0,
"pad_type_id": 0,
"pad_token": "<pad>"
},
"added_tokens": [
{
"id": 0,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "WhitespaceSplit"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
}
],
"special_tokens": {
"</s>": {
"id": "</s>",
"ids": [
3
],
"tokens": [
"</s>"
]
},
"<s>": {
"id": "<s>",
"ids": [
2
],
"tokens": [
"<s>"
]
}
}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"<pad>": 0,
"<unk>": 1,
"<s>": 2,
"</s>": 3,
"\"30d\"}}": 4,
"\"Account": 5,
"\"Accounting": 6,
"\"Affects": 7,
"\"Analytics": 8,
"\"Analytics\":": 9,
"\"Analyze": 10,
"\"Assign": 11,
"\"BDR": 12,
"\"Backend": 13,
"\"Budget": 14,
"\"Bug": 15,
"\"Build": 16,
"\"CMO\":": 17,
"\"CS": 18,
"\"CSM": 19,
"\"CTO\":": 20,
"\"Campaign": 21,
"\"Close": 22,
"\"Closing\"],": 23,
"\"Cloud": 24,
"\"Complete": 25,
"\"Compliance": 26,
"\"Content": 27,
"\"Contract": 28,
"\"Customer": 29,
"\"Data": 30,
"\"Database": 31,
"\"Demo": 32,
"\"Deploy": 33,
"\"Design": 34,
"\"Design\",": 35,
"\"DevOps": 36,
"\"Engineering": 37,
"\"Engineering\",": 38,
"\"Escalation": 39,
"\"Escalation\"],": 40,
"\"Finance": 41,
"\"Finance\",": 42,
"\"Finance\":": 43,
"\"Frontend": 44,
"\"Graphic": 45,
"\"HR": 46,
"\"HR\",": 47,
"\"HR\":": 48,
"\"Interview\",": 49,
"\"Legal": 50,
"\"Legal\",": 51,
"\"Logistics": 52,
"\"Low-risk": 53,
"\"Marketing": 54,
"\"Marketing\",": 55,
"\"Minor": 56,
"\"Monitoring\"],": 57,
"\"Negotiate\",": 58,
"\"Offer\",": 59,
"\"Onboard\"]}": 60,
"\"Onboarding": 61,
"\"Operations": 62,
"\"Operations\",": 63,
"\"Pricing": 64,
"\"Process": 65,
"\"Product": 66,
"\"Product\",": 67,
"\"Product\":": 68,
"\"QA": 69,
"\"Recruiting": 70,
"\"Regression": 71,
"\"Renewals": 72,
"\"Reporting\"],": 73,
"\"Resolve": 74,
"\"Responsive": 75,
"\"Run": 76,
"\"SEO": 77,
"\"Sales": 78,
"\"Sales\",": 79,
"\"Schedule": 80,
"\"Screening": 81,
"\"Security": 82,
"\"Security\",": 83,
"\"Send": 84,
"\"Social": 85,
"\"Support": 86,
"\"Support\",": 87,
"\"Support\":": 88,
"\"System": 89,
"\"Tier": 90,
"\"UI": 91,
"\"UX": 92,
"\"Welcome\"}}": 93,
"\"access_db\"]}": 94,
"\"active\"}}": 95,
"\"agent.deploy\",": 96,
"\"analytics.report\",": 97,
"\"conversion\",": 98,
"\"customer@example.com\",": 99,
"\"customer_onboarding\"}}": 100,
"\"customers\",": 101,
"\"database.query\",": 102,
"\"deploy_code\"]}": 103,
"\"email.send\",": 104,
"\"filter\":": 105,
"\"manager\":": 106,
"\"parameters\":": 107,
"\"period\":": 108,
"\"permissions\":": 109,
"\"publish_content\"]}": 110,
"\"reason\":": 111,
"\"skills\":": 112,
"\"subject\":": 113,
"\"task.create\",": 114,
"\"update_deals\"]}": 115,
"\"workers\":": 116,
"\"workflow.create\",": 117,
"2": 118,
"30%": 119,
"Add": 120,
"Agent\",": 121,
"Agent\"]}": 122,
"Agent\"}}": 123,
"Analyst": 124,
"Build": 125,
"Change": 126,
"Counsel\",": 127,
"Create": 128,
"Delete": 129,
"Deploy": 130,
"Design": 131,
"FAQ": 132,
"Fire": 133,
"Get": 134,
"Improve": 135,
"Increase": 136,
"Input:": 137,
"Launch": 138,
"Manager": 139,
"Manager\",": 140,
"Manager\"]}": 141,
"Media": 142,
"Open": 143,
"Output:": 144,
"Q4": 145,
"Send": 146,
"Success\",": 147,
"Success\":": 148,
"Support": 149,
"Team\":": 150,
"UI": 151,
"Update": 152,
"Vytre,": 153,
"You": 154,
"[\"API": 155,
"[\"Account": 156,
"[\"Accounting": 157,
"[\"Add": 158,
"[\"Allocate": 159,
"[\"Analytics": 160,
"[\"Analyze": 161,
"[\"BDR": 162,
"[\"Backend": 163,
"[\"Budget": 164,
"[\"CI/CD\",": 165,
"[\"CSM": 166,
"[\"Code": 167,
"[\"Compliance": 168,
"[\"Content": 169,
"[\"Contract": 170,
"[\"Create": 171,
"[\"Define": 172,
"[\"DevOps": 173,
"[\"Escalation": 174,
"[\"Estimate": 175,
"[\"Find": 176,
"[\"Frontend": 177,
"[\"Graphic": 178,
"[\"Hire": 179,
"[\"Improve": 180,
"[\"Invoicing\",": 181,
"[\"Lead": 182,
"[\"Logistics": 183,
"[\"Onboarding": 184,
"[\"Plan": 185,
"[\"Prepare": 186,
"[\"Process": 187,
"[\"Product": 188,
"[\"QA": 189,
"[\"Qualify": 190,
"[\"React": 191,
"[\"Receive": 192,
"[\"Recruiting": 193,
"[\"Renewals": 194,
"[\"Review": 195,
"[\"SEO": 196,
"[\"Sales": 197,
"[\"Security": 198,
"[\"Social": 199,
"[\"Support": 200,
"[\"Test": 201,
"[\"Ticket": 202,
"[\"Tier": 203,
"[\"Track": 204,
"[\"UI": 205,
"[\"UX": 206,
"[\"deploy_code\",": 207,
"[\"deploy_code\"]}": 208,
"[\"manage_servers\",": 209,
"[\"read_analytics\",": 210,
"[\"read_customer_records\"]}": 211,
"[\"read_financials\"]}": 212,
"[\"read_leads\",": 213,
"[\"read_test_cases\"]}": 214,
"account\",": 215,
"active": 216,
"ads\"],": 217,
"affect": 218,
"agent": 219,
"agent\",": 220,
"an": 221,
"analysis\",": 222,
"and": 223,
"architecture\"],": 224,
"are": 225,
"backend": 226,
"budget\"]}": 227,
"button": 228,
"by": 229,
"call\",": 230,
"campaign": 231,
"campaign\"]}": 232,
"change\"}": 233,
"changes": 234,
"churn\"],": 235,
"color": 236,
"compliance": 237,
"content": 238,
"content\",": 239,
"conversion": 240,
"creation\",": 241,
"customer": 242,
"customers": 243,
"customers\"}": 244,
"cycle": 245,
"data": 246,
"deal\"]}": 247,
"demo\",": 248,
"department": 249,
"design": 250,
"design\",": 251,
"design\"],": 252,
"development\",": 253,
"devops": 254,
"email": 255,
"email\",": 256,
"employee": 257,
"engineering": 258,
"enterprise": 259,
"experience\"}": 260,
"false,": 261,
"features\"]}": 262,
"finance": 263,
"for": 264,
"frontend": 265,
"hiring": 266,
"hours": 267,
"hr": 268,
"implementation\",": 269,
"implications\"}": 270,
"infrastructure\",": 271,
"infrastructure\"],": 272,
"intelligence": 273,
"issue\",": 274,
"launch": 275,
"lead\",": 276,
"legal": 277,
"management\"],": 278,
"marketing": 279,
"meeting\"]}": 280,
"model.": 281,
"new": 282,
"objectives\"],": 283,
"office": 284,
"onboarding": 285,
"operating": 286,
"operations": 287,
"optimization\",": 288,
"package\",": 289,
"page": 290,
"performance\"]}": 291,
"planning\",": 292,
"pricing": 293,
"privacy": 294,
"process": 295,
"product": 296,
"production\"]}": 297,
"proposal\",": 298,
"qa": 299,
"qualification\",": 300,
"quarterly": 301,
"release": 302,
"report": 303,
"report\"}}": 304,
"reporting\",": 305,
"requested": 306,
"response": 307,
"response\",": 308,
"resume\",": 309,
"retention": 310,
"review\",": 311,
"risk\"}": 312,
"sales": 313,
"scheduling\",": 314,
"security": 315,
"software": 316,
"space\"],": 317,
"staff\"],": 318,
"staging\",": 319,
"strategy\"],": 320,
"success": 321,
"support": 322,
"task": 323,
"testing\"],": 324,
"tests\",": 325,
"ticket\",": 326,
"ticket\"]}": 327,
"time\"],": 328,
"to": 329,
"tracking\",": 330,
"true,": 331,
"visual": 332,
"welcome": 333,
"workflow": 334,
"workforce": 335,
"{\"CEO\":": 336,
"{\"CMO\":": 337,
"{\"Customer": 338,
"{\"Operations\":": 339,
"{\"department\":": 340,
"{\"metric\":": 341,
"{\"name\":": 342,
"{\"requires_approval\":": 343,
"{\"table\":": 344,
"{\"title\":": 345,
"{\"to\":": 346,
"{\"tool\":": 347,
"{\"workflow\":": 348
},
"unk_token": "<unk>"
}
}