vytre-core / tokenizer.json
itstalmeez's picture
Upload 5 files
6a0c881 verified
Raw
History Blame Contribute Delete
10.7 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 128,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": {
"Fixed": 128
},
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 0,
"pad_type_id": 0,
"pad_token": "<pad>"
},
"added_tokens": [
{
"id": 0,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
}
],
"special_tokens": {
"</s>": {
"id": "</s>",
"ids": [
3
],
"tokens": [
"</s>"
]
},
"<s>": {
"id": "<s>",
"ids": [
2
],
"tokens": [
"<s>"
]
}
}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"<pad>": 0,
"<unk>": 1,
"<s>": 2,
"</s>": 3,
"\"Product": 4,
"\"Product\",": 5,
"\"UX": 6,
"\"manager\":": 7,
"\"workers\":": 8,
"Agent\",": 9,
"Agent\"]}": 10,
"Create": 11,
"Input:": 12,
"Manager": 13,
"Manager\",": 14,
"Output:": 15,
"Vytre,": 16,
"You": 17,
"[\"Product": 18,
"an": 19,
"are": 20,
"department": 21,
"enterprise": 22,
"intelligence": 23,
"model.": 24,
"operating": 25,
"product": 26,
"workforce": 27,
"{\"department\":": 28,
"\"Security": 29,
"\"Security\",": 30,
"Analyst": 31,
"[\"Compliance": 32,
"security": 33,
"\"CS": 34,
"\"Customer": 35,
"\"Renewals": 36,
"Success\",": 37,
"[\"CSM": 38,
"customer": 39,
"success": 40,
"\"Operations": 41,
"\"Operations\",": 42,
"\"Process": 43,
"[\"Logistics": 44,
"operations": 45,
"[\"UX": 46,
"\"Contract": 47,
"\"Legal": 48,
"\"Legal\",": 49,
"Counsel\",": 50,
"legal": 51,
"\"Compliance": 52,
"[\"Contract": 53,
"\"Account": 54,
"\"Sales": 55,
"\"Sales\",": 56,
"[\"BDR": 57,
"sales": 58,
"\"Support": 59,
"\"Support\",": 60,
"\"Tier": 61,
"2": 62,
"Support": 63,
"[\"Escalation": 64,
"support": 65,
"\"CSM": 66,
"[\"Renewals": 67,
"[\"Account": 68,
"\"HR": 69,
"\"HR\",": 70,
"\"Recruiting": 71,
"[\"Onboarding": 72,
"hr": 73,
"\"Marketing": 74,
"\"Marketing\",": 75,
"\"Social": 76,
"Media": 77,
"[\"Analytics": 78,
"marketing": 79,
"\"Onboarding": 80,
"[\"Recruiting": 81,
"[\"Support": 82,
"\"Design": 83,
"\"Design\",": 84,
"\"UI": 85,
"Design": 86,
"[\"Graphic": 87,
"design": 88,
"\"BDR": 89,
"\"Budget": 90,
"\"Finance": 91,
"\"Finance\",": 92,
"[\"Accounting": 93,
"finance": 94,
"\"Logistics": 95,
"[\"Process": 96,
"\"Escalation": 97,
"[\"Tier": 98,
"\"Content": 99,
"[\"SEO": 100,
"\"SEO": 101,
"\"DevOps": 102,
"\"Engineering": 103,
"\"Engineering\",": 104,
"\"Frontend": 105,
"\"QA": 106,
"[\"Backend": 107,
"engineering": 108,
"\"Accounting": 109,
"[\"Budget": 110,
"\"Analytics": 111,
"[\"Content": 112,
"Manager\"]}": 113,
"[\"Sales": 114,
"[\"Social": 115,
"\"Backend": 116,
"[\"DevOps": 117,
"[\"Security": 118,
"\"Graphic": 119,
"[\"UI": 120,
"[\"QA": 121,
"[\"Frontend": 122,
"\"Cloud": 123,
"\"Monitoring\"],": 124,
"\"deploy_code\"]}": 125,
"\"permissions\":": 126,
"\"skills\":": 127,
"[\"CI/CD\",": 128,
"[\"manage_servers\",": 129,
"agent": 130,
"devops": 131,
"infrastructure\",": 132,
"{\"name\":": 133,
"\"Campaign": 134,
"\"publish_content\"]}": 135,
"[\"read_analytics\",": 136,
"creation\",": 137,
"management\"],": 138,
"optimization\",": 139,
"\"Database": 140,
"\"System": 141,
"\"access_db\"]}": 142,
"[\"API": 143,
"[\"deploy_code\",": 144,
"architecture\"],": 145,
"backend": 146,
"design\",": 147,
"development\",": 148,
"\"Bug": 149,
"\"Regression": 150,
"[\"Test": 151,
"[\"read_test_cases\"]}": 152,
"planning\",": 153,
"qa": 154,
"reporting\",": 155,
"testing\"],": 156,
"\"Escalation\"],": 157,
"[\"Ticket": 158,
"[\"read_customer_records\"]}": 159,
"analysis\",": 160,
"response\",": 161,
"\"Reporting\"],": 162,
"[\"Invoicing\",": 163,
"[\"read_financials\"]}": 164,
"tracking\",": 165,
"\"Closing\"],": 166,
"\"Demo": 167,
"\"update_deals\"]}": 168,
"[\"Lead": 169,
"[\"read_leads\",": 170,
"qualification\",": 171,
"scheduling\",": 172,
"\"Responsive": 173,
"[\"React": 174,
"[\"deploy_code\"]}": 175,
"design\"],": 176,
"frontend": 177,
"implementation\",": 178,
"\"Interview\",": 179,
"\"Offer\",": 180,
"\"Onboard\"]}": 181,
"\"Screening": 182,
"Build": 183,
"[\"Review": 184,
"call\",": 185,
"hiring": 186,
"process": 187,
"resume\",": 188,
"workflow": 189,
"{\"workflow\":": 190,
"\"Build": 191,
"\"Deploy": 192,
"\"Run": 193,
"[\"Code": 194,
"package\",": 195,
"production\"]}": 196,
"release": 197,
"review\",": 198,
"software": 199,
"staging\",": 200,
"tests\",": 201,
"to": 202,
"\"Assign": 203,
"\"Schedule": 204,
"\"Send": 205,
"[\"Create": 206,
"account\",": 207,
"agent\",": 208,
"email\",": 209,
"meeting\"]}": 210,
"onboarding": 211,
"welcome": 212,
"\"Analyze": 213,
"\"Close": 214,
"\"Resolve": 215,
"[\"Receive": 216,
"issue\",": 217,
"ticket\",": 218,
"ticket\"]}": 219,
"\"Negotiate\",": 220,
"[\"Qualify": 221,
"cycle": 222,
"deal\"]}": 223,
"demo\",": 224,
"lead\",": 225,
"proposal\",": 226,
"\"Finance\":": 227,
"\"HR\":": 228,
"Open": 229,
"[\"Allocate": 230,
"[\"Find": 231,
"[\"Hire": 232,
"budget\"]}": 233,
"new": 234,
"office": 235,
"space\"],": 236,
"staff\"],": 237,
"{\"Operations\":": 238,
"\"CMO\":": 239,
"\"CTO\":": 240,
"Launch": 241,
"[\"Define": 242,
"[\"Estimate": 243,
"[\"Prepare": 244,
"campaign\"]}": 245,
"infrastructure\"],": 246,
"launch": 247,
"objectives\"],": 248,
"{\"CEO\":": 249,
"\"Analytics\":": 250,
"Q4": 251,
"Team\":": 252,
"[\"Plan": 253,
"[\"Track": 254,
"ads\"],": 255,
"campaign": 256,
"content\",": 257,
"performance\"]}": 258,
"strategy\"],": 259,
"{\"CMO\":": 260,
"\"Product\":": 261,
"\"Support\":": 262,
"Improve": 263,
"Success\":": 264,
"[\"Add": 265,
"[\"Analyze": 266,
"[\"Improve": 267,
"churn\"],": 268,
"features\"]}": 269,
"requested": 270,
"response": 271,
"retention": 272,
"time\"],": 273,
"{\"Customer": 274,
"\"Minor": 275,
"\"reason\":": 276,
"UI": 277,
"Update": 278,
"button": 279,
"change\"}": 280,
"color": 281,
"false,": 282,
"visual": 283,
"{\"requires_approval\":": 284,
"\"Data": 285,
"Delete": 286,
"and": 287,
"compliance": 288,
"data": 289,
"privacy": 290,
"risk\"}": 291,
"true,": 292,
"\"Affects": 293,
"Change": 294,
"experience\"}": 295,
"hours": 296,
"Fire": 297,
"employee": 298,
"implications\"}": 299,
"\"Low-risk": 300,
"Add": 301,
"FAQ": 302,
"content": 303,
"page": 304,
"\"Pricing": 305,
"30%": 306,
"Increase": 307,
"affect": 308,
"by": 309,
"changes": 310,
"customers\"}": 311,
"pricing": 312,
"\"agent.deploy\",": 313,
"\"parameters\":": 314,
"Agent\"}}": 315,
"Deploy": 316,
"{\"tool\":": 317,
"\"Welcome\"}}": 318,
"\"customer@example.com\",": 319,
"\"email.send\",": 320,
"\"subject\":": 321,
"Send": 322,
"email": 323,
"{\"to\":": 324,
"\"Complete": 325,
"\"task.create\",": 326,
"for": 327,
"quarterly": 328,
"report": 329,
"report\"}}": 330,
"task": 331,
"{\"title\":": 332,
"\"active\"}}": 333,
"\"customers\",": 334,
"\"database.query\",": 335,
"\"filter\":": 336,
"Get": 337,
"active": 338,
"customers": 339,
"{\"table\":": 340,
"\"30d\"}}": 341,
"\"analytics.report\",": 342,
"\"conversion\",": 343,
"\"period\":": 344,
"conversion": 345,
"{\"metric\":": 346,
"\"customer_onboarding\"}}": 347,
"\"workflow.create\",": 348
},
"unk_token": "<unk>"
}
}