Sentence Similarity
sentence-transformers
Safetensors
Transformers
English
atto
feature-extraction
ColBERT
PyLate
custom_code
Instructions to use NeuML/colbert-atto-9k-16d with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- sentence-transformers
How to use NeuML/colbert-atto-9k-16d with sentence-transformers:
from pylate import models queries = [ "Which planet is known as the Red Planet?", "What is the largest planet in our solar system?", ] documents = [ ["Mars is the Red Planet.", "Venus is Earth's twin."], ["Jupiter is the largest planet.", "Saturn has rings."], ] model = models.ColBERT(model_name_or_path="NeuML/colbert-atto-9k-16d") queries_emb = model.encode(queries, is_query=True) docs_emb = model.encode(documents, is_query=False) - Transformers
How to use NeuML/colbert-atto-9k-16d with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("NeuML/colbert-atto-9k-16d", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download tokenizer.json from NeuML/colbert-atto-9k-16d: direct link, hf CLI and curl.
- Browser
- Download file 22.2 kB
-
https://huggingface.co/NeuML/colbert-atto-9k-16d/resolve/main/tokenizer.json
- Command line
-
hf download hf://NeuML/colbert-atto-9k-16d/tokenizer.json
-
curl -L -o tokenizer.json https://huggingface.co/NeuML/colbert-atto-9k-16d/resolve/main/tokenizer.json
22.2 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 399,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 4,
"pad_type_id": 0,
"pad_token": "[MASK]"
},
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1000,
"content": "[Q] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
},
{
"id": 1001,
"content": "[D] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
}
],
"normalizer": {
"type": "BertNormalizer",
"clean_text": true,
"handle_chinese_chars": true,
"strip_accents": null,
"lowercase": true
},
"pre_tokenizer": {
"type": "BertPreTokenizer"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 1
}
}
],
"special_tokens": {
"[CLS]": {
"id": "[CLS]",
"ids": [
2
],
"tokens": [
"[CLS]"
]
},
"[SEP]": {
"id": "[SEP]",
"ids": [
3
],
"tokens": [
"[SEP]"
]
}
}
},
"decoder": {
"type": "WordPiece",
"prefix": "##",
"cleanup": true
},
"model": {
"type": "WordPiece",
"unk_token": "[UNK]",
"continuing_subword_prefix": "##",
"max_input_chars_per_word": 100,
"vocab": {
"[PAD]": 0,
"[UNK]": 1,
"[CLS]": 2,
"[SEP]": 3,
"[MASK]": 4,
"!": 5,
"\"": 6,
"#": 7,
"$": 8,
"%": 9,
"&": 10,
"'": 11,
"(": 12,
")": 13,
"*": 14,
"+": 15,
",": 16,
"-": 17,
".": 18,
"/": 19,
"0": 20,
"1": 21,
"2": 22,
"3": 23,
"4": 24,
"5": 25,
"6": 26,
"7": 27,
"8": 28,
"9": 29,
":": 30,
";": 31,
"<": 32,
"=": 33,
">": 34,
"?": 35,
"@": 36,
"[": 37,
"\\": 38,
"]": 39,
"^": 40,
"_": 41,
"`": 42,
"a": 43,
"b": 44,
"c": 45,
"d": 46,
"e": 47,
"f": 48,
"g": 49,
"h": 50,
"i": 51,
"j": 52,
"k": 53,
"l": 54,
"m": 55,
"n": 56,
"o": 57,
"p": 58,
"q": 59,
"r": 60,
"s": 61,
"t": 62,
"u": 63,
"v": 64,
"w": 65,
"x": 66,
"y": 67,
"z": 68,
"{": 69,
"|": 70,
"}": 71,
"~": 72,
"##0": 73,
"##4": 74,
"##3": 75,
"##2": 76,
"##5": 77,
"##7": 78,
"##a": 79,
"##l": 80,
"##f": 81,
"##s": 82,
"##h": 83,
"##t": 84,
"##e": 85,
"##r": 86,
"##c": 87,
"##k": 88,
"##8": 89,
"##i": 90,
"##y": 91,
"##v": 92,
"##x": 93,
"##o": 94,
"##m": 95,
"##n": 96,
"##d": 97,
"##z": 98,
"##w": 99,
"##u": 100,
"##p": 101,
"##6": 102,
"##9": 103,
"##1": 104,
"##g": 105,
"##b": 106,
"##j": 107,
"##q": 108,
"##he": 109,
"##er": 110,
"the": 111,
"##in": 112,
"##on": 113,
"##an": 114,
"##or": 115,
"##ed": 116,
"##es": 117,
"##en": 118,
"##at": 119,
"in": 120,
"##ar": 121,
"##al": 122,
"of": 123,
"an": 124,
"##is": 125,
"##it": 126,
"##ic": 127,
"##as": 128,
"##ing": 129,
"and": 130,
"##ion": 131,
"##ou": 132,
"to": 133,
"##om": 134,
"##re": 135,
"##il": 136,
"##el": 137,
"##ent": 138,
"##ol": 139,
"##am": 140,
"##ec": 141,
"##st": 142,
"##et": 143,
"##ur": 144,
"##ac": 145,
"19": 146,
"st": 147,
"##ad": 148,
"##ro": 149,
"th": 150,
"##le": 151,
"##iv": 152,
"on": 153,
"was": 154,
"##ers": 155,
"for": 156,
"20": 157,
"##ation": 158,
"##ir": 159,
"##ig": 160,
"re": 161,
"##ot": 162,
"as": 163,
"##id": 164,
"##ter": 165,
"##ay": 166,
"al": 167,
"##ly": 168,
"##em": 169,
"is": 170,
"##ch": 171,
"##ist": 172,
"##th": 173,
"fr": 174,
"##im": 175,
"##us": 176,
"##ith": 177,
"##ow": 178,
"##ut": 179,
"ch": 180,
"with": 181,
"##and": 182,
"he": 183,
"##ul": 184,
"by": 185,
"##ag": 186,
"at": 187,
"con": 188,
"##ts": 189,
"be": 190,
"##un": 191,
"##ce": 192,
"##os": 193,
"##um": 194,
"##op": 195,
"##oun": 196,
"com": 197,
"un": 198,
"wh": 199,
"##oc": 200,
"##ain": 201,
"de": 202,
"##her": 203,
"pro": 204,
"##ian": 205,
"##ver": 206,
"from": 207,
"##ct": 208,
"##est": 209,
"##ap": 210,
"##ri": 211,
"se": 212,
"pl": 213,
"##av": 214,
"##ab": 215,
"that": 216,
"##ess": 217,
"##all": 218,
"##ber": 219,
"##ia": 220,
"##ity": 221,
"ar": 222,
"##ies": 223,
"##od": 224,
"ne": 225,
"##ect": 226,
"ex": 227,
"##ate": 228,
"it": 229,
"le": 230,
"or": 231,
"en": 232,
"##if": 233,
"pr": 234,
"##art": 235,
"sp": 236,
"his": 237,
"201": 238,
"##ard": 239,
"su": 240,
"##ill": 241,
"##ud": 242,
"##ak": 243,
"##ip": 244,
"##igh": 245,
"tr": 246,
"##our": 247,
"##ov": 248,
"##ary": 249,
"sh": 250,
"200": 251,
"##ich": 252,
"##ated": 253,
"##ort": 254,
"##ish": 255,
"##se": 256,
"##mer": 257,
"##ment": 258,
"##ive": 259,
"##fer": 260,
"##ge": 261,
"##ld": 262,
"##ear": 263,
"##ial": 264,
"##og": 265,
"##ere": 266,
"##pp": 267,
"##ong": 268,
"##ie": 269,
"18": 270,
"cl": 271,
"##ant": 272,
"are": 273,
"##ell": 274,
"##ue": 275,
"##ub": 276,
"##ks": 277,
"##ast": 278,
"##ine": 279,
"wor": 280,
"comp": 281,
"br": 282,
"##ces": 283,
"##ust": 284,
"##ound": 285,
"##ame": 286,
"af": 287,
"were": 288,
"##ions": 289,
"mar": 290,
"##so": 291,
"gr": 292,
"##gh": 293,
"##ical": 294,
"##ore": 295,
"play": 296,
"##ign": 297,
"##ost": 298,
"new": 299,
"also": 300,
"##end": 301,
"rec": 302,
"ad": 303,
"##ord": 304,
"##pt": 305,
"which": 306,
"ser": 307,
"sc": 308,
"##tern": 309,
"##ack": 310,
"##uc": 311,
"##qu": 312,
"not": 313,
"res": 314,
"fir": 315,
"ac": 316,
"##ther": 317,
"col": 318,
"##ight": 319,
"##age": 320,
"inc": 321,
"##ican": 322,
"cent": 323,
"##per": 324,
"##ure": 325,
"199": 326,
"##ember": 327,
"##ass": 328,
"##der": 329,
"man": 330,
"first": 331,
"##ire": 332,
"her": 333,
"##ous": 334,
"whe": 335,
"##ra": 336,
"202": 337,
"this": 338,
"##man": 339,
"car": 340,
"had": 341,
"##feren": 342,
"##iz": 343,
"part": 344,
"cont": 345,
"ab": 346,
"she": 347,
"##ount": 348,
"comm": 349,
"##ang": 350,
"##ry": 351,
"##ational": 352,
"te": 353,
"ind": 354,
"##hed": 355,
"pe": 356,
"##ough": 357,
"##ans": 358,
"##ations": 359,
"##ric": 360,
"##ond": 361,
"##ory": 362,
"##orn": 363,
"fil": 364,
"year": 365,
"amer": 366,
"##land": 367,
"##ice": 368,
"##mp": 369,
"el": 370,
"##ok": 371,
"int": 372,
"tw": 373,
"##ave": 374,
"rep": 375,
"##ru": 376,
"##ome": 377,
"##amp": 378,
"has": 379,
"go": 380,
"after": 381,
"all": 382,
"rel": 383,
"##ide": 384,
"co": 385,
"##ib": 386,
"##ater": 387,
"##ition": 388,
"##out": 389,
"its": 390,
"per": 391,
"referen": 392,
"ag": 393,
"us": 394,
"one": 395,
"##ey": 396,
"##ach": 397,
"par": 398,
"their": 399,
"pol": 400,
"bo": 401,
"des": 402,
"film": 403,
"##act": 404,
"references": 405,
"ro": 406,
"sch": 407,
"##ace": 408,
"can": 409,
"app": 410,
"198": 411,
"##ens": 412,
"##ates": 413,
"fe": 414,
"##wn": 415,
"who": 416,
"##ime": 417,
"but": 418,
"##ited": 419,
"##ree": 420,
"american": 421,
"dis": 422,
"##ual": 423,
"##ult": 424,
"jo": 425,
"##own": 426,
"##rit": 427,
"##ail": 428,
"##ople": 429,
"##ubl": 430,
"##aw": 431,
"##ath": 432,
"act": 433,
"##ff": 434,
"##ivers": 435,
"##ik": 436,
"##lud": 437,
"ed": 438,
"fin": 439,
"##00": 440,
"off": 441,
"two": 442,
"##ence": 443,
"##ball": 444,
"##ents": 445,
"##ors": 446,
"##gr": 447,
"##les": 448,
"##ild": 449,
"##ool": 450,
"mus": 451,
"ass": 452,
"197": 453,
"dec": 454,
"they": 455,
"##ite": 456,
"##ance": 457,
"includ": 458,
"people": 459,
"bec": 460,
"##ould": 461,
"reg": 462,
"##ug": 463,
"ph": 464,
"other": 465,
"fo": 466,
"##uring": 467,
"we": 468,
"##oll": 469,
"sec": 470,
"##iss": 471,
"##ade": 472,
"##ward": 473,
"bas": 474,
"##ied": 475,
"eng": 476,
"bl": 477,
"lin": 478,
"##ile": 479,
"count": 480,
"pre": 481,
"ev": 482,
"##ury": 483,
"##hip": 484,
"me": 485,
"fl": 486,
"serv": 487,
"gen": 488,
"have": 489,
"up": 490,
"##for": 491,
"##io": 492,
"##te": 493,
"##we": 494,
"##ings": 495,
"univers": 496,
"may": 497,
"sou": 498,
"##hn": 499,
"min": 500,
"##ason": 501,
"bu": 502,
"##old": 503,
"over": 504,
"196": 505,
"hist": 506,
"school": 507,
"out": 508,
"nor": 509,
"##ely": 510,
"been": 511,
"##ics": 512,
"dist": 513,
"pres": 514,
"##eld": 515,
"am": 516,
"war": 517,
"sy": 518,
"mon": 519,
"loc": 520,
"ma": 521,
"##ose": 522,
"##able": 523,
"during": 524,
"publ": 525,
"##ft": 526,
"ear": 527,
"##ished": 528,
"exp": 529,
"kn": 530,
"time": 531,
"##als": 532,
"foot": 533,
"art": 534,
"##ision": 535,
"##gan": 536,
"med": 537,
"##az": 538,
"national": 539,
"work": 540,
"##ton": 541,
"att": 542,
"win": 543,
"##vel": 544,
"##urn": 545,
"##ral": 546,
"cons": 547,
"under": 548,
"char": 549,
"team": 550,
"acc": 551,
"dr": 552,
"##rib": 553,
"university": 554,
"elect": 555,
"prod": 556,
"##ames": 557,
"when": 558,
"##olog": 559,
"world": 560,
"united": 561,
"##ict": 562,
"##ock": 563,
"17": 564,
"##ale": 565,
"bet": 566,
"##ah": 567,
"194": 568,
"there": 569,
"century": 570,
"bro": 571,
"##ists": 572,
"gre": 573,
"##way": 574,
"est": 575,
"gu": 576,
"into": 577,
"##one": 578,
"stud": 579,
"football": 580,
"##int": 581,
"##eas": 582,
"##cc": 583,
"195": 584,
"season": 585,
"##ction": 586,
"##oup": 587,
"no": 588,
"##ick": 589,
"##anc": 590,
"state": 591,
"po": 592,
"born": 593,
"sub": 594,
"##ev": 595,
"##au": 596,
"##ake": 597,
"spec": 598,
"##eral": 599,
"sing": 600,
"players": 601,
"found": 602,
"##ix": 603,
"south": 604,
"record": 605,
"##ious": 606,
"##rap": 607,
"##ily": 608,
"##ons": 609,
"##ife": 610,
"inter": 611,
"##ternal": 612,
"alb": 613,
"nov": 614,
"##oth": 615,
"city": 616,
"trans": 617,
"ret": 618,
"##let": 619,
"states": 620,
"air": 621,
"##ep": 622,
"form": 623,
"pop": 624,
"##ank": 625,
"dep": 626,
"##ince": 627,
"high": 628,
"him": 629,
"##emb": 630,
"##ier": 631,
"will": 632,
"thr": 633,
"more": 634,
"##ollow": 635,
"##oci": 636,
"##ative": 637,
"10": 638,
"so": 639,
"second": 640,
"again": 641,
"##att": 642,
"follow": 643,
"would": 644,
"def": 645,
"##hes": 646,
"later": 647,
"##vern": 648,
"##form": 649,
"prov": 650,
"##ork": 651,
"##ouse": 652,
"bir": 653,
"most": 654,
"fam": 655,
"dire": 656,
"where": 657,
"imp": 658,
"##lish": 659,
"ap": 660,
"##ities": 661,
"external": 662,
"years": 663,
"##ood": 664,
"##ina": 665,
"193": 666,
"##ating": 667,
"inst": 668,
"16": 669,
"links": 670,
"##air": 671,
"##up": 672,
"end": 673,
"##ween": 674,
"##res": 675,
"liv": 676,
"three": 677,
"##hy": 678,
"govern": 679,
"between": 680,
"qu": 681,
"wom": 682,
"group": 683,
"##ular": 684,
"15": 685,
"##ann": 686,
"##ement": 687,
"cap": 688,
"rem": 689,
"only": 690,
"north": 691,
"rev": 692,
"writ": 693,
"met": 694,
"jun": 695,
"##son": 696,
"cr": 697,
"how": 698,
"##enn": 699,
"ter": 700,
"some": 701,
"about": 702,
"##ional": 703,
"music": 704,
"##ague": 705,
"award": 706,
"releas": 707,
"song": 708,
"direct": 709,
"##ograp": 710,
"sa": 711,
"la": 712,
"mov": 713,
"ann": 714,
"inv": 715,
"bar": 716,
"through": 717,
"album": 718,
"sur": 719,
"ge": 720,
"sm": 721,
"##ific": 722,
"compan": 723,
"champ": 724,
"##enc": 725,
"ger": 726,
"op": 727,
"12": 728,
"aust": 729,
"##lo": 730,
"while": 731,
"##iam": 732,
"##ery": 733,
"series": 734,
"##rict": 735,
"day": 736,
"192": 737,
"list": 738,
"##aus": 739,
"##orts": 740,
"id": 741,
"arm": 742,
"set": 743,
"back": 744,
"four": 745,
"care": 746,
"made": 747,
"cre": 748,
"##sp": 749,
"num": 750,
"history": 751,
"offic": 752,
"polit": 753,
"john": 754,
"##ounc": 755,
"jan": 756,
"##ss": 757,
"then": 758,
"county": 759,
"##ally": 760,
"bel": 761,
"them": 762,
"league": 763,
"##ress": 764,
"##itt": 765,
"see": 766,
"##ident": 767,
"such": 768,
"lar": 769,
"intern": 770,
"##ths": 771,
"##fore": 772,
"##erson": 773,
"lead": 774,
"oper": 775,
"##ph": 776,
"house": 777,
"known": 778,
"##uss": 779,
"mod": 780,
"sw": 781,
"supp": 782,
"191": 783,
"##ese": 784,
"##0s": 785,
"##ield": 786,
"brit": 787,
"##ley": 788,
"estab": 789,
"##und": 790,
"##ures": 791,
"men": 792,
"##ten": 793,
"##velop": 794,
"used": 795,
"call": 796,
"ju": 797,
"mat": 798,
"commun": 799,
"women": 800,
"death": 801,
"prof": 802,
"##are": 803,
"##ys": 804,
"add": 805,
"11": 806,
"##oss": 807,
"including": 808,
"desc": 809,
"club": 810,
"div": 811,
"##ket": 812,
"than": 813,
"##ern": 814,
"##ism": 815,
"mil": 816,
"##ather": 817,
"films": 818,
"##iew": 819,
"life": 820,
"main": 821,
"german": 822,
"became": 823,
"well": 824,
"develop": 825,
"tel": 826,
"do": 827,
"##til": 828,
"const": 829,
"##ourn": 830,
"district": 831,
"##ope": 832,
"class": 833,
"build": 834,
"##fl": 835,
"general": 836,
"##ins": 837,
"##ower": 838,
"public": 839,
"rece": 840,
"##erv": 841,
"being": 842,
"##ural": 843,
"##els": 844,
"early": 845,
"##ont": 846,
"sign": 847,
"##bs": 848,
"##its": 849,
"before": 850,
"##ai": 851,
"champions": 852,
"str": 853,
"cor": 854,
"against": 855,
"har": 856,
"21": 857,
"show": 858,
"##ments": 859,
"international": 860,
"aug": 861,
"##ase": 862,
"##ise": 863,
"oct": 864,
"##ection": 865,
"14": 866,
"orig": 867,
"##ert": 868,
"west": 869,
"em": 870,
"both": 871,
"march": 872,
"these": 873,
"town": 874,
"dem": 875,
"family": 876,
"##red": 877,
"law": 878,
"13": 879,
"career": 880,
"sept": 881,
"##uary": 882,
"align": 883,
"##urch": 884,
"coll": 885,
"##amed": 886,
"colle": 887,
"##other": 888,
"##ived": 889,
"leg": 890,
"sl": 891,
"august": 892,
"cro": 893,
"##ister": 894,
"##ike": 895,
"chr": 896,
"##iver": 897,
"##eng": 898,
"im": 899,
"##hen": 900,
"september": 901,
"##ober": 902,
"games": 903,
"many": 904,
"open": 905,
"following": 906,
"##yp": 907,
"long": 908,
"former": 909,
"cour": 910,
"cal": 911,
"##min": 912,
"design": 913,
"##co": 914,
"october": 915,
"##ke": 916,
"##ians": 917,
"austral": 918,
"origin": 919,
"##ution": 920,
"##ected": 921,
"##ained": 922,
"##ning": 923,
"red": 924,
"var": 925,
"progr": 926,
"game": 927,
"british": 928,
"##ole": 929,
"king": 930,
"##ril": 931,
"january": 932,
"##stem": 933,
"organ": 934,
"own": 935,
"produc": 936,
"##ull": 937,
"##ae": 938,
"eur": 939,
"contr": 940,
"since": 941,
"sim": 942,
"until": 943,
"##ever": 944,
"government": 945,
"sever": 946,
"june": 947,
"##lin": 948,
"run": 949,
"##ner": 950,
"july": 951,
"memb": 952,
"##ene": 953,
"land": 954,
"york": 955,
"val": 956,
"miss": 957,
"associ": 958,
"won": 959,
"system": 960,
"company": 961,
"##ized": 962,
"cur": 963,
"##led": 964,
"appear": 965,
"living": 966,
"##ology": 967,
"april": 968,
"star": 969,
"##ty": 970,
"lo": 971,
"english": 972,
"##by": 973,
"november": 974,
"members": 975,
"vill": 976,
"av": 977,
"arch": 978,
"##ind": 979,
"port": 980,
"number": 981,
"##xt": 982,
"based": 983,
"head": 984,
"ep": 985,
"championship": 986,
"december": 987,
"##ica": 988,
"maj": 989,
"child": 990,
"name": 991,
"released": 992,
"educ": 993,
"ext": 994,
"however": 995,
"played": 996,
"bus": 997,
"##ash": 998,
"canad": 999
}
}
}
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 399,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 4,
"pad_type_id": 0,
"pad_token": "[MASK]"
},
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1000,
"content": "[Q] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
},
{
"id": 1001,
"content": "[D] ",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
}
],
"normalizer": {
"type": "BertNormalizer",
"clean_text": true,
"handle_chinese_chars": true,
"strip_accents": null,
"lowercase": true
},
"pre_tokenizer": {
"type": "BertPreTokenizer"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},