Add tiny processor fixture for testing

.gitattributes

@@ -1,35 +1,28 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text 
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,158 @@
+{
+  "architectures": [
+    "VisionEncoderDecoderModel"
+  ],
+  "decoder": {
+    "_name_or_path": "",
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "add_cross_attention": true,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": 0.0,
+    "d_model": 1024,
+    "decoder_attention_heads": 16,
+    "decoder_ffn_dim": 4096,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 12,
+    "decoder_start_token_id": 2,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.1,
+    "early_stopping": false,
+    "cross_attention_hidden_size": 768,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "init_std": 0.02,
+    "is_decoder": true,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 512,
+    "min_length": 0,
+    "model_type": "trocr",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 1,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "scale_embedding": false,
+    "sep_token_id": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.12.0.dev0",
+    "use_bfloat16": false,
+    "use_cache": false,
+    "vocab_size": 50265
+  },
+  "encoder": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_probs_dropout_prob": 0.0,
+    "bad_words_ids": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "cross_attention_hidden_size": null,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 384,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-12,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "vit",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 16,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qkv_bias": false,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.12.0.dev0",
+    "use_bfloat16": false
+  },
+  "is_encoder_decoder": true,
+  "model_type": "vision-encoder-decoder",
+  "processor_class": "TrOCRProcessor",
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": null,
+  "vocab_size": 501
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.27.0.dev0",
+  "use_cache": false
+}

preprocessor_config.json

@@ -0,0 +1,17 @@
+{
+  "do_normalize": true,
+  "do_resize": true,
+  "image_processor_type": "ViTImageProcessor",
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "resample": 2,
+  "size": 384
+}

tokenizer.json

@@ -0,0 +1,689 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "special": true,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true
+    },
+    {
+      "id": 1,
+      "special": true,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true
+    },
+    {
+      "id": 2,
+      "special": true,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true
+    },
+    {
+      "id": 3,
+      "special": true,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true
+    },
+    {
+      "id": 500,
+      "special": true,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": false,
+      "normalized": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true
+  },
+  "post_processor": {
+    "type": "RobertaProcessing",
+    "sep": [
+      "</s>",
+      2
+    ],
+    "cls": [
+      "<s>",
+      0
+    ],
+    "trim_offsets": true,
+    "add_prefix_space": false
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true
+  },
+  "model": {
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "vocab": {
+      "<s>": 0,
+      "<pad>": 1,
+      "</s>": 2,
+      "<unk>": 3,
+      ".": 4,
+      "Ġthe": 5,
+      ",": 6,
+      "Ġto": 7,
+      "Ġand": 8,
+      "Ġof": 9,
+      "Ġa": 10,
+      "Ġin": 11,
+      "-": 12,
+      "Ġfor": 13,
+      "Ġthat": 14,
+      "Ġon": 15,
+      "Ġis": 16,
+      "âĢ": 17,
+      "'s": 18,
+      "Ġwith": 19,
+      "ĠThe": 20,
+      "Ġwas": 21,
+      "Ġ\"": 22,
+      "Ġat": 23,
+      "Ġit": 24,
+      "Ġas": 25,
+      "Ġsaid": 26,
+      "Ļ": 27,
+      "Ġbe": 28,
+      "s": 29,
+      "Ġby": 30,
+      "Ġfrom": 31,
+      "Ġare": 32,
+      "Ġhave": 33,
+      "Ġhas": 34,
+      ":": 35,
+      "Ġ(": 36,
+      "Ġhe": 37,
+      "ĠI": 38,
+      "Ġhis": 39,
+      "Ġwill": 40,
+      "Ġan": 41,
+      "Ġthis": 42,
+      ")": 43,
+      "ĠâĢ": 44,
+      "Ġnot": 45,
+      "Ŀ": 46,
+      "Ġyou": 47,
+      "ľ": 48,
+      "Ġtheir": 49,
+      "Ġor": 50,
+      "Ġthey": 51,
+      "Ġwe": 52,
+      "Ġbut": 53,
+      "Ġwho": 54,
+      "Ġmore": 55,
+      "Ġhad": 56,
+      "Ġbeen": 57,
+      "Ġwere": 58,
+      "Ġabout": 59,
+      ",\"": 60,
+      "Ġwhich": 61,
+      "Ġup": 62,
+      "Ġits": 63,
+      "Ġcan": 64,
+      "Ġone": 65,
+      "Ġout": 66,
+      "Ġalso": 67,
+      "Ġ$": 68,
+      "Ġher": 69,
+      "Ġall": 70,
+      "Ġafter": 71,
+      ".\"": 72,
+      "/": 73,
+      "Ġwould": 74,
+      "'t": 75,
+      "Ġyear": 76,
+      "Ġwhen": 77,
+      "Ġfirst": 78,
+      "Ġshe": 79,
+      "Ġtwo": 80,
+      "Ġover": 81,
+      "Ġpeople": 82,
+      "ĠA": 83,
+      "Ġour": 84,
+      "ĠIt": 85,
+      "Ġtime": 86,
+      "Ġthan": 87,
+      "Ġinto": 88,
+      "Ġthere": 89,
+      "t": 90,
+      "ĠHe": 91,
+      "Ġnew": 92,
+      "ĠâĢĶ": 93,
+      "Ġlast": 94,
+      "Ġjust": 95,
+      "ĠIn": 96,
+      "Ġother": 97,
+      "Ġso": 98,
+      "Ġwhat": 99,
+      "I": 100,
+      "Ġlike": 101,
+      "a": 102,
+      "Ġsome": 103,
+      "S": 104,
+      "ë": 105,
+      "Ġthem": 106,
+      "Ġyears": 107,
+      "'": 108,
+      "Ġdo": 109,
+      "Ġyour": 110,
+      "Ġ-": 111,
+      "Ġ1": 112,
+      "\"": 113,
+      "Ġif": 114,
+      "Ġcould": 115,
+      "?": 116,
+      "Ġno": 117,
+      "i": 118,
+      "m": 119,
+      "Ġget": 120,
+      "ĠU": 121,
+      "Ġnow": 122,
+      "Ġhim": 123,
+      "Ġback": 124,
+      "ĠBut": 125,
+      "ĠâĢĵ": 126,
+      "Ġmy": 127,
+      "Ġ'": 128,
+      "Ġonly": 129,
+      "Ġthree": 130,
+      ";": 131,
+      "Ġ2": 132,
+      "The": 133,
+      "1": 134,
+      "Ġpercent": 135,
+      "Ġagainst": 136,
+      "Ġbefore": 137,
+      "Ġcompany": 138,
+      "o": 139,
+      "ĠTrump": 140,
+      "Ġhow": 141,
+      "Ġbecause": 142,
+      "Ġany": 143,
+      "Ġmost": 144,
+      "Ġbeing": 145,
+      "Ġmake": 146,
+      "Ġwhere": 147,
+      "Ġduring": 148,
+      "Ġthrough": 149,
+      "Ġwhile": 150,
+      "000": 151,
+      "ĠThis": 152,
+      "Ġmillion": 153,
+      "ing": 154,
+      "Ġ3": 155,
+      "Ġmade": 156,
+      "Ġwell": 157,
+      "Ġ10": 158,
+      "Ġdown": 159,
+      "Ġoff": 160,
+      "Ġsays": 161,
+      "Ġme": 162,
+      "ĠB": 163,
+      "Ġgoing": 164,
+      "Ġteam": 165,
+      "ĠWe": 166,
+      "Ġthose": 167,
+      "Ġgovernment": 168,
+      "Ġway": 169,
+      "We": 170,
+      "Ġmany": 171,
+      "Ġthen": 172,
+      "Ġwork": 173,
+      "Ġtold": 174,
+      "com": 175,
+      "2": 176,
+      "Ġgame": 177,
+      "ĠAnd": 178,
+      "in": 179,
+      "year": 180,
+      "Ġp": 181,
+      "Ġvery": 182,
+      "Ġday": 183,
+      "Ġhome": 184,
+      "Ġtake": 185,
+      "Ġweek": 186,
+      "Ġsince": 187,
+      "ĠNew": 188,
+      "Ġmay": 189,
+      "Ġeven": 190,
+      "Ġseason": 191,
+      "Ġsee": 192,
+      "Ġ2017": 193,
+      "Ġstate": 194,
+      "Ġ5": 195,
+      "ed": 196,
+      "Ġshould": 197,
+      "Ġaround": 198,
+      "Ġ2018": 199,
+      "Ġsecond": 200,
+      "Ġus": 201,
+      "Ġstill": 202,
+      "Ġmuch": 203,
+      "Ġ4": 204,
+      "Ġgood": 205,
+      "Ġthink": 206,
+      "%": 207,
+      "ĠS": 208,
+      "Ġthese": 209,
+      "Ġmarket": 210,
+      "ĠD": 211,
+      "th": 212,
+      "Ġgo": 213,
+      "'re": 214,
+      "Ġsuch": 215,
+      "Ġknow": 216,
+      "Ġincluding": 217,
+      "Ġdon": 218,
+      "y": 219,
+      "Ġnext": 220,
+      "ĠP": 221,
+      "Ġdid": 222,
+      "Ġunder": 223,
+      "Ġsay": 224,
+      "en": 225,
+      "ĠL": 226,
+      "Ġbetween": 227,
+      "Ġper": 228,
+      "ĠK": 229,
+      "ĠC": 230,
+      "Ġ6": 231,
+      "Ġworld": 232,
+      "Ġpart": 233,
+      "ĠN": 234,
+      "Ġright": 235,
+      "Ġwant": 236,
+      "Ġfour": 237,
+      "),": 238,
+      "Ġhigh": 239,
+      "Ġneed": 240,
+      "re": 241,
+      "e": 242,
+      "It": 243,
+      "Ġhelp": 244,
+      "5": 245,
+      "3": 246,
+      "Ġcountry": 247,
+      "ĠR": 248,
+      "Ġpolice": 249,
+      "A": 250,
+      "Ġlong": 251,
+      "ĠThey": 252,
+      "Ġend": 253,
+      "er": 254,
+      "ĠT": 255,
+      "ĠM": 256,
+      "u": 257,
+      "Ġboth": 258,
+      "Ġhere": 259,
+      "an": 260,
+      "on": 261,
+      "Ġ7": 262,
+      "Ġde": 263,
+      "ĠShe": 264,
+      "Ġbusiness": 265,
+      "Ġreport": 266,
+      "j": 267,
+      "ers": 268,
+      "Ġreally": 269,
+      "ĠPresident": 270,
+      "ar": 271,
+      "ĠG": 272,
+      "ĠFriday": 273,
+      "ĠF": 274,
+      "Ġbest": 275,
+      "Ġsame": 276,
+      "Ġanother": 277,
+      "Ġset": 278,
+      "old": 279,
+      "ĠThat": 280,
+      "as": 281,
+      "n": 282,
+      "Ġcome": 283,
+      "Ġfamily": 284,
+      "Ġpublic": 285,
+      "ĠFor": 286,
+      "ĠAs": 287,
+      "0": 288,
+      "ĠH": 289,
+      "Ġ8": 290,
+      "Ġ20": 291,
+      "Ġfive": 292,
+      "es": 293,
+      "ĠTuesday": 294,
+      "Ġn": 295,
+      "ĠThursday": 296,
+      "Ġquarter": 297,
+      "h": 298,
+      "Ġtop": 299,
+      "Ġgot": 300,
+      "Ġlife": 301,
+      "ĠMonday": 302,
+      "Ġfound": 303,
+      "Ġuse": 304,
+      "ĠW": 305,
+      "4": 306,
+      "ĠWednesday": 307,
+      "Ġown": 308,
+      "Ġaccording": 309,
+      "Ġplay": 310,
+      "Ġshow": 311,
+      "ĠSt": 312,
+      "Ġman": 313,
+      "Ġleft": 314,
+      "ĠUnited": 315,
+      "Ġ12": 316,
+      "Ġplace": 317,
+      "ĠIf": 318,
+      "Ġlot": 319,
+      "Ġformer": 320,
+      "Ġ0": 321,
+      ").": 322,
+      "Ġsupport": 323,
+      "ie": 324,
+      "Ġbillion": 325,
+      "Ġt": 326,
+      "Ġshares": 327,
+      "!": 328,
+      "z": 329,
+      "k": 330,
+      "ĠState": 331,
+      "Ġpoints": 332,
+      "Ġgroup": 333,
+      "Ġschool": 334,
+      "Ġinformation": 335,
+      "Ġ2016": 336,
+      "al": 337,
+      "r": 338,
+      "Ġwin": 339,
+      "Ġnews": 340,
+      "Ġused": 341,
+      "Ġput": 342,
+      "Ġcity": 343,
+      "ĠJ": 344,
+      "ĠThere": 345,
+      "Ġnumber": 346,
+      "C": 347,
+      "'ve": 348,
+      "Ġeach": 349,
+      "Ġtoo": 350,
+      "Ġwon": 351,
+      "ly": 352,
+      "Ġmonth": 353,
+      "is": 354,
+      "Ġadded": 355,
+      "Ġlook": 356,
+      "Ġbetter": 357,
+      "Ġevery": 358,
+      "Ġ&": 359,
+      "Ġdays": 360,
+      "Ġ9": 361,
+      "Ġtook": 362,
+      "Ġnight": 363,
+      "Ġe": 364,
+      "Ġ11": 365,
+      "os": 366,
+      "Ġfew": 367,
+      "or": 368,
+      "ĠNorth": 369,
+      "ĠYou": 370,
+      "Ġthird": 371,
+      "Ġgreat": 372,
+      "Ġcalled": 373,
+      "ĠOn": 374,
+      "Ġpast": 375,
+      "Ġcame": 376,
+      "Ġmonths": 377,
+      "ĠSaturday": 378,
+      "Ġ15": 379,
+      "Ġbig": 380,
+      "ĠE": 381,
+      "ĠUS": 382,
+      "Ġthings": 383,
+      "ĠO": 384,
+      "Ġd": 385,
+      "Ġstart": 386,
+      "B": 387,
+      "Ġstock": 388,
+      "Ġ30": 389,
+      "Ġwomen": 390,
+      "ĠSouth": 391,
+      "ĠMay": 392,
+      "Ġnever": 393,
+      "Ġpresident": 394,
+      "ĠSunday": 395,
+      "Ġwithout": 396,
+      "man": 397,
+      "8": 398,
+      "Ġdidn": 399,
+      "Ġlocal": 400,
+      "6": 401,
+      "Ġsomething": 402,
+      "Ġcase": 403,
+      "ĠAll": 404,
+      "it": 405,
+      "7": 406,
+      "ĠSo": 407,
+      "Ġchildren": 408,
+      "Ġaway": 409,
+      "Ġlittle": 410,
+      "Ġsix": 411,
+      "ĠCity": 412,
+      "ĠCounty": 413,
+      "Ġdata": 414,
+      "at": 415,
+      "Ġalready": 416,
+      "d": 417,
+      "Ġmoney": 418,
+      "Ġearly": 419,
+      "Ġacross": 420,
+      "Ġexpected": 421,
+      "Ġrun": 422,
+      "Ġlater": 423,
+      "am": 424,
+      "Ġprice": 425,
+      "Ġgames": 426,
+      "ĠMr": 427,
+      "b": 428,
+      "Ġmight": 429,
+      "Ġdifferent": 430,
+      "Ġreported": 431,
+      "Ġdeal": 432,
+      "Ġmedia": 433,
+      "Ġgrowth": 434,
+      "Ġcommunity": 435,
+      "ĠChina": 436,
+      "'m": 437,
+      "c": 438,
+      "Ġwent": 439,
+      "ĠNo": 440,
+      "Ġable": 441,
+      "Ġmaking": 442,
+      "Ġarea": 443,
+      "Ġfar": 444,
+      "Ġstatement": 445,
+      "ĠHouse": 446,
+      "Ġworking": 447,
+      "M": 448,
+      "Ġk": 449,
+      "Ġseen": 450,
+      "Ġcompanies": 451,
+      "Ġtoday": 452,
+      "Ġmembers": 453,
+      "Ġuntil": 454,
+      "Ġfull": 455,
+      "Ġagain": 456,
+      "Ġhalf": 457,
+      "Ġshare": 458,
+      "le": 459,
+      "Ġalways": 460,
+      "Ġcourt": 461,
+      "l": 462,
+      "and": 463,
+      "Ġchange": 464,
+      "Ġfind": 465,
+      "9": 466,
+      "Ġsystem": 467,
+      "ĠV": 468,
+      "ĠYork": 469,
+      "ĠAmerican": 470,
+      "Ġhead": 471,
+      "Ġplayers": 472,
+      "Ġdoes": 473,
+      "Ġhealth": 474,
+      "Ġm": 475,
+      "Ġpower": 476,
+      "Ġpoint": 477,
+      "Ġhit": 478,
+      "Ġ.": 479,
+      "Ġ--": 480,
+      "Ġfree": 481,
+      ".,": 482,
+      "Ġlead": 483,
+      "Ġseveral": 484,
+      "Ġrecent": 485,
+      "Ġcall": 486,
+      "N": 487,
+      "Ġlaw": 488,
+      "Ġkeep": 489,
+      "Ġopen": 490,
+      "ĠNews": 491,
+      "Ġgive": 492,
+      "ia": 493,
+      "ĠMarch": 494,
+      "D": 495,
+      "ĠNational": 496,
+      "ĠAt": 497,
+      "Ġtimes": 498,
+      "Ġfuture": 499,
+      "<mask>": 500
+    },
+    "merges": [
+      "i n",
+      "r e",
+      "o n",
+      "e r",
+      "a t",
+      "e n",
+      "i t",
+      "i s",
+      "a n",
+      "o r",
+      "e s",
+      "e d",
+      "Ġa n",
+      "a l",
+      "a r",
+      "Ġt o",
+      "Ġan d",
+      "a s",
+      "l e",
+      "l y",
+      "a m",
+      "' s",
+      "Ġa s",
+      "er s",
+      "Ġa t",
+      "Ġa re",
+      "Ġd e",
+      "an d",
+      "t h",
+      "o s",
+      "Ġd o",
+      "' t",
+      "Ġthe y",
+      "i e",
+      "Ġm e",
+      "ĠS t",
+      ". \"",
+      "Ġon e",
+      "Ġyou r",
+      "i a",
+      "Ġwe re",
+      ", \"",
+      "ĠI n",
+      "Ġm an",
+      "Ġp er",
+      "Ġbe en",
+      "Ġan y",
+      "Ġthe m",
+      "Ġhe r",
+      "Ġthe re",
+      "Ġm y",
+      "ĠI t",
+      "Ġn o",
+      "Ġit s",
+      "ĠH e",
+      "Ġon ly",
+      ") .",
+      "ĠW e",
+      "Ġthe n",
+      "m an",
+      "Ġyear s",
+      "' re",
+      ") ,",
+      "Ġd on",
+      "Ġ1 0",
+      "Ġbe ing",
+      "Ġdo es",
+      "Ġman y",
+      "Ġus ed",
+      "Ġhe re",
+      "Ġgo ing",
+      "I t",
+      "ĠA s",
+      "' m",
+      "Ġ1 2",
+      "ĠF or",
+      "ĠThe y",
+      "Ġ2 0",
+      "Ġto o",
+      "ĠU S",
+      "Ġ1 5",
+      "ĠThe re",
+      "Ġ1 1",
+      "ĠN o",
+      "ĠS o",
+      "Ġdid n",
+      "Ġcall ed",
+      ". ,",
+      "Ġ3 0",
+      "ĠO n",
+      "ĠA t",
+      "Ġnew s",
+      "Ġde al",
+      "Ġwork ing",
+      "ĠM r",
+      "Ġmonth s",
+      "Ġplay ers",
+      "Ġare a",
+      "Ġreport ed",
+      "Ġpoint s",
+      "ĠNew s"
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,64 @@
+{
+  "errors": "replace",
+  "unk_token": {
+    "content": "<unk>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true,
+    "__type": "AddedToken"
+  },
+  "bos_token": {
+    "content": "<s>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true,
+    "__type": "AddedToken"
+  },
+  "eos_token": {
+    "content": "</s>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true,
+    "__type": "AddedToken"
+  },
+  "add_prefix_space": false,
+  "sep_token": {
+    "content": "</s>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true,
+    "__type": "AddedToken"
+  },
+  "cls_token": {
+    "content": "<s>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true,
+    "__type": "AddedToken"
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true,
+    "__type": "AddedToken"
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "single_word": false,
+    "lstrip": true,
+    "rstrip": false,
+    "normalized": true,
+    "__type": "AddedToken"
+  },
+  "model_max_length": 512,
+  "special_tokens_map_file": null,
+  "name_or_path": "roberta-large",
+  "tokenizer_class": "RobertaTokenizer"
+}