jpancorbTaniwa commited on
Commit
5cdcfc1
·
1 Parent(s): d98f1fe

truching tokenizer config

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -2
tokenizer_config.json CHANGED
@@ -46,12 +46,13 @@
46
  "do_basic_tokenize": true,
47
  "do_lower_case": false,
48
  "mask_token": "[MASK]",
49
- "model_max_length": 512,
50
  "never_split": null,
51
  "pad_token": "[PAD]",
52
  "sep_token": "[SEP]",
53
  "strip_accents": false,
54
  "tokenize_chinese_chars": true,
55
  "tokenizer_class": "BertTokenizer",
56
- "unk_token": "[UNK]"
 
57
  }
 
46
  "do_basic_tokenize": true,
47
  "do_lower_case": false,
48
  "mask_token": "[MASK]",
49
+ "model_max_length": 30,
50
  "never_split": null,
51
  "pad_token": "[PAD]",
52
  "sep_token": "[SEP]",
53
  "strip_accents": false,
54
  "tokenize_chinese_chars": true,
55
  "tokenizer_class": "BertTokenizer",
56
+ "unk_token": "[UNK]",
57
+ "name_or_path": "dccuchile/bert-base-spanish-wwm-cased"
58
  }