kwyong commited on
Commit
25abc24
·
verified ·
1 Parent(s): 7a430d2

Robust set test train

Browse files
preprocessor_config.json CHANGED
@@ -20,7 +20,7 @@
20
  "resample": 2,
21
  "rescale_factor": 0.00392156862745098,
22
  "size": [
23
- 1200,
24
- 2400
25
  ]
26
  }
 
20
  "resample": 2,
21
  "rescale_factor": 0.00392156862745098,
22
  "size": [
23
+ 1800,
24
+ 1350
25
  ]
26
  }
tokenizer.json CHANGED
@@ -23,8 +23,8 @@
23
  "single_word": false,
24
  "lstrip": false,
25
  "rstrip": false,
26
- "normalized": false,
27
- "special": true
28
  },
29
  {
30
  "id": 1,
@@ -41,8 +41,8 @@
41
  "single_word": false,
42
  "lstrip": false,
43
  "rstrip": false,
44
- "normalized": false,
45
- "special": true
46
  },
47
  {
48
  "id": 3,
 
23
  "single_word": false,
24
  "lstrip": false,
25
  "rstrip": false,
26
+ "normalized": true,
27
+ "special": false
28
  },
29
  {
30
  "id": 1,
 
41
  "single_word": false,
42
  "lstrip": false,
43
  "rstrip": false,
44
+ "normalized": true,
45
+ "special": false
46
  },
47
  {
48
  "id": 3,
tokenizer_config.json CHANGED
@@ -3,10 +3,10 @@
3
  "0": {
4
  "content": "<s>",
5
  "lstrip": false,
6
- "normalized": false,
7
  "rstrip": false,
8
  "single_word": false,
9
- "special": true
10
  },
11
  "1": {
12
  "content": "<pad>",
@@ -19,10 +19,10 @@
19
  "2": {
20
  "content": "</s>",
21
  "lstrip": false,
22
- "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
- "special": true
26
  },
27
  "3": {
28
  "content": "<unk>",
 
3
  "0": {
4
  "content": "<s>",
5
  "lstrip": false,
6
+ "normalized": true,
7
  "rstrip": false,
8
  "single_word": false,
9
+ "special": false
10
  },
11
  "1": {
12
  "content": "<pad>",
 
19
  "2": {
20
  "content": "</s>",
21
  "lstrip": false,
22
+ "normalized": true,
23
  "rstrip": false,
24
  "single_word": false,
25
+ "special": false
26
  },
27
  "3": {
28
  "content": "<unk>",