Hugging Face's logo

Spaces:
Emirinth
/
ReviewGuard
Sleeping

App Files Community
ReviewGuard / artifacts /reports /model_comparison.json
Emirinth's picture
Emirinth
Deploy ReviewGuard
edfb6c0 verified
Raw History Blame Contribute Delete
24.9 kB
{
"test_sets": {
"test": "Amazon 2023 balanced test",
"real_world": "Amazon 2023 real-world mix",
"sandisk": "Brief dataset 1 (SanDisk reviews)",
"generic": "Brief dataset 2 (generic sentences)"
},
"results": {
"TextBlob": {
"test": {
"accuracy": 0.5127,
"macro_f1": 0.4827,
"per_class": {
"negative": {
"precision": 0.7624,
"recall": 0.2247,
"f1": 0.3471,
"support": 3000
},
"neutral": {
"precision": 0.4032,
"recall": 0.4807,
"f1": 0.4386,
"support": 3000
},
"positive": {
"precision": 0.5502,
"recall": 0.8327,
"f1": 0.6626,
"support": 3000
}
},
"confusion_matrix": [
[
674,
1653,
673
],
[
189,
1442,
1369
],
[
21,
481,
2498
]
],
"reviews_per_second": 4620.0
},
"real_world": {
"accuracy": 0.731,
"macro_f1": 0.4827,
"per_class": {
"negative": {
"precision": 0.6979,
"recall": 0.2339,
"f1": 0.3503,
"support": 1146
},
"neutral": {
"precision": 0.1515,
"recall": 0.4693,
"f1": 0.229,
"support": 748
},
"positive": {
"precision": 0.9167,
"recall": 0.8254,
"f1": 0.8687,
"support": 8106
}
},
"confusion_matrix": [
[
268,
624,
254
],
[
43,
351,
354
],
[
73,
1342,
6691
]
],
"reviews_per_second": 4915.8
},
"sandisk": {
"accuracy": 0.7558,
"macro_f1": 0.4168,
"per_class": {
"negative": {
"precision": 0.4472,
"recall": 0.1698,
"f1": 0.2461,
"support": 324
},
"neutral": {
"precision": 0.0713,
"recall": 0.5423,
"f1": 0.126,
"support": 142
},
"positive": {
"precision": 0.9659,
"recall": 0.8055,
"f1": 0.8784,
"support": 4436
}
},
"confusion_matrix": [
[
55,
198,
71
],
[
10,
77,
55
],
[
58,
805,
3573
]
],
"reviews_per_second": 5226.5
},
"generic": {
"accuracy": 0.8646,
"macro_f1": 0.9271,
"per_class": {
"negative": {
"precision": 1.0,
"recall": 0.8605,
"f1": 0.925,
"support": 43
},
"positive": {
"precision": 1.0,
"recall": 0.8679,
"f1": 0.9293,
"support": 53
}
},
"confusion_matrix": [
[
37,
6,
0
],
[
0,
0,
0
],
[
0,
7,
46
]
],
"reviews_per_second": 8092.2
}
},
"VADER": {
"test": {
"accuracy": 0.5304,
"macro_f1": 0.4969,
"per_class": {
"negative": {
"precision": 0.7036,
"recall": 0.4193,
"f1": 0.5255,
"support": 3000
},
"neutral": {
"precision": 0.4189,
"recall": 0.2427,
"f1": 0.3073,
"support": 3000
},
"positive": {
"precision": 0.5093,
"recall": 0.9293,
"f1": 0.658,
"support": 3000
}
},
"confusion_matrix": [
[
1258,
847,
895
],
[
481,
728,
1791
],
[
49,
163,
2788
]
],
"reviews_per_second": 4304.8
},
"real_world": {
"accuracy": 0.8112,
"macro_f1": 0.529,
"per_class": {
"negative": {
"precision": 0.6432,
"recall": 0.3901,
"f1": 0.4856,
"support": 1146
},
"neutral": {
"precision": 0.163,
"recall": 0.2193,
"f1": 0.187,
"support": 748
},
"positive": {
"precision": 0.9038,
"recall": 0.9254,
"f1": 0.9145,
"support": 8106
}
},
"confusion_matrix": [
[
447,
362,
337
],
[
123,
164,
461
],
[
125,
480,
7501
]
],
"reviews_per_second": 4370.6
},
"sandisk": {
"accuracy": 0.8135,
"macro_f1": 0.4966,
"per_class": {
"negative": {
"precision": 0.4735,
"recall": 0.5525,
"f1": 0.51,
"support": 324
},
"neutral": {
"precision": 0.0482,
"recall": 0.1972,
"f1": 0.0775,
"support": 142
},
"positive": {
"precision": 0.9589,
"recall": 0.8523,
"f1": 0.9025,
"support": 4436
}
},
"confusion_matrix": [
[
179,
56,
89
],
[
41,
28,
73
],
[
158,
497,
3781
]
],
"reviews_per_second": 6493.0
},
"generic": {
"accuracy": 0.8125,
"macro_f1": 0.886,
"per_class": {
"negative": {
"precision": 1.0,
"recall": 0.7907,
"f1": 0.8831,
"support": 43
},
"positive": {
"precision": 0.9565,
"recall": 0.8302,
"f1": 0.8889,
"support": 53
}
},
"confusion_matrix": [
[
34,
7,
2
],
[
0,
0,
0
],
[
0,
9,
44
]
],
"reviews_per_second": 34099.4
}
},
"TF-IDF + Naive Bayes": {
"test": {
"accuracy": 0.7313,
"macro_f1": 0.7343,
"per_class": {
"negative": {
"precision": 0.7358,
"recall": 0.7147,
"f1": 0.7251,
"support": 3000
},
"neutral": {
"precision": 0.6132,
"recall": 0.6817,
"f1": 0.6456,
"support": 3000
},
"positive": {
"precision": 0.8699,
"recall": 0.7977,
"f1": 0.8322,
"support": 3000
}
},
"confusion_matrix": [
[
2144,
795,
61
],
[
658,
2045,
297
],
[
112,
495,
2393
]
],
"reviews_per_second": 17211.8
},
"real_world": {
"accuracy": 0.7775,
"macro_f1": 0.6381,
"per_class": {
"negative": {
"precision": 0.6421,
"recall": 0.7216,
"f1": 0.6795,
"support": 1146
},
"neutral": {
"precision": 0.2382,
"recall": 0.7019,
"f1": 0.3557,
"support": 748
},
"positive": {
"precision": 0.9869,
"recall": 0.7924,
"f1": 0.879,
"support": 8106
}
},
"confusion_matrix": [
[
827,
298,
21
],
[
159,
525,
64
],
[
302,
1381,
6423
]
],
"reviews_per_second": 18088.0
},
"sandisk": {
"accuracy": 0.8082,
"macro_f1": 0.5115,
"per_class": {
"negative": {
"precision": 0.3302,
"recall": 0.9753,
"f1": 0.4934,
"support": 324
},
"neutral": {
"precision": 0.1061,
"recall": 0.2324,
"f1": 0.1457,
"support": 142
},
"positive": {
"precision": 0.9942,
"recall": 0.8145,
"f1": 0.8954,
"support": 4436
}
},
"confusion_matrix": [
[
316,
5,
3
],
[
91,
33,
18
],
[
550,
273,
3613
]
],
"reviews_per_second": 20414.6
},
"generic": {
"accuracy": 0.8333,
"macro_f1": 0.8722,
"per_class": {
"negative": {
"precision": 0.8372,
"recall": 0.8372,
"f1": 0.8372,
"support": 43
},
"positive": {
"precision": 1.0,
"recall": 0.8302,
"f1": 0.9072,
"support": 53
}
},
"confusion_matrix": [
[
36,
7,
0
],
[
0,
0,
0
],
[
7,
2,
44
]
],
"reviews_per_second": 26705.2
}
},
"TF-IDF + Logistic Regression": {
"test": {
"accuracy": 0.7552,
"macro_f1": 0.7553,
"per_class": {
"negative": {
"precision": 0.7518,
"recall": 0.7633,
"f1": 0.7575,
"support": 3000
},
"neutral": {
"precision": 0.6563,
"recall": 0.653,
"f1": 0.6546,
"support": 3000
},
"positive": {
"precision": 0.8582,
"recall": 0.8493,
"f1": 0.8537,
"support": 3000
}
},
"confusion_matrix": [
[
2290,
647,
63
],
[
683,
1959,
358
],
[
73,
379,
2548
]
],
"reviews_per_second": 17404.1
},
"real_world": {
"accuracy": 0.8213,
"macro_f1": 0.6736,
"per_class": {
"negative": {
"precision": 0.693,
"recall": 0.7504,
"f1": 0.7206,
"support": 1146
},
"neutral": {
"precision": 0.2775,
"recall": 0.6604,
"f1": 0.3908,
"support": 748
},
"positive": {
"precision": 0.9828,
"recall": 0.8462,
"f1": 0.9094,
"support": 8106
}
},
"confusion_matrix": [
[
860,
254,
32
],
[
166,
494,
88
],
[
215,
1032,
6859
]
],
"reviews_per_second": 17583.7
},
"sandisk": {
"accuracy": 0.8623,
"macro_f1": 0.5883,
"per_class": {
"negative": {
"precision": 0.4991,
"recall": 0.8673,
"f1": 0.6336,
"support": 324
},
"neutral": {
"precision": 0.1346,
"recall": 0.3944,
"f1": 0.2007,
"support": 142
},
"positive": {
"precision": 0.9916,
"recall": 0.8769,
"f1": 0.9307,
"support": 4436
}
},
"confusion_matrix": [
[
281,
34,
9
],
[
62,
56,
24
],
[
220,
326,
3890
]
],
"reviews_per_second": 19402.3
},
"generic": {
"accuracy": 0.8229,
"macro_f1": 0.8487,
"per_class": {
"negative": {
"precision": 0.7708,
"recall": 0.8605,
"f1": 0.8132,
"support": 43
},
"positive": {
"precision": 1.0,
"recall": 0.7925,
"f1": 0.8842,
"support": 53
}
},
"confusion_matrix": [
[
37,
6,
0
],
[
0,
0,
0
],
[
11,
0,
42
]
],
"reviews_per_second": 57402.5
}
},
"RoBERTa (no fine-tuning)": {
"test": {
"accuracy": 0.6593,
"macro_f1": 0.6093,
"per_class": {
"negative": {
"precision": 0.63,
"recall": 0.8497,
"f1": 0.7235,
"support": 3000
},
"neutral": {
"precision": 0.5955,
"recall": 0.2037,
"f1": 0.3035,
"support": 3000
},
"positive": {
"precision": 0.7062,
"recall": 0.9247,
"f1": 0.8008,
"support": 3000
}
},
"confusion_matrix": [
[
2549,
269,
182
],
[
1417,
611,
972
],
[
80,
146,
2774
]
],
"reviews_per_second": 172.1
},
"real_world": {
"accuracy": 0.8594,
"macro_f1": 0.6182,
"per_class": {
"negative": {
"precision": 0.6201,
"recall": 0.849,
"f1": 0.7168,
"support": 1146
},
"neutral": {
"precision": 0.217,
"recall": 0.1805,
"f1": 0.1971,
"support": 748
},
"positive": {
"precision": 0.9586,
"recall": 0.9235,
"f1": 0.9407,
"support": 8106
}
},
"confusion_matrix": [
[
973,
90,
83
],
[
373,
135,
240
],
[
223,
397,
7486
]
],
"reviews_per_second": 172.7
},
"sandisk": {
"accuracy": 0.9004,
"macro_f1": 0.5957,
"per_class": {
"negative": {
"precision": 0.6368,
"recall": 0.8549,
"f1": 0.7299,
"support": 324
},
"neutral": {
"precision": 0.0756,
"recall": 0.1549,
"f1": 0.1016,
"support": 142
},
"positive": {
"precision": 0.9854,
"recall": 0.9276,
"f1": 0.9556,
"support": 4436
}
},
"confusion_matrix": [
[
277,
29,
18
],
[
77,
22,
43
],
[
81,
240,
4115
]
],
"reviews_per_second": 181.4
},
"generic": {
"accuracy": 1.0,
"macro_f1": 1.0,
"per_class": {
"negative": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 43
},
"positive": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 53
}
},
"confusion_matrix": [
[
43,
0,
0
],
[
0,
0,
0
],
[
0,
0,
53
]
],
"reviews_per_second": 1751.4
}
},
"Fine-tuned DistilRoBERTa": {
"test": {
"accuracy": 0.7934,
"macro_f1": 0.7932,
"per_class": {
"negative": {
"precision": 0.781,
"recall": 0.8,
"f1": 0.7904,
"support": 3000
},
"neutral": {
"precision": 0.7014,
"recall": 0.6867,
"f1": 0.694,
"support": 3000
},
"positive": {
"precision": 0.8967,
"recall": 0.8937,
"f1": 0.8952,
"support": 3000
}
},
"confusion_matrix": [
[
2400,
579,
21
],
[
652,
2060,
288
],
[
21,
298,
2681
]
],
"reviews_per_second": 327.0
},
"real_world": {
"accuracy": 0.869,
"macro_f1": 0.7315,
"per_class": {
"negative": {
"precision": 0.7925,
"recall": 0.7967,
"f1": 0.7946,
"support": 1146
},
"neutral": {
"precision": 0.3439,
"recall": 0.6939,
"f1": 0.4599,
"support": 748
},
"positive": {
"precision": 0.989,
"recall": 0.8954,
"f1": 0.9399,
"support": 8106
}
},
"confusion_matrix": [
[
913,
217,
16
],
[
164,
519,
65
],
[
75,
773,
7258
]
],
"reviews_per_second": 333.1
},
"sandisk": {
"accuracy": 0.92,
"macro_f1": 0.687,
"per_class": {
"negative": {
"precision": 0.7345,
"recall": 0.9136,
"f1": 0.8143,
"support": 324
},
"neutral": {
"precision": 0.2057,
"recall": 0.4577,
"f1": 0.2838,
"support": 142
},
"positive": {
"precision": 0.9919,
"recall": 0.9353,
"f1": 0.9628,
"support": 4436
}
},
"confusion_matrix": [
[
296,
23,
5
],
[
48,
65,
29
],
[
59,
228,
4149
]
],
"reviews_per_second": 352.7
},
"generic": {
"accuracy": 0.9688,
"macro_f1": 0.9819,
"per_class": {
"negative": {
"precision": 1.0,
"recall": 0.9302,
"f1": 0.9639,
"support": 43
},
"positive": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 53
}
},
"confusion_matrix": [
[
40,
3,
0
],
[
0,
0,
0
],
[
0,
0,
53
]
],
"reviews_per_second": 3139.3
}
},
"Fine-tuned RoBERTa": {
"test": {
"accuracy": 0.8056,
"macro_f1": 0.8054,
"per_class": {
"negative": {
"precision": 0.7905,
"recall": 0.8123,
"f1": 0.8012,
"support": 3000
},
"neutral": {
"precision": 0.7186,
"recall": 0.704,
"f1": 0.7112,
"support": 3000
},
"positive": {
"precision": 0.907,
"recall": 0.9003,
"f1": 0.9036,
"support": 3000
}
},
"confusion_matrix": [
[
2437,
544,
19
],
[
630,
2112,
258
],
[
16,
283,
2701
]
],
"reviews_per_second": 170.5
},
"real_world": {
"accuracy": 0.8764,
"macro_f1": 0.7417,
"per_class": {
"negative": {
"precision": 0.818,
"recall": 0.8002,
"f1": 0.809,
"support": 1146
},
"neutral": {
"precision": 0.3555,
"recall": 0.7019,
"f1": 0.4719,
"support": 748
},
"positive": {
"precision": 0.9892,
"recall": 0.9033,
"f1": 0.9443,
"support": 8106
}
},
"confusion_matrix": [
[
917,
216,
13
],
[
156,
525,
67
],
[
48,
736,
7322
]
],
"reviews_per_second": 170.2
},
"sandisk": {
"accuracy": 0.9325,
"macro_f1": 0.7163,
"per_class": {
"negative": {
"precision": 0.7752,
"recall": 0.9259,
"f1": 0.8439,
"support": 324
},
"neutral": {
"precision": 0.2509,
"recall": 0.507,
"f1": 0.3357,
"support": 142
},
"positive": {
"precision": 0.9931,
"recall": 0.9466,
"f1": 0.9693,
"support": 4436
}
},
"confusion_matrix": [
[
300,
21,
3
],
[
44,
72,
26
],
[
43,
194,
4199
]
],
"reviews_per_second": 180.7
},
"generic": {
"accuracy": 0.9062,
"macro_f1": 0.9416,
"per_class": {
"negative": {
"precision": 1.0,
"recall": 0.7907,
"f1": 0.8831,
"support": 43
},
"positive": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 53
}
},
"confusion_matrix": [
[
34,
9,
0
],
[
0,
0,
0
],
[
0,
0,
53
]
],
"reviews_per_second": 1772.6
}
}
}
}