{
  "test_sets": {
    "test": "Amazon 2023 balanced test",
    "real_world": "Amazon 2023 real-world mix",
    "sandisk": "Brief dataset 1 (SanDisk reviews)",
    "generic": "Brief dataset 2 (generic sentences)"
  },
  "results": {
    "TextBlob": {
      "test": {
        "accuracy": 0.5127,
        "macro_f1": 0.4827,
        "per_class": {
          "negative": {
            "precision": 0.7624,
            "recall": 0.2247,
            "f1": 0.3471,
            "support": 3000
          },
          "neutral": {
            "precision": 0.4032,
            "recall": 0.4807,
            "f1": 0.4386,
            "support": 3000
          },
          "positive": {
            "precision": 0.5502,
            "recall": 0.8327,
            "f1": 0.6626,
            "support": 3000
          }
        },
        "confusion_matrix": [
          [
            674,
            1653,
            673
          ],
          [
            189,
            1442,
            1369
          ],
          [
            21,
            481,
            2498
          ]
        ],
        "reviews_per_second": 4620.0
      },
      "real_world": {
        "accuracy": 0.731,
        "macro_f1": 0.4827,
        "per_class": {
          "negative": {
            "precision": 0.6979,
            "recall": 0.2339,
            "f1": 0.3503,
            "support": 1146
          },
          "neutral": {
            "precision": 0.1515,
            "recall": 0.4693,
            "f1": 0.229,
            "support": 748
          },
          "positive": {
            "precision": 0.9167,
            "recall": 0.8254,
            "f1": 0.8687,
            "support": 8106
          }
        },
        "confusion_matrix": [
          [
            268,
            624,
            254
          ],
          [
            43,
            351,
            354
          ],
          [
            73,
            1342,
            6691
          ]
        ],
        "reviews_per_second": 4915.8
      },
      "sandisk": {
        "accuracy": 0.7558,
        "macro_f1": 0.4168,
        "per_class": {
          "negative": {
            "precision": 0.4472,
            "recall": 0.1698,
            "f1": 0.2461,
            "support": 324
          },
          "neutral": {
            "precision": 0.0713,
            "recall": 0.5423,
            "f1": 0.126,
            "support": 142
          },
          "positive": {
            "precision": 0.9659,
            "recall": 0.8055,
            "f1": 0.8784,
            "support": 4436
          }
        },
        "confusion_matrix": [
          [
            55,
            198,
            71
          ],
          [
            10,
            77,
            55
          ],
          [
            58,
            805,
            3573
          ]
        ],
        "reviews_per_second": 5226.5
      },
      "generic": {
        "accuracy": 0.8646,
        "macro_f1": 0.9271,
        "per_class": {
          "negative": {
            "precision": 1.0,
            "recall": 0.8605,
            "f1": 0.925,
            "support": 43
          },
          "positive": {
            "precision": 1.0,
            "recall": 0.8679,
            "f1": 0.9293,
            "support": 53
          }
        },
        "confusion_matrix": [
          [
            37,
            6,
            0
          ],
          [
            0,
            0,
            0
          ],
          [
            0,
            7,
            46
          ]
        ],
        "reviews_per_second": 8092.2
      }
    },
    "VADER": {
      "test": {
        "accuracy": 0.5304,
        "macro_f1": 0.4969,
        "per_class": {
          "negative": {
            "precision": 0.7036,
            "recall": 0.4193,
            "f1": 0.5255,
            "support": 3000
          },
          "neutral": {
            "precision": 0.4189,
            "recall": 0.2427,
            "f1": 0.3073,
            "support": 3000
          },
          "positive": {
            "precision": 0.5093,
            "recall": 0.9293,
            "f1": 0.658,
            "support": 3000
          }
        },
        "confusion_matrix": [
          [
            1258,
            847,
            895
          ],
          [
            481,
            728,
            1791
          ],
          [
            49,
            163,
            2788
          ]
        ],
        "reviews_per_second": 4304.8
      },
      "real_world": {
        "accuracy": 0.8112,
        "macro_f1": 0.529,
        "per_class": {
          "negative": {
            "precision": 0.6432,
            "recall": 0.3901,
            "f1": 0.4856,
            "support": 1146
          },
          "neutral": {
            "precision": 0.163,
            "recall": 0.2193,
            "f1": 0.187,
            "support": 748
          },
          "positive": {
            "precision": 0.9038,
            "recall": 0.9254,
            "f1": 0.9145,
            "support": 8106
          }
        },
        "confusion_matrix": [
          [
            447,
            362,
            337
          ],
          [
            123,
            164,
            461
          ],
          [
            125,
            480,
            7501
          ]
        ],
        "reviews_per_second": 4370.6
      },
      "sandisk": {
        "accuracy": 0.8135,
        "macro_f1": 0.4966,
        "per_class": {
          "negative": {
            "precision": 0.4735,
            "recall": 0.5525,
            "f1": 0.51,
            "support": 324
          },
          "neutral": {
            "precision": 0.0482,
            "recall": 0.1972,
            "f1": 0.0775,
            "support": 142
          },
          "positive": {
            "precision": 0.9589,
            "recall": 0.8523,
            "f1": 0.9025,
            "support": 4436
          }
        },
        "confusion_matrix": [
          [
            179,
            56,
            89
          ],
          [
            41,
            28,
            73
          ],
          [
            158,
            497,
            3781
          ]
        ],
        "reviews_per_second": 6493.0
      },
      "generic": {
        "accuracy": 0.8125,
        "macro_f1": 0.886,
        "per_class": {
          "negative": {
            "precision": 1.0,
            "recall": 0.7907,
            "f1": 0.8831,
            "support": 43
          },
          "positive": {
            "precision": 0.9565,
            "recall": 0.8302,
            "f1": 0.8889,
            "support": 53
          }
        },
        "confusion_matrix": [
          [
            34,
            7,
            2
          ],
          [
            0,
            0,
            0
          ],
          [
            0,
            9,
            44
          ]
        ],
        "reviews_per_second": 34099.4
      }
    },
    "TF-IDF + Naive Bayes": {
      "test": {
        "accuracy": 0.7313,
        "macro_f1": 0.7343,
        "per_class": {
          "negative": {
            "precision": 0.7358,
            "recall": 0.7147,
            "f1": 0.7251,
            "support": 3000
          },
          "neutral": {
            "precision": 0.6132,
            "recall": 0.6817,
            "f1": 0.6456,
            "support": 3000
          },
          "positive": {
            "precision": 0.8699,
            "recall": 0.7977,
            "f1": 0.8322,
            "support": 3000
          }
        },
        "confusion_matrix": [
          [
            2144,
            795,
            61
          ],
          [
            658,
            2045,
            297
          ],
          [
            112,
            495,
            2393
          ]
        ],
        "reviews_per_second": 17211.8
      },
      "real_world": {
        "accuracy": 0.7775,
        "macro_f1": 0.6381,
        "per_class": {
          "negative": {
            "precision": 0.6421,
            "recall": 0.7216,
            "f1": 0.6795,
            "support": 1146
          },
          "neutral": {
            "precision": 0.2382,
            "recall": 0.7019,
            "f1": 0.3557,
            "support": 748
          },
          "positive": {
            "precision": 0.9869,
            "recall": 0.7924,
            "f1": 0.879,
            "support": 8106
          }
        },
        "confusion_matrix": [
          [
            827,
            298,
            21
          ],
          [
            159,
            525,
            64
          ],
          [
            302,
            1381,
            6423
          ]
        ],
        "reviews_per_second": 18088.0
      },
      "sandisk": {
        "accuracy": 0.8082,
        "macro_f1": 0.5115,
        "per_class": {
          "negative": {
            "precision": 0.3302,
            "recall": 0.9753,
            "f1": 0.4934,
            "support": 324
          },
          "neutral": {
            "precision": 0.1061,
            "recall": 0.2324,
            "f1": 0.1457,
            "support": 142
          },
          "positive": {
            "precision": 0.9942,
            "recall": 0.8145,
            "f1": 0.8954,
            "support": 4436
          }
        },
        "confusion_matrix": [
          [
            316,
            5,
            3
          ],
          [
            91,
            33,
            18
          ],
          [
            550,
            273,
            3613
          ]
        ],
        "reviews_per_second": 20414.6
      },
      "generic": {
        "accuracy": 0.8333,
        "macro_f1": 0.8722,
        "per_class": {
          "negative": {
            "precision": 0.8372,
            "recall": 0.8372,
            "f1": 0.8372,
            "support": 43
          },
          "positive": {
            "precision": 1.0,
            "recall": 0.8302,
            "f1": 0.9072,
            "support": 53
          }
        },
        "confusion_matrix": [
          [
            36,
            7,
            0
          ],
          [
            0,
            0,
            0
          ],
          [
            7,
            2,
            44
          ]
        ],
        "reviews_per_second": 26705.2
      }
    },
    "TF-IDF + Logistic Regression": {
      "test": {
        "accuracy": 0.7552,
        "macro_f1": 0.7553,
        "per_class": {
          "negative": {
            "precision": 0.7518,
            "recall": 0.7633,
            "f1": 0.7575,
            "support": 3000
          },
          "neutral": {
            "precision": 0.6563,
            "recall": 0.653,
            "f1": 0.6546,
            "support": 3000
          },
          "positive": {
            "precision": 0.8582,
            "recall": 0.8493,
            "f1": 0.8537,
            "support": 3000
          }
        },
        "confusion_matrix": [
          [
            2290,
            647,
            63
          ],
          [
            683,
            1959,
            358
          ],
          [
            73,
            379,
            2548
          ]
        ],
        "reviews_per_second": 17404.1
      },
      "real_world": {
        "accuracy": 0.8213,
        "macro_f1": 0.6736,
        "per_class": {
          "negative": {
            "precision": 0.693,
            "recall": 0.7504,
            "f1": 0.7206,
            "support": 1146
          },
          "neutral": {
            "precision": 0.2775,
            "recall": 0.6604,
            "f1": 0.3908,
            "support": 748
          },
          "positive": {
            "precision": 0.9828,
            "recall": 0.8462,
            "f1": 0.9094,
            "support": 8106
          }
        },
        "confusion_matrix": [
          [
            860,
            254,
            32
          ],
          [
            166,
            494,
            88
          ],
          [
            215,
            1032,
            6859
          ]
        ],
        "reviews_per_second": 17583.7
      },
      "sandisk": {
        "accuracy": 0.8623,
        "macro_f1": 0.5883,
        "per_class": {
          "negative": {
            "precision": 0.4991,
            "recall": 0.8673,
            "f1": 0.6336,
            "support": 324
          },
          "neutral": {
            "precision": 0.1346,
            "recall": 0.3944,
            "f1": 0.2007,
            "support": 142
          },
          "positive": {
            "precision": 0.9916,
            "recall": 0.8769,
            "f1": 0.9307,
            "support": 4436
          }
        },
        "confusion_matrix": [
          [
            281,
            34,
            9
          ],
          [
            62,
            56,
            24
          ],
          [
            220,
            326,
            3890
          ]
        ],
        "reviews_per_second": 19402.3
      },
      "generic": {
        "accuracy": 0.8229,
        "macro_f1": 0.8487,
        "per_class": {
          "negative": {
            "precision": 0.7708,
            "recall": 0.8605,
            "f1": 0.8132,
            "support": 43
          },
          "positive": {
            "precision": 1.0,
            "recall": 0.7925,
            "f1": 0.8842,
            "support": 53
          }
        },
        "confusion_matrix": [
          [
            37,
            6,
            0
          ],
          [
            0,
            0,
            0
          ],
          [
            11,
            0,
            42
          ]
        ],
        "reviews_per_second": 57402.5
      }
    },
    "RoBERTa (no fine-tuning)": {
      "test": {
        "accuracy": 0.6593,
        "macro_f1": 0.6093,
        "per_class": {
          "negative": {
            "precision": 0.63,
            "recall": 0.8497,
            "f1": 0.7235,
            "support": 3000
          },
          "neutral": {
            "precision": 0.5955,
            "recall": 0.2037,
            "f1": 0.3035,
            "support": 3000
          },
          "positive": {
            "precision": 0.7062,
            "recall": 0.9247,
            "f1": 0.8008,
            "support": 3000
          }
        },
        "confusion_matrix": [
          [
            2549,
            269,
            182
          ],
          [
            1417,
            611,
            972
          ],
          [
            80,
            146,
            2774
          ]
        ],
        "reviews_per_second": 172.1
      },
      "real_world": {
        "accuracy": 0.8594,
        "macro_f1": 0.6182,
        "per_class": {
          "negative": {
            "precision": 0.6201,
            "recall": 0.849,
            "f1": 0.7168,
            "support": 1146
          },
          "neutral": {
            "precision": 0.217,
            "recall": 0.1805,
            "f1": 0.1971,
            "support": 748
          },
          "positive": {
            "precision": 0.9586,
            "recall": 0.9235,
            "f1": 0.9407,
            "support": 8106
          }
        },
        "confusion_matrix": [
          [
            973,
            90,
            83
          ],
          [
            373,
            135,
            240
          ],
          [
            223,
            397,
            7486
          ]
        ],
        "reviews_per_second": 172.7
      },
      "sandisk": {
        "accuracy": 0.9004,
        "macro_f1": 0.5957,
        "per_class": {
          "negative": {
            "precision": 0.6368,
            "recall": 0.8549,
            "f1": 0.7299,
            "support": 324
          },
          "neutral": {
            "precision": 0.0756,
            "recall": 0.1549,
            "f1": 0.1016,
            "support": 142
          },
          "positive": {
            "precision": 0.9854,
            "recall": 0.9276,
            "f1": 0.9556,
            "support": 4436
          }
        },
        "confusion_matrix": [
          [
            277,
            29,
            18
          ],
          [
            77,
            22,
            43
          ],
          [
            81,
            240,
            4115
          ]
        ],
        "reviews_per_second": 181.4
      },
      "generic": {
        "accuracy": 1.0,
        "macro_f1": 1.0,
        "per_class": {
          "negative": {
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0,
            "support": 43
          },
          "positive": {
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0,
            "support": 53
          }
        },
        "confusion_matrix": [
          [
            43,
            0,
            0
          ],
          [
            0,
            0,
            0
          ],
          [
            0,
            0,
            53
          ]
        ],
        "reviews_per_second": 1751.4
      }
    },
    "Fine-tuned DistilRoBERTa": {
      "test": {
        "accuracy": 0.7934,
        "macro_f1": 0.7932,
        "per_class": {
          "negative": {
            "precision": 0.781,
            "recall": 0.8,
            "f1": 0.7904,
            "support": 3000
          },
          "neutral": {
            "precision": 0.7014,
            "recall": 0.6867,
            "f1": 0.694,
            "support": 3000
          },
          "positive": {
            "precision": 0.8967,
            "recall": 0.8937,
            "f1": 0.8952,
            "support": 3000
          }
        },
        "confusion_matrix": [
          [
            2400,
            579,
            21
          ],
          [
            652,
            2060,
            288
          ],
          [
            21,
            298,
            2681
          ]
        ],
        "reviews_per_second": 327.0
      },
      "real_world": {
        "accuracy": 0.869,
        "macro_f1": 0.7315,
        "per_class": {
          "negative": {
            "precision": 0.7925,
            "recall": 0.7967,
            "f1": 0.7946,
            "support": 1146
          },
          "neutral": {
            "precision": 0.3439,
            "recall": 0.6939,
            "f1": 0.4599,
            "support": 748
          },
          "positive": {
            "precision": 0.989,
            "recall": 0.8954,
            "f1": 0.9399,
            "support": 8106
          }
        },
        "confusion_matrix": [
          [
            913,
            217,
            16
          ],
          [
            164,
            519,
            65
          ],
          [
            75,
            773,
            7258
          ]
        ],
        "reviews_per_second": 333.1
      },
      "sandisk": {
        "accuracy": 0.92,
        "macro_f1": 0.687,
        "per_class": {
          "negative": {
            "precision": 0.7345,
            "recall": 0.9136,
            "f1": 0.8143,
            "support": 324
          },
          "neutral": {
            "precision": 0.2057,
            "recall": 0.4577,
            "f1": 0.2838,
            "support": 142
          },
          "positive": {
            "precision": 0.9919,
            "recall": 0.9353,
            "f1": 0.9628,
            "support": 4436
          }
        },
        "confusion_matrix": [
          [
            296,
            23,
            5
          ],
          [
            48,
            65,
            29
          ],
          [
            59,
            228,
            4149
          ]
        ],
        "reviews_per_second": 352.7
      },
      "generic": {
        "accuracy": 0.9688,
        "macro_f1": 0.9819,
        "per_class": {
          "negative": {
            "precision": 1.0,
            "recall": 0.9302,
            "f1": 0.9639,
            "support": 43
          },
          "positive": {
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0,
            "support": 53
          }
        },
        "confusion_matrix": [
          [
            40,
            3,
            0
          ],
          [
            0,
            0,
            0
          ],
          [
            0,
            0,
            53
          ]
        ],
        "reviews_per_second": 3139.3
      }
    },
    "Fine-tuned RoBERTa": {
      "test": {
        "accuracy": 0.8056,
        "macro_f1": 0.8054,
        "per_class": {
          "negative": {
            "precision": 0.7905,
            "recall": 0.8123,
            "f1": 0.8012,
            "support": 3000
          },
          "neutral": {
            "precision": 0.7186,
            "recall": 0.704,
            "f1": 0.7112,
            "support": 3000
          },
          "positive": {
            "precision": 0.907,
            "recall": 0.9003,
            "f1": 0.9036,
            "support": 3000
          }
        },
        "confusion_matrix": [
          [
            2437,
            544,
            19
          ],
          [
            630,
            2112,
            258
          ],
          [
            16,
            283,
            2701
          ]
        ],
        "reviews_per_second": 170.5
      },
      "real_world": {
        "accuracy": 0.8764,
        "macro_f1": 0.7417,
        "per_class": {
          "negative": {
            "precision": 0.818,
            "recall": 0.8002,
            "f1": 0.809,
            "support": 1146
          },
          "neutral": {
            "precision": 0.3555,
            "recall": 0.7019,
            "f1": 0.4719,
            "support": 748
          },
          "positive": {
            "precision": 0.9892,
            "recall": 0.9033,
            "f1": 0.9443,
            "support": 8106
          }
        },
        "confusion_matrix": [
          [
            917,
            216,
            13
          ],
          [
            156,
            525,
            67
          ],
          [
            48,
            736,
            7322
          ]
        ],
        "reviews_per_second": 170.2
      },
      "sandisk": {
        "accuracy": 0.9325,
        "macro_f1": 0.7163,
        "per_class": {
          "negative": {
            "precision": 0.7752,
            "recall": 0.9259,
            "f1": 0.8439,
            "support": 324
          },
          "neutral": {
            "precision": 0.2509,
            "recall": 0.507,
            "f1": 0.3357,
            "support": 142
          },
          "positive": {
            "precision": 0.9931,
            "recall": 0.9466,
            "f1": 0.9693,
            "support": 4436
          }
        },
        "confusion_matrix": [
          [
            300,
            21,
            3
          ],
          [
            44,
            72,
            26
          ],
          [
            43,
            194,
            4199
          ]
        ],
        "reviews_per_second": 180.7
      },
      "generic": {
        "accuracy": 0.9062,
        "macro_f1": 0.9416,
        "per_class": {
          "negative": {
            "precision": 1.0,
            "recall": 0.7907,
            "f1": 0.8831,
            "support": 43
          },
          "positive": {
            "precision": 1.0,
            "recall": 1.0,
            "f1": 1.0,
            "support": 53
          }
        },
        "confusion_matrix": [
          [
            34,
            9,
            0
          ],
          [
            0,
            0,
            0
          ],
          [
            0,
            0,
            53
          ]
        ],
        "reviews_per_second": 1772.6
      }
    }
  }
}