spoortishankar commited on
Commit
1bfe727
·
verified ·
1 Parent(s): b96ef11

Create evaluate.py

Browse files
Files changed (1) hide show
  1. evaluate.py +86 -0
evaluate.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ evaluate.py - measure what each component of SentimentDetector actually adds.
3
+
4
+ pip install datasets scikit-learn
5
+ python evaluate.py # hand-built suite + TweetEval irony/sentiment
6
+ python evaluate.py --n 1000 # TweetEval sentiment sample size
7
+
8
+ Thresholds were NOT tuned on the TweetEval test sets. Paste your own measured numbers
9
+ into README.md - never numbers you have not run.
10
+ """
11
+ import argparse
12
+
13
+ import numpy as np
14
+ from sklearn.metrics import accuracy_score, f1_score
15
+
16
+ from detector import SARCASM_THRESHOLD, SentimentDetector, label_from_score
17
+
18
+ CASES = [
19
+ ("Oh great, another Monday morning meeting. Just what I needed 🙄", "negative"),
20
+ ("I absolutely love waiting 3 hours at the DMV.", "negative"),
21
+ ("Thanks a lot for telling me at the last minute 🙃", "negative"),
22
+ ("Yeah right, because that always works /s", "negative"),
23
+ ("Wow, what a brilliant idea. Nobody has ever thought of that.", "negative"),
24
+ ("Great, my laptop died right before the deadline.", "negative"),
25
+ ("Nothing beats debugging at 3 AM, truly living the dream 🙃", "negative"),
26
+ ("Perfect. Just perfect. My flight got cancelled.", "negative"),
27
+ ("Sure, because 'customer service' is exactly what this was.", "negative"),
28
+ ("I can't believe how terrible this update is 😡", "negative"),
29
+ ("Ugh, the wifi is down again.", "negative"),
30
+ ("Just got my acceptance letter!!! 🎉😭", "positive"),
31
+ ("I'm so proud of my little brother 🥰", "positive"),
32
+ ("Best pizza I've had in years 🍕😍", "positive"),
33
+ ("Not bad at all, honestly impressed.", "positive"),
34
+ ("lol I'm dead 💀 that was hilarious", "positive"),
35
+ ("So happy for you!! Congrats 🎊", "positive"),
36
+ ("I actually enjoyed the workshop, learned a lot!", "positive"),
37
+ ("Wow, you finished the whole project in one night? Amazing work!", "positive"),
38
+ ("I hate how much I love this song 😍", "positive"),
39
+ ("The package arrived on Tuesday.", "neutral"),
40
+ ("It's a meeting at 3 PM in room 204.", "neutral"),
41
+ ]
42
+
43
+
44
+ def run_suite(det):
45
+ print(f"\n== Hand-built suite ({len(CASES)} cases; small and written by the author - indicative only) ==")
46
+ configs = {"baseline (text only)": (False, False), "+ emoji": (False, True), "+ emoji + sarcasm": (True, True)}
47
+ y = [c[1] for c in CASES]
48
+ for name, (sarc, emo) in configs.items():
49
+ pred = [label_from_score(det.analyze(t, sarcasm_aware=sarc, emoji_aware=emo).mean) for t, _ in CASES]
50
+ print(f"{name:24s} accuracy = {accuracy_score(y, pred):.2%}")
51
+ if sarc and emo:
52
+ for (t, gold), p in zip(CASES, pred):
53
+ if gold != p:
54
+ print(f" miss: {t!r} gold={gold} pred={p}")
55
+
56
+
57
+ def run_tweeteval(det, n):
58
+ from datasets import load_dataset # hub id: cardiffnlp/tweet_eval
59
+
60
+ print("\n== TweetEval irony (test) - F1 on the irony class ==")
61
+ ds = load_dataset("cardiffnlp/tweet_eval", "irony", split="test")
62
+ texts, y = ds["text"], np.array(ds["label"])
63
+ base = det.irony_probs(texts) >= 0.5
64
+ fused = np.array([max(r.sarcasm for r in det.analyze(t).sentences) >= SARCASM_THRESHOLD for t in texts])
65
+ print(f"irony model alone : F1 = {f1_score(y, base):.3f}")
66
+ print(f"fused sarcasm : F1 = {f1_score(y, fused):.3f}")
67
+
68
+ print(f"\n== TweetEval sentiment (test sample, n={n}) - macro-F1 ==")
69
+ ds = load_dataset("cardiffnlp/tweet_eval", "sentiment", split="test").shuffle(seed=0).select(range(n))
70
+ texts, y = ds["text"], np.array(ds["label"]) # 0 neg, 1 neu, 2 pos
71
+ base = det.sentiment_probs(texts).argmax(1)
72
+ ids = {"negative": 0, "neutral": 1, "mixed": 1, "positive": 2}
73
+ full = np.array([ids[label_from_score(det.analyze(t).mean)] for t in texts])
74
+ print(f"baseline model : macro-F1 = {f1_score(y, base, average='macro'):.3f}")
75
+ print(f"full pipeline : macro-F1 = {f1_score(y, full, average='macro'):.3f}")
76
+
77
+
78
+ if __name__ == "__main__":
79
+ ap = argparse.ArgumentParser()
80
+ ap.add_argument("--n", type=int, default=1000)
81
+ ap.add_argument("--skip-tweeteval", action="store_true")
82
+ a = ap.parse_args()
83
+ det = SentimentDetector()
84
+ run_suite(det)
85
+ if not a.skip_tweeteval:
86
+ run_tweeteval(det, a.n)