PROSTLE commited on
Commit
94da49e
·
unverified ·
1 Parent(s): d759a8a

Update and rename sentiment.py to backend/sentiment.py

Browse files
Files changed (1) hide show
  1. sentiment.py → backend/sentiment.py +184 -14
sentiment.py → backend/sentiment.py RENAMED
@@ -17,11 +17,34 @@ MODEL_NAME = "ProsusAI/finbert"
17
  _tokenizer = BertTokenizer.from_pretrained(MODEL_NAME)
18
  _finbert = BertForSequenceClassification.from_pretrained(MODEL_NAME)
19
  _finbert.eval()
20
- FINBERT_LABELS = ["negative", "neutral", "positive"]
 
21
 
22
  VADER_WEIGHT = 0.4
23
  FINBERT_WEIGHT = 0.6
24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
  def _score_vader(text: str) -> dict:
27
  scores = _vader.polarity_scores(text)
@@ -46,7 +69,10 @@ def _score_finbert(text: str) -> dict:
46
  probs = torch.nn.functional.softmax(logits, dim=-1).squeeze().tolist()
47
  idx = int(np.argmax(probs))
48
 
49
- signed_score = probs[2] - probs[0]
 
 
 
50
  return {
51
  "label": FINBERT_LABELS[idx],
52
  "score": round(signed_score, 4),
@@ -91,9 +117,155 @@ def fetch_news_headlines(ticker: str, max_items: int = 10) -> list[str]:
91
  return headlines
92
 
93
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
  def analyze_sentiment(ticker: str) -> dict:
95
  headlines = fetch_news_headlines(ticker)
96
- if not headlines:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  return {
98
  "ticker": ticker,
99
  "articles_analyzed": 0,
@@ -101,15 +273,13 @@ def analyze_sentiment(ticker: str) -> dict:
101
  "overall_sentiment": "neutral",
102
  "overall_score": 0.0,
103
  "contradictory_dampened": False,
104
- "method": "VADER (40%) + FinBERT (60%) ensemble",
105
  }
106
 
107
- details = [_score_text(h) for h in headlines]
108
- raw_scores = [d["combined_score"] for d in details]
109
  avg = float(np.mean(raw_scores))
110
 
111
- # Dampen one-off outliers: e.g. one strong positive headline amid sell-off
112
- # shouldn't override macro. Use variance and median to detect contradiction.
113
  contradictory_dampened = False
114
  n = len(raw_scores)
115
  if n >= 2:
@@ -119,22 +289,22 @@ def analyze_sentiment(ticker: str) -> dict:
119
  contradictory = variance > 0.15 or abs(avg - median) > 0.35
120
  if contradictory:
121
  contradictory_dampened = True
122
- # Pull toward median (reduces impact of single extreme headline)
123
  avg = 0.4 * avg + 0.6 * median
124
 
125
- if avg > 0.15:
126
  overall = "positive"
127
- elif avg < -0.15:
128
  overall = "negative"
129
  else:
130
  overall = "neutral"
131
 
132
  return {
133
  "ticker": ticker,
134
- "articles_analyzed": len(details),
135
- "details": details,
136
  "overall_sentiment": overall,
137
  "overall_score": round(avg, 4),
138
  "contradictory_dampened": contradictory_dampened,
139
- "method": "VADER (40%) + FinBERT (60%) ensemble",
140
  }
 
17
  _tokenizer = BertTokenizer.from_pretrained(MODEL_NAME)
18
  _finbert = BertForSequenceClassification.from_pretrained(MODEL_NAME)
19
  _finbert.eval()
20
+ # Correct mapping from transformers config: 0=positive, 1=negative, 2=neutral
21
+ FINBERT_LABELS = ["positive", "negative", "neutral"]
22
 
23
  VADER_WEIGHT = 0.4
24
  FINBERT_WEIGHT = 0.6
25
 
26
+ # Extend VADER lexicon with financial context terms
27
+ _vader.lexicon.update({
28
+ 'hike': 1.5,
29
+ 'stake': 1.0,
30
+ 'approve': 2.0,
31
+ 'approved': 2.0,
32
+ 'approval': 2.0,
33
+ 'raise': 1.5,
34
+ 'raised': 1.5,
35
+ 'growth': 2.0,
36
+ 'gain': 2.0,
37
+ 'jump': 1.5,
38
+ 'surge': 1.5,
39
+ 'fall': -1.5,
40
+ 'drop': -1.5,
41
+ 'plunge': -2.0,
42
+ 'cut': -1.5,
43
+ 'slash': -2.0,
44
+ 'warn': -1.5,
45
+ 'alert': -1.0,
46
+ })
47
+
48
 
49
  def _score_vader(text: str) -> dict:
50
  scores = _vader.polarity_scores(text)
 
69
  probs = torch.nn.functional.softmax(logits, dim=-1).squeeze().tolist()
70
  idx = int(np.argmax(probs))
71
 
72
+ # Logits match FINBERT_LABELS = ["positive", "negative", "neutral"]
73
+ # Score = Positive_Prob - Negative_Prob
74
+ signed_score = probs[0] - probs[1]
75
+
76
  return {
77
  "label": FINBERT_LABELS[idx],
78
  "score": round(signed_score, 4),
 
117
  return headlines
118
 
119
 
120
+
121
+ def _is_relevant(headline: str, ticker: str) -> bool:
122
+ """
123
+ Relevance check. yfinance already returns ticker-specific news,
124
+ so we default to True (trust yfinance). We only reject headlines
125
+ that mention a clearly different well-known company.
126
+ """
127
+ headline_lower = headline.lower()
128
+ ticker_base = ticker.split(".")[0].lower() # "RELIANCE" from "RELIANCE.NS"
129
+
130
+ # Quick accept: headline mentions the ticker or company directly
131
+ if ticker_base in headline_lower:
132
+ return True
133
+
134
+ # Common variations map
135
+ aliases = {
136
+ "RELIANCE": ["reliance", "ril", "ambani"],
137
+ "TCS": ["tcs", "tata consultancy"],
138
+ "INFY": ["infy", "infosys"],
139
+ "HDFCBANK": ["hdfc", "housing development"],
140
+ "ICICIBANK": ["icici"],
141
+ "SBIN": ["sbi", "state bank"],
142
+ "LT": ["l&t", "larsen", "toubro"],
143
+ "ITC": ["itc"],
144
+ "BHARTIARTL": ["airtel", "bharti"],
145
+ "KOTAKBANK": ["kotak"],
146
+ "TATAMOTORS": ["tata motors", "jlr"],
147
+ "MARUTI": ["maruti", "suzuki"],
148
+ "WIPRO": ["wipro"],
149
+ "ADANIENT": ["adani ent", "adani group"],
150
+ "TATASTEEL": ["tata steel"],
151
+ "BAJFINANCE": ["bajaj finance"],
152
+ "BAJAJFINSV": ["bajaj finserv"],
153
+ "HINDUNILVR": ["hul", "hindustan unilever"],
154
+ "SUNPHARMA": ["sun pharma"],
155
+ "TITAN": ["titan"],
156
+ "ASIANPAINT": ["asian paints"],
157
+ "ULTRACEMCO": ["ultratech"],
158
+ "NESTLEIND": ["nestle"],
159
+ "POWERGRID": ["power grid", "pgcil"],
160
+ "NTPC": ["ntpc"],
161
+ "M&M": ["mahindra"],
162
+ "JSWSTEEL": ["jsw steel"],
163
+ "GRASIM": ["grasim"],
164
+ "COALINDIA": ["coal india"],
165
+ "ADANIPORTS": ["adani ports"],
166
+ "ADANIGREEN": ["adani green"],
167
+ "TECHM": ["tech mahindra"],
168
+ "BPCL": ["bpcl", "bharat petroleum"],
169
+ "EICHERMOT": ["eicher"],
170
+ "DIVISLAB": ["divi's", "divis"],
171
+ "CIPLA": ["cipla"],
172
+ "HCLTECH": ["hcl tech"],
173
+ "ONGC": ["ongc", "oil and natural gas"],
174
+ "HINDALCO": ["hindalco"],
175
+ "DRREDDY": ["dr reddy"],
176
+ "INDUSINDBK": ["indusind"],
177
+ "BEL": ["bharat electronics"],
178
+ "HAL": ["hindustan aeronautics"],
179
+ "IOC": ["indian oil"],
180
+ "VEDL": ["vedanta"],
181
+ "DLF": ["dlf"],
182
+ "SBILIFE": ["sbi life"],
183
+ "HDFCLIFE": ["hdfc life"],
184
+ "ZOMATO": ["zomato"],
185
+ "PAYTM": ["paytm", "one97"],
186
+ "NYKAA": ["nykaa", "fsn"],
187
+ "IRCTC": ["irctc", "railway"],
188
+ }
189
+
190
+ # Check known aliases
191
+ if ticker_base.upper() in aliases:
192
+ for alias in aliases[ticker_base.upper()]:
193
+ if alias in headline_lower:
194
+ return True
195
+
196
+ # Default: trust yfinance — it already curates news for this ticker
197
+ return True
198
+
199
+
200
+ def _check_overrides(headline: str) -> float | None:
201
+ """
202
+ Force scores for clear financial signals that NLP models often miss.
203
+ Returns: Score (-1 to 1) or None.
204
+ """
205
+ hl = headline.lower()
206
+
207
+ # Strong Negative Signals
208
+ if any(x in hl for x in ["cuts", "lowers", "slashes", "misses", "below"]) and \
209
+ any(x in hl for x in ["guidance", "forecast", "outlook", "estimate", "target"]):
210
+ return -0.8
211
+
212
+ if "bankruptcy" in hl or "fraud" in hl or "scam" in hl or "default" in hl:
213
+ return -0.9
214
+
215
+ # Strong Positive Signals
216
+ if any(x in hl for x in ["raises", "hikes", "beats", "above", "record"]) and \
217
+ any(x in hl for x in ["guidance", "forecast", "outlook", "estimate", "profit", "revenue"]):
218
+ return 0.8
219
+
220
+ if "wins order" in hl or "wins contract" in hl or "bagged order" in hl:
221
+ return 0.7
222
+
223
+ return None
224
+
225
+
226
+ def _is_noise(headline: str) -> bool:
227
+ """Discard junk headlines."""
228
+ # Too short or just a company name
229
+ if len(headline) < 20:
230
+ return True
231
+ if "share price" in headline.lower() and "live" in headline.lower():
232
+ # "Reliance share price live" -> Noise, no sentiment
233
+ return True
234
+ return False
235
+
236
+
237
  def analyze_sentiment(ticker: str) -> dict:
238
  headlines = fetch_news_headlines(ticker)
239
+
240
+ valid_details = []
241
+
242
+ for h in headlines:
243
+ # 1. Filter Noise
244
+ if _is_noise(h):
245
+ continue
246
+
247
+ # 2. Filter Relevance
248
+ if not _is_relevant(h, ticker):
249
+ continue
250
+
251
+ # 3. Check Overrides (Priority 1)
252
+ override_score = _check_overrides(h)
253
+ if override_score is not None:
254
+ label = "positive" if override_score > 0 else "negative"
255
+ valid_details.append({
256
+ "text": h,
257
+ "label": label,
258
+ "confidence": 1.0,
259
+ "combined_score": override_score,
260
+ "method": "Keyword Override"
261
+ })
262
+ continue
263
+
264
+ # 4. Use Model (Priority 2)
265
+ score_data = _score_text(h)
266
+ valid_details.append(score_data)
267
+
268
+ if not valid_details:
269
  return {
270
  "ticker": ticker,
271
  "articles_analyzed": 0,
 
273
  "overall_sentiment": "neutral",
274
  "overall_score": 0.0,
275
  "contradictory_dampened": False,
276
+ "method": "Hybrid (Keywords + VADER/FinBERT)",
277
  }
278
 
279
+ raw_scores = [d["combined_score"] for d in valid_details]
 
280
  avg = float(np.mean(raw_scores))
281
 
282
+ # Dampen one-off outliers
 
283
  contradictory_dampened = False
284
  n = len(raw_scores)
285
  if n >= 2:
 
289
  contradictory = variance > 0.15 or abs(avg - median) > 0.35
290
  if contradictory:
291
  contradictory_dampened = True
292
+ # Pull toward median
293
  avg = 0.4 * avg + 0.6 * median
294
 
295
+ if avg > 0.1:
296
  overall = "positive"
297
+ elif avg < -0.1:
298
  overall = "negative"
299
  else:
300
  overall = "neutral"
301
 
302
  return {
303
  "ticker": ticker,
304
+ "articles_analyzed": len(valid_details),
305
+ "details": valid_details,
306
  "overall_sentiment": overall,
307
  "overall_score": round(avg, 4),
308
  "contradictory_dampened": contradictory_dampened,
309
+ "method": "Hybrid (Keywords + VADER/FinBERT)",
310
  }