File size: 22,058 Bytes
792c6a9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
# utils.py
from googlesearch import search
import requests
from bs4 import BeautifulSoup
import time
from huggingface_hub import InferenceClient
from langchain_groq import ChatGroq
from langchain_core.prompts import ChatPromptTemplate
from gtts import gTTS
import os
from google import genai
from translate import Translator
import re
from openai import OpenAI
import json
from config import (
    llm,
    client,
    API_URL,
    HEADERS,
    openrouter_client,
    translator,
    genai_client,
    genai_client2,
)


from googlesearch import search as google_search

def extract_news(company, num_articles=10):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
    }

    articles = []
    seen_urls = set()
    sentiment_count = {"Positive": 0, "Negative": 0, "Neutral": 0}

    print(f"Fetching {num_articles} valid articles...")

    while len(articles) < num_articles:
        try:
            for url in google_search(f"{company} news", num=num_articles * 2, stop=num_articles * 2):
                if url in seen_urls or any(
                    blocked in url
                    for blocked in ["youtube.com", "youtu.be", "twitter.com", "x.com"]
                ):
                    continue

                seen_urls.add(url)
                print(f"\nFetching article {len(articles) + 1}: {url}")

                try:
                    article_response = requests.get(url, headers=headers, timeout=10)
                    article_response.raise_for_status()

                    article_soup = BeautifulSoup(article_response.text, "html.parser")
                    title = (
                        article_soup.find("title").get_text()
                        if article_soup.find("title")
                        else "No Title Found"
                    )

                    if title == "No Title Found":
                        print(f"Skipping article {url} due to missing title")
                        continue

                    paragraphs = article_soup.find_all("p")
                    content = " ".join([para.get_text() for para in paragraphs])

                    if not content.strip():
                        print(f"No content found for {url}")
                        continue

                    # Summarize the article
                    summary, translated_summary = summarize_article(content)

                    # Sentiment analysis
                    sentiment, reason = analyze_sentiment(summary)
                    if sentiment in sentiment_count:
                        sentiment_count[sentiment] += 1

                    # Extract relevant topics using LLM
                    topics = extract_topics(content)

                    metadata = {
                        "URL": url,
                        "Title": title,
                        "Summary": summary,
                        "Summary(in Hindi)": translated_summary,
                        "Sentiment": sentiment,
                        "Reason(for Sentiment)": reason,
                        "Topics": topics,
                    }
                    articles.append(metadata)

                    if len(articles) >= num_articles:
                        break

                    time.sleep(1)

                except requests.exceptions.Timeout:
                    print(f"Timeout error while fetching {url}")
                except requests.exceptions.RequestException as e:
                    print(f"Failed to fetch article {url}: {e}")

        except Exception as e:
            print(f"Error while searching for articles: {e}")
    final_sentiment = final_sentiment_analysis(sentiment_count)
    # Create final JSON output
    output = {
        "Company": company,
        "Articles": articles,
        "Comparative Sentiment Score": {
            "Sentiment Distribution": sentiment_count,
            "Total Sentiments": sum(sentiment_count.values()),
        },
        "Final Sentiment Analysis": final_sentiment,
    }

    print(f"\nSuccessfully fetched {len(articles)} valid articles.")

    # Pretty print JSON
    print(json.dumps(output, indent=4))
    return output


def clean_and_parse_json(response):
    try:
        # Extract JSON block using regex
        cleaned_response = re.search(r"\{.*\}", response, re.DOTALL)
        if cleaned_response:
            cleaned_response = cleaned_response.group()

            # Replace single quotes with double quotes
            cleaned_response = cleaned_response.replace("'", '"')

            # Remove trailing commas
            cleaned_response = re.sub(r",\s*([\]}])", r"\1", cleaned_response)

            # Parse JSON
            return json.loads(cleaned_response)
        else:
            raise ValueError("No JSON found in response.")
    except Exception as e:
        print(f"Parsing error: {e}")
        return {"Analysis": "Failed to parse structured output from the model."}


def final_sentiment_analysis(sentiment_count):
    try:
        sentiment_data = json.dumps(sentiment_count, indent=2)
        print("Sentiment Data:\n", sentiment_data)

        # Use Gemma for sentiment analysis
        response = genai_client2.models.generate_content(
            model="gemma-3-27b-it",
            contents=(
                f"Analyze the following sentiment distribution:\n\n"
                f"{sentiment_data}\n\n"
                f"Provide a FINAL sentiment analysis based on the distribution. "
                f"Summarize the overall sentiment and any patterns or insights in EXACTLY THREE LINES. "
                f"Return the result in a normal string format with summary about the sentiment and particular company current form."
            ),
        )

        if response and response.text:
            generated_text = response.text.strip()
            print("Generated Text (Gemma):", generated_text)
        else:
            print("Error: No response from the model.")
            generated_text = "Failed to generate response from Gemma."
            print("Generated Text:", generated_text)

        # Clean and parse JSON
        # generated_text = clean_and_parse_json(generated_text)
        return generated_text

    except Exception as e:
        print(f"Gemma sentiment analysis failed: {e}")
        try:
            print("Falling back to OpenRouter (qwen/qwq)...")
            completion = openrouter_client.chat.completions.create(
                model="qwen/qwq-32b:free",
                messages=[
                    {
                        "role": "user",
                        "content": (
                            f"Analyze the following sentiment distribution:\n\n"
                            f"{sentiment_data}\n\n"
                            f"Provide a FINAL sentiment analysis based on the distribution. "
                            f"Summarize the overall sentiment and any patterns or insights in EXACTLY THREE LINES. "
                            f"Return the result in a normal string format with summary about the sentiment and particular company current form."
                        ),
                    }
                ],
            )

            generated_text = completion.choices[0].message.content.strip()
            print("OpenRouter Generated Text:", generated_text)

            # Clean and parse JSON
            # generated_text = clean_and_parse_json(generated_text)
            return generated_text

        except Exception as e:
            print(f"OpenRouter sentiment analysis failed: {e}")
            return {
                "Summary": "Sentiment analysis failed using both models.",
                "Dominant Sentiment": "Unknown",
                "Insight": "No insights available.",
            }


def summarize_article(content):
    try:
        # Use the Inference API for summarization
        result = client.summarization(content, model="facebook/bart-large-cnn")
        summary = result["summary_text"]

        # Translate summary to Hindi
        translated_summary = translator.translate(summary)

        return summary, translated_summary

    except Exception as e:
        print(f"Hugging Face summarization failed: {e}")
        print("Trying Gemini as fallback...")

        try:
            truncated_content = content[:1024]
            # Use Gemini model as fallback
            response = genai_client2.models.generate_content(
                model="gemma-3-27b-it",
                contents=f"Summarize the following article:\n\n{truncated_content}",
            )
            summary = response.text.strip()
            translated_summary = translator.translate(summary)

            return summary, translated_summary

        except Exception as e:
            print(f"Gemini summarization also failed: {e}")
            return "Summary not available", "Summary not available"


def convert_text_to_speech(text, filename):
    try:
        # Try using gTTS for Hindi TTS
        tts = gTTS(text=text, lang="hi")
        tts.save(filename)

        return filename

    except Exception as e:
        print(f"Error during TTS conversion with gTTS: {e}")
        print("Switching to Hugging Face MMS-TTS...")

        # Use Hugging Face MMS-TTS as a fallback
        try:
            payload = {"inputs": text}
            response = requests.post(API_URL, headers=HEADERS, json=payload)
            if response.status_code == 200:
                with open(filename, "wb") as f:
                    f.write(response.content)
                print("Audio file generated using Hugging Face MMS-TTS.")
                return filename
            else:
                print(
                    f"Hugging Face API error: {response.status_code} - {response.text}"
                )
                return None

        except Exception as e:
            print(f"Error during TTS conversion with Hugging Face MMS-TTS: {e}")
            return None


def analyze_sentiment(content):
    try:
        # Use Gemini LLM for sentiment analysis as the primary method
        truncated_content = content
        response = genai_client.models.generate_content(
            model="gemma-3-27b-it",
            contents=(
                f"Analyze the sentiment of the following article:\n\n{truncated_content}\n\n"
                "Return the response as simple string . It shoild have sentiment and reason both are seperated by comma"
                "Sentiment should be Positive, Negative, or Neutral. No other extra text or contents."
            ),
        )
        # print("RESponse", response)
        # Parse JSON response
        # generated_text = response.text.strip()
        # print("Generated Response:", generated_text)

        # Split the response into sentiment and reason
        parts = response.text.split(",", 1)
        if len(parts) == 2:
            sentiment = parts[0].strip()
            reason = parts[1].strip()
            print(f"Sentiment: {sentiment}")
            print(f"Reason: {reason}")
        else:
            print("Invalid response format")

        if sentiment.lower() not in ["positive", "negative", "neutral"]:
            sentiment = "Sentiment analysis not available"

        return sentiment, reason

    except Exception as e:
        print(f"Error during Gemini LLM sentiment analysis: {e}")
        print("Trying HF Inference as fallback for sentiment analysis...")

        try:
            # Fallback to HF Inference for sentiment analysis
            truncated_content = content
            result = client.text_classification(
                truncated_content,
                model="finiteautomata/bertweet-base-sentiment-analysis",
            )
            hf_sentiment = result[0]["label"]

            # Map HF inference sentiment labels to required format
            sentiment_mapping = {"POS": "Positive", "NEG": "Negative", "NEU": "Neutral"}

            sentiment = sentiment_mapping.get(
                hf_sentiment, "Sentiment analysis not available"
            )
            return sentiment, ""

        except Exception as e:
            print(f"HF Inference sentiment analysis also failed: {e}")
            return "Sentiment analysis not available", ""


def extract_topics(content):
    try:
        prompt = ChatPromptTemplate.from_messages(
            [
                (
                    "system",
                    "You are a helpful assistant that extracts key topics from articles.",
                ),
                (
                    "human",
                    f"Extract the **three most relevant and suitable two words topics** from the following article:\n\n{content}\n\n"
                    "Provide exactly 5 topics, separated by commas. Avoid numbering or extra details.",
                ),
            ]
        )
        chain = prompt | llm
        response = chain.invoke({})

        # Clean and process response
        if isinstance(response, str):
            topics = response.split(",")
        elif hasattr(response, "content") and isinstance(response.content, str):
            topics = response.content.split(",")
        else:
            topics = ["Topics not available"]

        # Remove unwanted characters and extra spaces
        topics = [re.sub(r"^\d+[\.\)\-]?\s*", "", topic.strip()) for topic in topics]

        return topics

    except Exception as e:
        print(f"Groq topic extraction failed: {e}")
        print("Trying Gemini as fallback...")

        try:
            # Use Gemini for topic extraction
            response = genai_client.models.generate_content(
                model="gemma-3-27b-it",
                contents=f"Extract exactly 5 key topics (two or one words) from the following article:\n\n{content}\n\n"
                "Return them as a comma-separated list without numbering or extra details.",
            )
            topics = response.text.strip().split(",")

            # Clean up and format the topics
            topics = [
                re.sub(r"^\d+[\.\)\-]?\s*", "", topic.strip()) for topic in topics
            ]

            return topics

        except Exception as e:
            print(f"Gemini topic extraction also failed: {e}")
            return ["Topics not available"]


def clean_response(response):
    cleaned_response = re.search(r"\{.*\}", response, re.DOTALL)
    if cleaned_response:
        return cleaned_response.group()
    return response


def generate_comparative_analysis(articles):
    num_articles = min(len(articles["Articles"]), 5)

    all_contents = "\n\n".join(
        [
            f"Article {i + 1}: {article['Summary']}"
            for i, article in enumerate(articles["Articles"][:num_articles])
        ]
    )

    print(all_contents)

    try:
        # Use Gemma for comparative analysis
        response = genai_client.models.generate_content(
            model="gemma-3-27b-it",
            contents=(
                f"Perform a detailed comparative analysis of the following {num_articles} articles:\n\n"
                f"{all_contents}\n\n"
                f"Provide differences in coverage and potential impacts between each pair of articles.\n"
                f"Return the result in the following structured JSON format:\n"
                f"{{\n"
                f'  "Coverage Differences": [\n'
                f"    {{\n"
                f'      "Comparison": "<summary of difference between article X and article Y>",\n'
                f'      "Impact": "<impact of these differences>"\n'
                f"    }}\n"
                f"  ]\n"
                f"}}"
            ),
        )

        generated_text = response.text.strip()
        print("Generated Text (Gemma):", generated_text)

        # Clean and parse the response
        cleaned_response = clean_response(generated_text)
        structured_response = json.loads(cleaned_response)

    except Exception as e:
        print(f"Gemma comparative analysis failed: {e}")

        try:
            print("Falling back to OpenRouter (qwq)...")
            if "all_contents" not in locals():
                all_contents = "\n\n".join(
                    [
                        f"Article {i+1}: {article['Summary']}"
                        for i, article in enumerate(articles["Articles"])
                    ]
                )
            completion = openrouter_client.chat.completions.create(
                model="qwen/qwq-32b:free",
                messages=[
                    {
                        "role": "user",
                        "content": (
                            f"Perform a detailed comparative analysis of the following {num_articles} articles:\n\n"
                            f"{all_contents}\n\n"
                            f"Provide differences in coverage and potential impacts between each pair of articles.\n"
                            f"Return the result in the following structured JSON format:\n"
                            f"{{\n"
                            f'  "Coverage Differences": [\n'
                            f"    {{\n"
                            f'      "Comparison": "<summary of difference between article X and article Y>",\n'
                            f'      "Impact": "<impact of these differences>"\n'
                            f"    }}\n"
                            f"  ]\n"
                            f"}}"
                        ),
                    }
                ],
            )

            # Extract response
            generated_text = completion.choices[0].message.content.strip()
            print("Generated Text (OpenRouter):", generated_text)

            # Clean and parse the response
            cleaned_response = clean_response(generated_text)
            structured_response = json.loads(cleaned_response)

        except Exception as e:
            print(f"OpenRouter fallback failed: {e}")
            structured_response = {
                "Coverage Differences": [
                    {
                        "Comparison": "Failed to generate comparative analysis from both Gemma and OpenRouter.",
                        "Impact": "Check the model's output format or prompt.",
                    }
                ]
            }

    #  Merge the result back into the input
    articles["Coverage Differences"] = structured_response["Coverage Differences"]
    return articles


def add_topic_overlap_to_coverage_differences(news_articles):
    try:
        # Get the list of topics from each article
        article_topics = [
            set(article["Topics"]) for article in news_articles["Articles"]
        ]

        if len(article_topics) < 2:
            raise ValueError(
                "At least two articles are required for topic overlap analysis."
            )

        # Step 1: Find topics that are common in at least two articles
        topic_occurrences = {}
        for i, topics in enumerate(article_topics):
            for topic in topics:
                if topic not in topic_occurrences:
                    topic_occurrences[topic] = []
                topic_occurrences[topic].append(
                    i + 1
                )  # Storing article index (1-based)

        # Step 2: Filter out topics appearing in at least two articles
        common_topics = [
            f"{topic} (Article {', '.join(map(str, indexes))})"
            for topic, indexes in topic_occurrences.items()
            if len(indexes) >= 2
        ]

        # Step 3: Find unique topics for each article
        unique_topics = []
        for i, topics in enumerate(article_topics):
            other_topics = set.union(*(article_topics[:i] + article_topics[i + 1 :]))
            unique = topics - other_topics
            unique_topics.append(list(unique))

        # Step 4: Build the output structure for Coverage Differences
        topic_overlap = {"Common Topics": common_topics, "Unique Topics": {}}

        for i, topics in enumerate(unique_topics):
            topic_overlap["Unique Topics"][f"Article {i + 1}"] = topics

        #  Step 5: Add topic_overlap under "Coverage Differences"
        if "Coverage Differences" not in news_articles:
            news_articles["Coverage Differences"] = []

        news_articles["Coverage Differences"].append({"Topic Overlap": topic_overlap})

        return news_articles

    except Exception as e:
        print(f"Error: {e}")
        return None


def generate_company_analysis_in_hindi(company_name, articles):
    try:
        # Extract summaries and sentiment from updated articles
        summaries = "\n\n".join(
            [
                f"Article {i+1}: {article['Summary']}\nSentiment: {article.get('Sentiment', 'Unknown')}"
                for i, article in enumerate(articles["Articles"])
            ]
        )

        # Create the prompt in English, request output in Hindi
        prompt = (
            f"Please prepare a detailed analysis of the company '{company_name}' based on the following articles. "
            f"Include the sentiment and key points from each article.\n\n"
            f"{summaries}\n\n"
            f"The analysis should cover the overall performance of the company, the sentiment reflected in the articles, "
            f"and a comparison of the articles' coverage.\n\n"
            f"Output should be in **Hindi** and in a single detailed paragraph based on these articles.Don't provide in english"
        )

        # Generate content using Gemini Gemma model
        response = genai_client2.models.generate_content(
            model="gemma-3-27b-it",
            contents=prompt,
        )

        if response and response.text:
            analysis_text = response.text.strip()
            print("\n🔎 Generated Company Analysis (Hindi):\n")
            print(analysis_text)
            return analysis_text
        else:
            raise ValueError("Gemini returned an empty response")

    except Exception as e:
        print(f"Gemini analysis generation failed: {e}")
        return None