Aditya Amol Patil commited on
Commit
6ecb61e
·
unverified ·
1 Parent(s): 8034e8c

Create web_scrap.py

Browse files
Files changed (1) hide show
  1. web_scrap.py +110 -0
web_scrap.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import random
4
+ from flask import Flask, jsonify
5
+ import requests
6
+ from bs4 import BeautifulSoup
7
+
8
+ app = Flask(__name__)
9
+
10
+ NEWS_SOURCES = {
11
+ "Hacker News": {"url": "https://news.ycombinator.com/", "selectors": {"article": ".athing", "title": ".storylink", "link": ".storylink"}},
12
+ "FreeCodeCamp": {"url": "https://www.freecodecamp.org/news/", "selectors": {"article": ".post-card", "title": "h2 a", "link": "h2 a", "image": "img"}},
13
+ "Ars Technica": {"url": "https://arstechnica.com/information-technology/", "selectors": {"article": "li.article", "title": "h2 a", "link": "h2 a", "image": "img"}},
14
+ "InfoQ": {"url": "https://www.infoq.com/development/news/", "selectors": {"article": ".news-list .news_item", "title": "h2 a", "link": "h2 a", "image": ".news_image img"}},
15
+ "TechCrunch": {"url": "https://techcrunch.com/startups/", "selectors": {"article": "article", "title": "h2 a", "link": "h2 a", "image": "img"}},
16
+ "Dev.to": {"url": "https://dev.to/", "selectors": {"article": ".crayons-story", "title": "h2 a", "link": "h2 a", "image": "img"}},
17
+ "Google AI Blog": {"url": "https://ai.googleblog.com/", "selectors": {"article": ".date-outer", "title": ".post-title.entry-title a", "link": ".post-title.entry-title a"}},
18
+ "NVIDIA Developer Blog": {"url": "https://developer.nvidia.com/blog/", "selectors": {"article": "article", "title": "h2 a", "link": "h2 a", "image": "img"}},
19
+ "JetBrains Blog": {"url": "https://blog.jetbrains.com/", "selectors": {"article": ".post-item", "title": ".post-title a", "link": ".post-title a", "image": "img"}},
20
+ "Stack Overflow Blog": {"url": "https://stackoverflow.blog/", "selectors": {"article": ".post", "title": "h2 a", "link": "h2 a", "image": "img"}},
21
+ "OpenAI Blog": {"url": "https://openai.com/research/", "selectors": {"article": ".post", "title": "h3 a", "link": "h3 a", "image": "img"}},
22
+ "Smashing Magazine": {"url": "https://www.smashingmagazine.com/articles/", "selectors": {"article": "article", "title": "h2 a", "link": "h2 a", "image": "img"}},
23
+ "TechCrunch": {"url": "https://techcrunch.com/startups/", "selectors": {"article": "article", "title": "h2 a", "link": "h2 a", "image": "img"}},
24
+ "Dev.to": {"url": "https://dev.to/", "selectors": {"article": ".crayons-story", "title": "h2 a", "link": "h2 a", "image": "img"}},
25
+ "Google AI Blog": {"url": "https://ai.googleblog.com/", "selectors": {"article": ".date-outer", "title": ".post-title.entry-title a", "link": ".post-title.entry-title a"}},
26
+ "NVIDIA Developer Blog": {"url": "https://developer.nvidia.com/blog/", "selectors": {"article": "article", "title": "h2 a", "link": "h2 a", "image": "img"}},
27
+ "JetBrains Blog": {"url": "https://blog.jetbrains.com/", "selectors": {"article": ".post-item", "title": ".post-title a", "link": ".post-title a", "image": "img"}},
28
+ "Stack Overflow Blog": {"url": "https://stackoverflow.blog/", "selectors": {"article": ".post", "title": "h2 a", "link": "h2 a", "image": "img"}},
29
+ "OpenAI Blog": {"url": "https://openai.com/research/", "selectors": {"article": ".post", "title": "h3 a", "link": "h3 a", "image": "img"}},
30
+ "MIT Technology Review": {"url": "https://www.technologyreview.com/topic/artificial-intelligence/", "selectors": {"article": ".river__item", "title": "h3 a", "link": "h3 a", "image": "img"}},
31
+ "Android Developers Blog": {"url": "https://android-developers.googleblog.com/", "selectors": {"article": ".post-outer", "title": "h3 a", "link": "h3 a", "image": "img"}},
32
+ "GitHub Trending": {"url": "https://github.com/trending", "selectors": {}}
33
+ }
34
+
35
+ def fetch_page(url):
36
+ """Fetch HTML page content safely."""
37
+ try:
38
+ response = requests.get(url, timeout=5)
39
+ response.raise_for_status()
40
+ return response.text
41
+ except requests.RequestException:
42
+ return None
43
+
44
+ def scrape_news(source_name, config):
45
+ """Scrapes news articles from a given source."""
46
+ if source_name == "GitHub Trending":
47
+ return get_github_trending()
48
+
49
+ html = fetch_page(config["url"])
50
+ if not html:
51
+ return []
52
+
53
+ soup = BeautifulSoup(html, "html.parser")
54
+ news_list = []
55
+
56
+ for article in soup.select(config["selectors"]["article"])[:5]: # Get top 5 articles
57
+ title_tag = article.select_one(config["selectors"]["title"])
58
+ link_tag = article.select_one(config["selectors"]["link"])
59
+ img_tag = article.select_one(config["selectors"].get("image", ""))
60
+
61
+ title = title_tag.text.strip() if title_tag else "No Title"
62
+ link = link_tag["href"] if link_tag else "#"
63
+ image = img_tag["src"] if img_tag and "src" in img_tag.attrs else None
64
+
65
+ if "base_url" in config and not link.startswith("http"):
66
+ link = config["base_url"] + link
67
+
68
+ news_list.append({"source": source_name, "title": title, "url": link, "image": image})
69
+
70
+ return news_list
71
+
72
+ def get_github_trending():
73
+ """Scrapes GitHub trending repositories."""
74
+ url = "https://github.com/trending"
75
+ headers = {"User-Agent": "Mozilla/5.0"}
76
+
77
+ response = requests.get(url, headers=headers)
78
+ if response.status_code != 200:
79
+ return []
80
+
81
+ soup = BeautifulSoup(response.text, "html.parser")
82
+ articles = []
83
+
84
+ for repo in soup.find_all("article", class_="Box-row")[:5]: # Get top 5 trending
85
+ title = repo.find("h2").text.strip().replace("\n", "").replace(" ", "")
86
+ repo_url = "https://github.com" + repo.find("h2").find("a")["href"]
87
+ image = "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png"
88
+
89
+ articles.append({"source": "GitHub Trending", "title": title, "url": repo_url, "image": image})
90
+
91
+ return articles
92
+
93
+ @app.route('/news', methods=['GET'])
94
+ def get_coding_news():
95
+ """Fetches news from random sources and shuffles output."""
96
+ try:
97
+ selected_sources = random.sample(list(NEWS_SOURCES.keys()), 8) # Select 8 random sources
98
+ all_news = []
99
+
100
+ for source in selected_sources:
101
+ all_news.extend(scrape_news(source, NEWS_SOURCES[source]))
102
+
103
+ random.shuffle(all_news) # Shuffle news articles to avoid repetition
104
+
105
+ return jsonify(all_news[:15]) # Limit response to 15 articles
106
+ except Exception as e:
107
+ return jsonify({"error": str(e)}), 500
108
+
109
+ if __name__ == "__main__":
110
+ app.run(host="0.0.0.0", port=7860, debug=True)