peterparker48 commited on
Commit
3fb5618
·
1 Parent(s): c6cbfc9

Enhance API with Randomization and OWASP Security Controls

Browse files
Files changed (2) hide show
  1. Web_scrapper.py +27 -3
  2. requirements.txt +4 -0
Web_scrapper.py CHANGED
@@ -3,9 +3,28 @@ import requests
3
  from bs4 import BeautifulSoup
4
  import feedparser
5
  import concurrent.futures
 
 
 
 
 
6
 
7
  app = Flask(__name__)
8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  # Strictly Tech & Coding RSS Feeds
10
  NEWS_SOURCES = [
11
  # Global Developer Communities
@@ -39,7 +58,7 @@ def scrape_github_trending():
39
  try:
40
  response = requests.get(url, headers=headers, timeout=5)
41
  soup = BeautifulSoup(response.text, "html.parser")
42
- for article in soup.select("article.Box-row")[:8]:
43
  h2 = article.select_one("h2.h3 a")
44
  if not h2: continue
45
  title = h2.text.strip().replace("\\n", "").replace(" ", "")
@@ -92,7 +111,7 @@ def scrape_feed(source):
92
  try:
93
  feed = feedparser.parse(source["url"])
94
 
95
- for entry in feed.entries[:8]: # Limit to top 8 from each source
96
  title = entry.title
97
  link = entry.link
98
 
@@ -161,6 +180,7 @@ def index():
161
  return render_template_string(html_content)
162
 
163
  @app.route("/news", methods=["GET"])
 
164
  def get_coding_news():
165
  def fetch_source(source):
166
  if "scraper" in source:
@@ -175,7 +195,11 @@ def get_coding_news():
175
  for res in results:
176
  all_news.extend(res)
177
 
178
- return jsonify(all_news)
 
 
 
 
179
 
180
  if __name__ == "__main__":
181
  app.run(host="0.0.0.0", port=7860, debug=True)
 
3
  from bs4 import BeautifulSoup
4
  import feedparser
5
  import concurrent.futures
6
+ import random
7
+ from flask_limiter import Limiter
8
+ from flask_limiter.util import get_remote_address
9
+ from flask_talisman import Talisman
10
+ from flask_cors import CORS
11
 
12
  app = Flask(__name__)
13
 
14
+ # OWASP Security: Cross-Origin Resource Sharing
15
+ CORS(app)
16
+
17
+ # OWASP Security: HTTP Security Headers
18
+ Talisman(app, force_https=False, content_security_policy=None)
19
+
20
+ # OWASP Security: Rate Limiting (Prevent DDoS/Brute-force)
21
+ limiter = Limiter(
22
+ get_remote_address,
23
+ app=app,
24
+ default_limits=["1000 per day", "60 per minute"],
25
+ storage_uri="memory://"
26
+ )
27
+
28
  # Strictly Tech & Coding RSS Feeds
29
  NEWS_SOURCES = [
30
  # Global Developer Communities
 
58
  try:
59
  response = requests.get(url, headers=headers, timeout=5)
60
  soup = BeautifulSoup(response.text, "html.parser")
61
+ for article in soup.select("article.Box-row")[:15]:
62
  h2 = article.select_one("h2.h3 a")
63
  if not h2: continue
64
  title = h2.text.strip().replace("\\n", "").replace(" ", "")
 
111
  try:
112
  feed = feedparser.parse(source["url"])
113
 
114
+ for entry in feed.entries[:15]: # Fetch more for randomization
115
  title = entry.title
116
  link = entry.link
117
 
 
180
  return render_template_string(html_content)
181
 
182
  @app.route("/news", methods=["GET"])
183
+ @limiter.limit("20 per minute") # Stricter rate limit on the actual scraping endpoint
184
  def get_coding_news():
185
  def fetch_source(source):
186
  if "scraper" in source:
 
195
  for res in results:
196
  all_news.extend(res)
197
 
198
+ # Randomize the feed
199
+ random.shuffle(all_news)
200
+
201
+ # Return a randomized master list of up to 60 articles
202
+ return jsonify(all_news[:60])
203
 
204
  if __name__ == "__main__":
205
  app.run(host="0.0.0.0", port=7860, debug=True)
requirements.txt CHANGED
@@ -3,3 +3,7 @@ requests
3
  beautifulsoup4
4
  feedparser
5
  gunicorn
 
 
 
 
 
3
  beautifulsoup4
4
  feedparser
5
  gunicorn
6
+
7
+ Flask-Limiter
8
+ Flask-Talisman
9
+ Flask-Cors