Commit ·
3fb5618
1
Parent(s): c6cbfc9
Enhance API with Randomization and OWASP Security Controls
Browse files- Web_scrapper.py +27 -3
- requirements.txt +4 -0
Web_scrapper.py
CHANGED
|
@@ -3,9 +3,28 @@ import requests
|
|
| 3 |
from bs4 import BeautifulSoup
|
| 4 |
import feedparser
|
| 5 |
import concurrent.futures
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
app = Flask(__name__)
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
# Strictly Tech & Coding RSS Feeds
|
| 10 |
NEWS_SOURCES = [
|
| 11 |
# Global Developer Communities
|
|
@@ -39,7 +58,7 @@ def scrape_github_trending():
|
|
| 39 |
try:
|
| 40 |
response = requests.get(url, headers=headers, timeout=5)
|
| 41 |
soup = BeautifulSoup(response.text, "html.parser")
|
| 42 |
-
for article in soup.select("article.Box-row")[:
|
| 43 |
h2 = article.select_one("h2.h3 a")
|
| 44 |
if not h2: continue
|
| 45 |
title = h2.text.strip().replace("\\n", "").replace(" ", "")
|
|
@@ -92,7 +111,7 @@ def scrape_feed(source):
|
|
| 92 |
try:
|
| 93 |
feed = feedparser.parse(source["url"])
|
| 94 |
|
| 95 |
-
for entry in feed.entries[:
|
| 96 |
title = entry.title
|
| 97 |
link = entry.link
|
| 98 |
|
|
@@ -161,6 +180,7 @@ def index():
|
|
| 161 |
return render_template_string(html_content)
|
| 162 |
|
| 163 |
@app.route("/news", methods=["GET"])
|
|
|
|
| 164 |
def get_coding_news():
|
| 165 |
def fetch_source(source):
|
| 166 |
if "scraper" in source:
|
|
@@ -175,7 +195,11 @@ def get_coding_news():
|
|
| 175 |
for res in results:
|
| 176 |
all_news.extend(res)
|
| 177 |
|
| 178 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
|
| 180 |
if __name__ == "__main__":
|
| 181 |
app.run(host="0.0.0.0", port=7860, debug=True)
|
|
|
|
| 3 |
from bs4 import BeautifulSoup
|
| 4 |
import feedparser
|
| 5 |
import concurrent.futures
|
| 6 |
+
import random
|
| 7 |
+
from flask_limiter import Limiter
|
| 8 |
+
from flask_limiter.util import get_remote_address
|
| 9 |
+
from flask_talisman import Talisman
|
| 10 |
+
from flask_cors import CORS
|
| 11 |
|
| 12 |
app = Flask(__name__)
|
| 13 |
|
| 14 |
+
# OWASP Security: Cross-Origin Resource Sharing
|
| 15 |
+
CORS(app)
|
| 16 |
+
|
| 17 |
+
# OWASP Security: HTTP Security Headers
|
| 18 |
+
Talisman(app, force_https=False, content_security_policy=None)
|
| 19 |
+
|
| 20 |
+
# OWASP Security: Rate Limiting (Prevent DDoS/Brute-force)
|
| 21 |
+
limiter = Limiter(
|
| 22 |
+
get_remote_address,
|
| 23 |
+
app=app,
|
| 24 |
+
default_limits=["1000 per day", "60 per minute"],
|
| 25 |
+
storage_uri="memory://"
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
# Strictly Tech & Coding RSS Feeds
|
| 29 |
NEWS_SOURCES = [
|
| 30 |
# Global Developer Communities
|
|
|
|
| 58 |
try:
|
| 59 |
response = requests.get(url, headers=headers, timeout=5)
|
| 60 |
soup = BeautifulSoup(response.text, "html.parser")
|
| 61 |
+
for article in soup.select("article.Box-row")[:15]:
|
| 62 |
h2 = article.select_one("h2.h3 a")
|
| 63 |
if not h2: continue
|
| 64 |
title = h2.text.strip().replace("\\n", "").replace(" ", "")
|
|
|
|
| 111 |
try:
|
| 112 |
feed = feedparser.parse(source["url"])
|
| 113 |
|
| 114 |
+
for entry in feed.entries[:15]: # Fetch more for randomization
|
| 115 |
title = entry.title
|
| 116 |
link = entry.link
|
| 117 |
|
|
|
|
| 180 |
return render_template_string(html_content)
|
| 181 |
|
| 182 |
@app.route("/news", methods=["GET"])
|
| 183 |
+
@limiter.limit("20 per minute") # Stricter rate limit on the actual scraping endpoint
|
| 184 |
def get_coding_news():
|
| 185 |
def fetch_source(source):
|
| 186 |
if "scraper" in source:
|
|
|
|
| 195 |
for res in results:
|
| 196 |
all_news.extend(res)
|
| 197 |
|
| 198 |
+
# Randomize the feed
|
| 199 |
+
random.shuffle(all_news)
|
| 200 |
+
|
| 201 |
+
# Return a randomized master list of up to 60 articles
|
| 202 |
+
return jsonify(all_news[:60])
|
| 203 |
|
| 204 |
if __name__ == "__main__":
|
| 205 |
app.run(host="0.0.0.0", port=7860, debug=True)
|
requirements.txt
CHANGED
|
@@ -3,3 +3,7 @@ requests
|
|
| 3 |
beautifulsoup4
|
| 4 |
feedparser
|
| 5 |
gunicorn
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
beautifulsoup4
|
| 4 |
feedparser
|
| 5 |
gunicorn
|
| 6 |
+
|
| 7 |
+
Flask-Limiter
|
| 8 |
+
Flask-Talisman
|
| 9 |
+
Flask-Cors
|