Spaces:
Build error
Build error
Upload 2 files
Browse files- dataExtraction.py +102 -0
- requirements.txt +7 -0
dataExtraction.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import requests
|
| 2 |
+
from bs4 import BeautifulSoup
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
# Define the URL for the courses list
|
| 6 |
+
courses_url = "https://courses.analyticsvidhya.com/pages/all-free-courses"
|
| 7 |
+
|
| 8 |
+
# Initialize a list to store all course dictionaries
|
| 9 |
+
all_courses = []
|
| 10 |
+
|
| 11 |
+
try:
|
| 12 |
+
# Send a GET request to fetch the page content
|
| 13 |
+
response = requests.get(courses_url)
|
| 14 |
+
|
| 15 |
+
# Check if the response is successful
|
| 16 |
+
if response.status_code == 200:
|
| 17 |
+
print("Successfully fetched the courses page.")
|
| 18 |
+
|
| 19 |
+
# Parse the main courses page
|
| 20 |
+
soup = BeautifulSoup(response.content, 'html.parser')
|
| 21 |
+
|
| 22 |
+
# Find all course links
|
| 23 |
+
course_links = soup.find_all('a', class_='course-card course-card__public published')
|
| 24 |
+
|
| 25 |
+
if not course_links:
|
| 26 |
+
print("No course links found on the page.")
|
| 27 |
+
|
| 28 |
+
for course_link in course_links:
|
| 29 |
+
# Initialize dictionary for current course
|
| 30 |
+
course_data = {
|
| 31 |
+
'title': '',
|
| 32 |
+
'url': '',
|
| 33 |
+
'description': '',
|
| 34 |
+
'topics': [],
|
| 35 |
+
'curriculum': ''
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
# Extract the href URL
|
| 39 |
+
course_url = course_link.get('href')
|
| 40 |
+
if not course_url:
|
| 41 |
+
print("No URL found for this course")
|
| 42 |
+
continue
|
| 43 |
+
|
| 44 |
+
# If the URL is relative, make it absolute
|
| 45 |
+
if course_url.startswith('/'):
|
| 46 |
+
course_url = f"https://courses.analyticsvidhya.com{course_url}"
|
| 47 |
+
|
| 48 |
+
course_data['url'] = course_url
|
| 49 |
+
print(f"Processing course URL: {course_url}")
|
| 50 |
+
|
| 51 |
+
try:
|
| 52 |
+
# Send a GET request to fetch the course detail page content
|
| 53 |
+
course_response = requests.get(course_url)
|
| 54 |
+
|
| 55 |
+
if course_response.status_code == 200:
|
| 56 |
+
course_soup = BeautifulSoup(course_response.content, 'html.parser')
|
| 57 |
+
|
| 58 |
+
# Extract course title
|
| 59 |
+
title_tag = course_soup.find('h1')
|
| 60 |
+
course_data['title'] = title_tag.text.strip() if title_tag else "N/A"
|
| 61 |
+
|
| 62 |
+
# Extract course description
|
| 63 |
+
description_section = course_soup.find('div', class_='rich-text__wrapper')
|
| 64 |
+
course_data['description'] = description_section.get_text(strip=True) if description_section else "No description available."
|
| 65 |
+
|
| 66 |
+
# Extract topics (if available)
|
| 67 |
+
topics_section = course_soup.find('div', class_='course-topics__container')
|
| 68 |
+
if topics_section:
|
| 69 |
+
topics = topics_section.find_all('div', class_='topic-card')
|
| 70 |
+
course_data['topics'] = [topic.get_text(strip=True) for topic in topics]
|
| 71 |
+
|
| 72 |
+
# Extract curriculum structure
|
| 73 |
+
curriculum_section = course_soup.find('div', class_='course-curriculum__container')
|
| 74 |
+
course_data['curriculum'] = curriculum_section.get_text(strip=True) if curriculum_section else "No curriculum available."
|
| 75 |
+
|
| 76 |
+
# Add the course data to our list of all courses
|
| 77 |
+
all_courses.append(course_data)
|
| 78 |
+
print(f"Successfully processed course: {course_data['title']}")
|
| 79 |
+
|
| 80 |
+
else:
|
| 81 |
+
print(f"Failed to retrieve the details from {course_url}")
|
| 82 |
+
|
| 83 |
+
except requests.RequestException as e:
|
| 84 |
+
print(f"Error processing course URL {course_url}: {str(e)}")
|
| 85 |
+
|
| 86 |
+
# Save all courses data to a JSON file
|
| 87 |
+
with open('all_courses.json', 'w', encoding='utf-8') as f:
|
| 88 |
+
json.dump(all_courses, f, indent=4, ensure_ascii=False)
|
| 89 |
+
print(f"\nSuccessfully saved data for {len(all_courses)} courses to all_courses.json")
|
| 90 |
+
|
| 91 |
+
# Print example of the first course data (if available)
|
| 92 |
+
if all_courses:
|
| 93 |
+
print("\nExample of first course data structure:")
|
| 94 |
+
print(json.dumps(all_courses[0], indent=2))
|
| 95 |
+
|
| 96 |
+
else:
|
| 97 |
+
print(f"Failed to retrieve the courses page. Status code: {response.status_code}")
|
| 98 |
+
|
| 99 |
+
except requests.RequestException as e:
|
| 100 |
+
print(f"Error fetching the courses page: {str(e)}")
|
| 101 |
+
except Exception as e:
|
| 102 |
+
print(f"An unexpected error occurred: {str(e)}")
|
requirements.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
streamlit==1.31.0
|
| 2 |
+
google-generativeai==0.3.2
|
| 3 |
+
beautifulsoup4==4.12.3
|
| 4 |
+
requests==2.31.0
|
| 5 |
+
typing-extensions==4.9.0
|
| 6 |
+
python-dotenv==1.0.1
|
| 7 |
+
protobuf==4.25.2
|