Spaces:
Build error
Build error
File size: 4,579 Bytes
63fa23b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | import requests
from bs4 import BeautifulSoup
import json
# Define the URL for the courses list
courses_url = "https://courses.analyticsvidhya.com/pages/all-free-courses"
# Initialize a list to store all course dictionaries
all_courses = []
try:
# Send a GET request to fetch the page content
response = requests.get(courses_url)
# Check if the response is successful
if response.status_code == 200:
print("Successfully fetched the courses page.")
# Parse the main courses page
soup = BeautifulSoup(response.content, 'html.parser')
# Find all course links
course_links = soup.find_all('a', class_='course-card course-card__public published')
if not course_links:
print("No course links found on the page.")
for course_link in course_links:
# Initialize dictionary for current course
course_data = {
'title': '',
'url': '',
'description': '',
'topics': [],
'curriculum': ''
}
# Extract the href URL
course_url = course_link.get('href')
if not course_url:
print("No URL found for this course")
continue
# If the URL is relative, make it absolute
if course_url.startswith('/'):
course_url = f"https://courses.analyticsvidhya.com{course_url}"
course_data['url'] = course_url
print(f"Processing course URL: {course_url}")
try:
# Send a GET request to fetch the course detail page content
course_response = requests.get(course_url)
if course_response.status_code == 200:
course_soup = BeautifulSoup(course_response.content, 'html.parser')
# Extract course title
title_tag = course_soup.find('h1')
course_data['title'] = title_tag.text.strip() if title_tag else "N/A"
# Extract course description
description_section = course_soup.find('div', class_='rich-text__wrapper')
course_data['description'] = description_section.get_text(strip=True) if description_section else "No description available."
# Extract topics (if available)
topics_section = course_soup.find('div', class_='course-topics__container')
if topics_section:
topics = topics_section.find_all('div', class_='topic-card')
course_data['topics'] = [topic.get_text(strip=True) for topic in topics]
# Extract curriculum structure
curriculum_section = course_soup.find('div', class_='course-curriculum__container')
course_data['curriculum'] = curriculum_section.get_text(strip=True) if curriculum_section else "No curriculum available."
# Add the course data to our list of all courses
all_courses.append(course_data)
print(f"Successfully processed course: {course_data['title']}")
else:
print(f"Failed to retrieve the details from {course_url}")
except requests.RequestException as e:
print(f"Error processing course URL {course_url}: {str(e)}")
# Save all courses data to a JSON file
with open('all_courses.json', 'w', encoding='utf-8') as f:
json.dump(all_courses, f, indent=4, ensure_ascii=False)
print(f"\nSuccessfully saved data for {len(all_courses)} courses to all_courses.json")
# Print example of the first course data (if available)
if all_courses:
print("\nExample of first course data structure:")
print(json.dumps(all_courses[0], indent=2))
else:
print(f"Failed to retrieve the courses page. Status code: {response.status_code}")
except requests.RequestException as e:
print(f"Error fetching the courses page: {str(e)}")
except Exception as e:
print(f"An unexpected error occurred: {str(e)}") |