import requests from bs4 import BeautifulSoup import json # Define the URL for the courses list courses_url = "https://courses.analyticsvidhya.com/pages/all-free-courses" # Initialize a list to store all course dictionaries all_courses = [] try: # Send a GET request to fetch the page content response = requests.get(courses_url) # Check if the response is successful if response.status_code == 200: print("Successfully fetched the courses page.") # Parse the main courses page soup = BeautifulSoup(response.content, 'html.parser') # Find all course links course_links = soup.find_all('a', class_='course-card course-card__public published') if not course_links: print("No course links found on the page.") for course_link in course_links: # Initialize dictionary for current course course_data = { 'title': '', 'url': '', 'description': '', 'topics': [], 'curriculum': '' } # Extract the href URL course_url = course_link.get('href') if not course_url: print("No URL found for this course") continue # If the URL is relative, make it absolute if course_url.startswith('/'): course_url = f"https://courses.analyticsvidhya.com{course_url}" course_data['url'] = course_url print(f"Processing course URL: {course_url}") try: # Send a GET request to fetch the course detail page content course_response = requests.get(course_url) if course_response.status_code == 200: course_soup = BeautifulSoup(course_response.content, 'html.parser') # Extract course title title_tag = course_soup.find('h1') course_data['title'] = title_tag.text.strip() if title_tag else "N/A" # Extract course description description_section = course_soup.find('div', class_='rich-text__wrapper') course_data['description'] = description_section.get_text(strip=True) if description_section else "No description available." # Extract topics (if available) topics_section = course_soup.find('div', class_='course-topics__container') if topics_section: topics = topics_section.find_all('div', class_='topic-card') course_data['topics'] = [topic.get_text(strip=True) for topic in topics] # Extract curriculum structure curriculum_section = course_soup.find('div', class_='course-curriculum__container') course_data['curriculum'] = curriculum_section.get_text(strip=True) if curriculum_section else "No curriculum available." # Add the course data to our list of all courses all_courses.append(course_data) print(f"Successfully processed course: {course_data['title']}") else: print(f"Failed to retrieve the details from {course_url}") except requests.RequestException as e: print(f"Error processing course URL {course_url}: {str(e)}") # Save all courses data to a JSON file with open('all_courses.json', 'w', encoding='utf-8') as f: json.dump(all_courses, f, indent=4, ensure_ascii=False) print(f"\nSuccessfully saved data for {len(all_courses)} courses to all_courses.json") # Print example of the first course data (if available) if all_courses: print("\nExample of first course data structure:") print(json.dumps(all_courses[0], indent=2)) else: print(f"Failed to retrieve the courses page. Status code: {response.status_code}") except requests.RequestException as e: print(f"Error fetching the courses page: {str(e)}") except Exception as e: print(f"An unexpected error occurred: {str(e)}")