File size: 4,579 Bytes
63fa23b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import requests
from bs4 import BeautifulSoup
import json

# Define the URL for the courses list
courses_url = "https://courses.analyticsvidhya.com/pages/all-free-courses"

# Initialize a list to store all course dictionaries
all_courses = []

try:
    # Send a GET request to fetch the page content
    response = requests.get(courses_url)
    
    # Check if the response is successful
    if response.status_code == 200:
        print("Successfully fetched the courses page.")
        
        # Parse the main courses page
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # Find all course links
        course_links = soup.find_all('a', class_='course-card course-card__public published')
        
        if not course_links:
            print("No course links found on the page.")
        
        for course_link in course_links:
            # Initialize dictionary for current course
            course_data = {
                'title': '',
                'url': '',
                'description': '',
                'topics': [],
                'curriculum': ''
            }
            
            # Extract the href URL
            course_url = course_link.get('href')
            if not course_url:
                print("No URL found for this course")
                continue
            
            # If the URL is relative, make it absolute
            if course_url.startswith('/'):
                course_url = f"https://courses.analyticsvidhya.com{course_url}"
            
            course_data['url'] = course_url
            print(f"Processing course URL: {course_url}")
            
            try:
                # Send a GET request to fetch the course detail page content
                course_response = requests.get(course_url)
                
                if course_response.status_code == 200:
                    course_soup = BeautifulSoup(course_response.content, 'html.parser')
                    
                    # Extract course title
                    title_tag = course_soup.find('h1')
                    course_data['title'] = title_tag.text.strip() if title_tag else "N/A"
                    
                    # Extract course description
                    description_section = course_soup.find('div', class_='rich-text__wrapper')
                    course_data['description'] = description_section.get_text(strip=True) if description_section else "No description available."
                    
                    # Extract topics (if available)
                    topics_section = course_soup.find('div', class_='course-topics__container')
                    if topics_section:
                        topics = topics_section.find_all('div', class_='topic-card')
                        course_data['topics'] = [topic.get_text(strip=True) for topic in topics]
                    
                    # Extract curriculum structure
                    curriculum_section = course_soup.find('div', class_='course-curriculum__container')
                    course_data['curriculum'] = curriculum_section.get_text(strip=True) if curriculum_section else "No curriculum available."
                    
                    # Add the course data to our list of all courses
                    all_courses.append(course_data)
                    print(f"Successfully processed course: {course_data['title']}")
                    
                else:
                    print(f"Failed to retrieve the details from {course_url}")
                    
            except requests.RequestException as e:
                print(f"Error processing course URL {course_url}: {str(e)}")
        
        # Save all courses data to a JSON file
        with open('all_courses.json', 'w', encoding='utf-8') as f:
            json.dump(all_courses, f, indent=4, ensure_ascii=False)
        print(f"\nSuccessfully saved data for {len(all_courses)} courses to all_courses.json")
        
        # Print example of the first course data (if available)
        if all_courses:
            print("\nExample of first course data structure:")
            print(json.dumps(all_courses[0], indent=2))
            
    else:
        print(f"Failed to retrieve the courses page. Status code: {response.status_code}")
        
except requests.RequestException as e:
    print(f"Error fetching the courses page: {str(e)}")
except Exception as e:
    print(f"An unexpected error occurred: {str(e)}")