mayur1811 commited on
Commit
63fa23b
·
verified ·
1 Parent(s): f56fa42

Upload 2 files

Browse files
Files changed (2) hide show
  1. dataExtraction.py +102 -0
  2. requirements.txt +7 -0
dataExtraction.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ from bs4 import BeautifulSoup
3
+ import json
4
+
5
+ # Define the URL for the courses list
6
+ courses_url = "https://courses.analyticsvidhya.com/pages/all-free-courses"
7
+
8
+ # Initialize a list to store all course dictionaries
9
+ all_courses = []
10
+
11
+ try:
12
+ # Send a GET request to fetch the page content
13
+ response = requests.get(courses_url)
14
+
15
+ # Check if the response is successful
16
+ if response.status_code == 200:
17
+ print("Successfully fetched the courses page.")
18
+
19
+ # Parse the main courses page
20
+ soup = BeautifulSoup(response.content, 'html.parser')
21
+
22
+ # Find all course links
23
+ course_links = soup.find_all('a', class_='course-card course-card__public published')
24
+
25
+ if not course_links:
26
+ print("No course links found on the page.")
27
+
28
+ for course_link in course_links:
29
+ # Initialize dictionary for current course
30
+ course_data = {
31
+ 'title': '',
32
+ 'url': '',
33
+ 'description': '',
34
+ 'topics': [],
35
+ 'curriculum': ''
36
+ }
37
+
38
+ # Extract the href URL
39
+ course_url = course_link.get('href')
40
+ if not course_url:
41
+ print("No URL found for this course")
42
+ continue
43
+
44
+ # If the URL is relative, make it absolute
45
+ if course_url.startswith('/'):
46
+ course_url = f"https://courses.analyticsvidhya.com{course_url}"
47
+
48
+ course_data['url'] = course_url
49
+ print(f"Processing course URL: {course_url}")
50
+
51
+ try:
52
+ # Send a GET request to fetch the course detail page content
53
+ course_response = requests.get(course_url)
54
+
55
+ if course_response.status_code == 200:
56
+ course_soup = BeautifulSoup(course_response.content, 'html.parser')
57
+
58
+ # Extract course title
59
+ title_tag = course_soup.find('h1')
60
+ course_data['title'] = title_tag.text.strip() if title_tag else "N/A"
61
+
62
+ # Extract course description
63
+ description_section = course_soup.find('div', class_='rich-text__wrapper')
64
+ course_data['description'] = description_section.get_text(strip=True) if description_section else "No description available."
65
+
66
+ # Extract topics (if available)
67
+ topics_section = course_soup.find('div', class_='course-topics__container')
68
+ if topics_section:
69
+ topics = topics_section.find_all('div', class_='topic-card')
70
+ course_data['topics'] = [topic.get_text(strip=True) for topic in topics]
71
+
72
+ # Extract curriculum structure
73
+ curriculum_section = course_soup.find('div', class_='course-curriculum__container')
74
+ course_data['curriculum'] = curriculum_section.get_text(strip=True) if curriculum_section else "No curriculum available."
75
+
76
+ # Add the course data to our list of all courses
77
+ all_courses.append(course_data)
78
+ print(f"Successfully processed course: {course_data['title']}")
79
+
80
+ else:
81
+ print(f"Failed to retrieve the details from {course_url}")
82
+
83
+ except requests.RequestException as e:
84
+ print(f"Error processing course URL {course_url}: {str(e)}")
85
+
86
+ # Save all courses data to a JSON file
87
+ with open('all_courses.json', 'w', encoding='utf-8') as f:
88
+ json.dump(all_courses, f, indent=4, ensure_ascii=False)
89
+ print(f"\nSuccessfully saved data for {len(all_courses)} courses to all_courses.json")
90
+
91
+ # Print example of the first course data (if available)
92
+ if all_courses:
93
+ print("\nExample of first course data structure:")
94
+ print(json.dumps(all_courses[0], indent=2))
95
+
96
+ else:
97
+ print(f"Failed to retrieve the courses page. Status code: {response.status_code}")
98
+
99
+ except requests.RequestException as e:
100
+ print(f"Error fetching the courses page: {str(e)}")
101
+ except Exception as e:
102
+ print(f"An unexpected error occurred: {str(e)}")
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ streamlit==1.31.0
2
+ google-generativeai==0.3.2
3
+ beautifulsoup4==4.12.3
4
+ requests==2.31.0
5
+ typing-extensions==4.9.0
6
+ python-dotenv==1.0.1
7
+ protobuf==4.25.2