Spaces:
Build error
Build error
Download dataExtraction.py from mayur1811/AnalyticsVidhyaCourseSearchAssistant: direct link, hf CLI and curl.
- Browser
- Download file 4.58 kB
-
https://huggingface.co/spaces/mayur1811/AnalyticsVidhyaCourseSearchAssistant/resolve/main/dataExtraction.py
- Command line
-
hf download hf://spaces/mayur1811/AnalyticsVidhyaCourseSearchAssistant/dataExtraction.py
-
curl -L -o dataExtraction.py https://huggingface.co/spaces/mayur1811/AnalyticsVidhyaCourseSearchAssistant/resolve/main/dataExtraction.py
4.58 kB
| import requests | |
| from bs4 import BeautifulSoup | |
| import json | |
| # Define the URL for the courses list | |
| courses_url = "https://courses.analyticsvidhya.com/pages/all-free-courses" | |
| # Initialize a list to store all course dictionaries | |
| all_courses = [] | |
| try: | |
| # Send a GET request to fetch the page content | |
| response = requests.get(courses_url) | |
| # Check if the response is successful | |
| if response.status_code == 200: | |
| print("Successfully fetched the courses page.") | |
| # Parse the main courses page | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| # Find all course links | |
| course_links = soup.find_all('a', class_='course-card course-card__public published') | |
| if not course_links: | |
| print("No course links found on the page.") | |
| for course_link in course_links: | |
| # Initialize dictionary for current course | |
| course_data = { | |
| 'title': '', | |
| 'url': '', | |
| 'description': '', | |
| 'topics': [], | |
| 'curriculum': '' | |
| } | |
| # Extract the href URL | |
| course_url = course_link.get('href') | |
| if not course_url: | |
| print("No URL found for this course") | |
| continue | |
| # If the URL is relative, make it absolute | |
| if course_url.startswith('/'): | |
| course_url = f"https://courses.analyticsvidhya.com{course_url}" | |
| course_data['url'] = course_url | |
| print(f"Processing course URL: {course_url}") | |
| try: | |
| # Send a GET request to fetch the course detail page content | |
| course_response = requests.get(course_url) | |
| if course_response.status_code == 200: | |
| course_soup = BeautifulSoup(course_response.content, 'html.parser') | |
| # Extract course title | |
| title_tag = course_soup.find('h1') | |
| course_data['title'] = title_tag.text.strip() if title_tag else "N/A" | |
| # Extract course description | |
| description_section = course_soup.find('div', class_='rich-text__wrapper') | |
| course_data['description'] = description_section.get_text(strip=True) if description_section else "No description available." | |
| # Extract topics (if available) | |
| topics_section = course_soup.find('div', class_='course-topics__container') | |
| if topics_section: | |
| topics = topics_section.find_all('div', class_='topic-card') | |
| course_data['topics'] = [topic.get_text(strip=True) for topic in topics] | |
| # Extract curriculum structure | |
| curriculum_section = course_soup.find('div', class_='course-curriculum__container') | |
| course_data['curriculum'] = curriculum_section.get_text(strip=True) if curriculum_section else "No curriculum available." | |
| # Add the course data to our list of all courses | |
| all_courses.append(course_data) | |
| print(f"Successfully processed course: {course_data['title']}") | |
| else: | |
| print(f"Failed to retrieve the details from {course_url}") | |
| except requests.RequestException as e: | |
| print(f"Error processing course URL {course_url}: {str(e)}") | |
| # Save all courses data to a JSON file | |
| with open('all_courses.json', 'w', encoding='utf-8') as f: | |
| json.dump(all_courses, f, indent=4, ensure_ascii=False) | |
| print(f"\nSuccessfully saved data for {len(all_courses)} courses to all_courses.json") | |
| # Print example of the first course data (if available) | |
| if all_courses: | |
| print("\nExample of first course data structure:") | |
| print(json.dumps(all_courses[0], indent=2)) | |
| else: | |
| print(f"Failed to retrieve the courses page. Status code: {response.status_code}") | |
| except requests.RequestException as e: | |
| print(f"Error fetching the courses page: {str(e)}") | |
| except Exception as e: | |
| print(f"An unexpected error occurred: {str(e)}") |