{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "4AuMIyZkmokI"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import json"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import json\n",
        "\n",
        "base_url = \"https://courses.analyticsvidhya.com/collections\"\n",
        "courses = []\n",
        "page = 1\n",
        "\n",
        "while True:\n",
        "    url = f\"{base_url}?page={page}\" if page > 1 else base_url\n",
        "    response = requests.get(url)\n",
        "\n",
        "    if response.status_code != 200:\n",
        "        print(f\"Stopped at page {page}, no more pages or error.\")\n",
        "        break\n",
        "\n",
        "    soup = BeautifulSoup(response.content, 'html.parser')\n",
        "    course_cards = soup.find_all(\"div\", class_=\"course-card__body\")\n",
        "    if not course_cards:\n",
        "        print(f\"Stopped at page {page}, no course cards found.\")\n",
        "        break\n",
        "\n",
        "    for course_card in course_cards:\n",
        "        title_tag = course_card.find(\"h3\")\n",
        "        title = title_tag.get_text(strip=True) if title_tag else \"Title not found\"\n",
        "\n",
        "        footer_tag = course_card.find_next_sibling(\"footer\")\n",
        "        lesson_tag = footer_tag.find(\"span\", class_=\"course-card__lesson-count\") if footer_tag else None\n",
        "        lesson_count = lesson_tag.get_text(strip=True) if lesson_tag else \"Lesson count not found\"\n",
        "\n",
        "        courses.append({\n",
        "            \"title\": title,\n",
        "            \"lessons\": lesson_count\n",
        "        })\n",
        "\n",
        "    print(f\"Scraped page {page} successfully.\")\n",
        "    page += 1\n",
        "with open(\"free_courses.json\", \"w\", encoding=\"utf-8\") as file:\n",
        "    json.dump(courses, file, ensure_ascii=False, indent=4)\n",
        "\n",
        "print(\"Scraped data saved to free_courses.json\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bZsP9ZRbuDBF",
        "outputId": "6be5b9b3-5348-4f95-bd4d-f80ab48bf3b1"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Scraped page 1 successfully.\n",
            "Scraped page 2 successfully.\n",
            "Scraped page 3 successfully.\n",
            "Scraped page 4 successfully.\n",
            "Scraped page 5 successfully.\n",
            "Scraped page 6 successfully.\n",
            "Scraped page 7 successfully.\n",
            "Scraped page 8 successfully.\n",
            "Scraped page 9 successfully.\n",
            "Stopped at page 10, no course cards found.\n",
            "Scraped data saved to free_courses.json\n"
          ]
        }
      ]
    }
  ]
}