{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Web Scrape Oura Ring App Content" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from selenium import webdriver\n", "from selenium.webdriver.chrome.options import Options\n", "from selenium.webdriver.common.by import By\n", "import time\n", "import re\n", "\n", "# Setup Chrome options\n", "chrome_options = Options()\n", "chrome_options.add_argument(\"--headless\")\n", "chrome_options.add_argument(\"--disable-gpu\")\n", "chrome_options.add_argument(\"--no-sandbox\")\n", "chrome_options.add_argument(\"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36\")\n", "\n", "# Initialize the WebDriver\n", "driver = webdriver.Chrome(options=chrome_options)\n", "\n", "# Open the main page containing the list of articles\n", "url = \"https://support.ouraring.com/hc/en-us/categories/27782541623059-Oura-App\"\n", "driver.get(url)\n", "time.sleep(3)\n", "\n", "# section_links = [link.get_attribute(\"href\") for link in driver.find_elements(By.XPATH, \"//h3[contains(@class, 'section-tree-title')]/a\")]\n", "section_links = ['https://support.ouraring.com/hc/en-us/sections/9721296007827-App-Settings',\n", " 'https://support.ouraring.com/hc/en-us/sections/9721508785171-App-Integrations',\n", " 'https://support.ouraring.com/hc/en-us/sections/4415723882003-Readiness',\n", " 'https://support.ouraring.com/hc/en-us/sections/4415717595155-Sleep',\n", " 'https://support.ouraring.com/hc/en-us/sections/4415723875347-Activity',\n", " 'https://support.ouraring.com/hc/en-us/sections/4415730373779-Insights-Reports',\n", " 'https://support.ouraring.com/hc/en-us/sections/28332751996435-Heart-Health',\n", " 'https://support.ouraring.com/hc/en-us/sections/17984010150803-Women-s-Health',\n", " 'https://support.ouraring.com/hc/en-us/sections/4415723874451-Mindfulness-Meditation']\n", "\n", "articles_data = []\n", "\n", "# Function to split large sections into smaller RAG-friendly chunks\n", "def chunk_text(text, chunk_size=512):\n", " \n", " # Split text into sentences and list items\n", " list_item_pattern = re.compile(r\"^(\\d+\\.\\s+|- )\") \n", " lines = text.split(\"\\n\") \n", "\n", " chunks = []\n", " current_chunk = \"\"\n", "\n", " for line in lines:\n", " line = line.strip()\n", " if not line:\n", " continue \n", " \n", " # Check if this line starts a new list item\n", " is_list_item = bool(list_item_pattern.match(line))\n", " \n", " if is_list_item and current_chunk and len(current_chunk) + len(line) >= chunk_size:\n", " chunks.append(current_chunk.strip())\n", " current_chunk = line \n", " elif len(current_chunk) + len(line) >= chunk_size:\n", " chunks.append(current_chunk.strip())\n", " current_chunk = line \n", " else:\n", " current_chunk += \"\\n\" + line \n", " \n", " if current_chunk:\n", " chunks.append(current_chunk.strip()) \n", "\n", " return chunks\n", "\n", "\n", "def extractSectionChunks(driver):\n", " # Extract article title\n", " title_element = driver.find_element(By.XPATH, \"//h1[contains(@class, 'article-title')]\")\n", " article_title = title_element.text if title_element else \"No Title Found\"\n", "\n", " # Extract all content divs\n", " content_elements = driver.find_elements(By.XPATH, \"//div[contains(@class, 'article-body')]/*\")\n", "\n", " chunks = []\n", " intro_content = []\n", " current_section = None\n", " sections = []\n", " chunk_id = 1 \n", "\n", " for element in content_elements:\n", " tag_name = element.tag_name.lower()\n", " text = element.text.strip()\n", "\n", " if not text:\n", " continue \n", "\n", " # If a header is encountered, start a new section\n", " if tag_name in [\"h2\"]:\n", " # Store previous section\n", " if current_section:\n", " sections.append(current_section)\n", " \n", " # Start a new section\n", " current_section = {\"section_title\": text, \"content\": []}\n", " elif tag_name == \"ol\":\n", " # Handle ordered (numbered) lists\n", " list_items = element.find_elements(By.TAG_NAME, \"li\")\n", " list_text = \"\\n\".join([f\"{idx+1}. {li.text.strip()}\" for idx, li in enumerate(list_items) if li.text.strip()])\n", " if current_section:\n", " current_section[\"content\"].append(list_text)\n", " else:\n", " intro_content.append(list_text)\n", " elif tag_name == \"ul\":\n", " # Handle unordered (bulleted) lists\n", " list_items = element.find_elements(By.TAG_NAME, \"li\")\n", " list_text = \"\\n\".join([f\"- {li.text.strip()}\" for li in list_items if li.text.strip()])\n", " if current_section:\n", " current_section[\"content\"].append(list_text)\n", " else:\n", " intro_content.append(list_text)\n", " else:\n", " if current_section:\n", " current_section[\"content\"].append(text)\n", " else:\n", " # This is part of the introduction\n", " intro_content.append(text)\n", "\n", " # Store last section\n", " if current_section:\n", " sections.append(current_section)\n", "\n", " # Store introduction **FIRST**\n", " if intro_content:\n", " intro_chunks = chunk_text(\"\\n\".join(intro_content))\n", " for chunk in intro_chunks:\n", " chunks.append({\n", " \"article_title\": article_title,\n", " \"section_title\": \"Introduction\",\n", " \"chunk\": chunk,\n", " \"chunk_id\": chunk_id, \n", " \"link\": article_link\n", " })\n", " chunk_id += 1 \n", "\n", " # Store all sections\n", " for section in sections:\n", " section_chunks = chunk_text(\"\\n\".join(section[\"content\"]))\n", " for chunk in section_chunks:\n", " chunks.append({\n", " \"article_title\": article_title,\n", " \"section_title\": section[\"section_title\"],\n", " \"chunk\": chunk,\n", " \"chunk_id\": chunk_id, \n", " \"link\": article_link\n", " })\n", " chunk_id += 1 \n", " return article_title, chunks\n", "\n", "for section_link in section_links:\n", " driver.get(section_link)\n", " time.sleep(3)\n", "\n", " article_links = [link.get_attribute(\"href\") for link in driver.find_elements(By.XPATH, \"//li[contains(@class, 'article-list-item')]/a\")]\n", "\n", " for article_link in article_links:\n", " driver.get(article_link)\n", " time.sleep(3)\n", "\n", " try:\n", " article_title,chunks = extractSectionChunks(driver)\n", " articles_data.extend(chunks)\n", " print(f\"Extracted: {article_title} - {len(chunks)} chunks\")\n", " except Exception as e:\n", " try:\n", " driver.get(article_link)\n", " sub_article_links = [link.get_attribute(\"href\") for link in driver.find_elements(By.XPATH, \"//li[contains(@class, 'article-list-item')]/a\")]\n", " for sub_article_link in sub_article_links:\n", " driver.get(sub_article_link)\n", " time.sleep(3)\n", " article_title,chunks = extractSectionChunks(driver)\n", " articles_data.extend(chunks)\n", " print(f\"Extracted: {article_title} - {len(chunks)} chunks\")\n", " except Exception as e:\n", " print(f\"Failed to extract content from {article_link}: {str(e)}\")\n", "\n", "\n", "driver.quit()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Save Chunks" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "# Save extracted articles to JSON\n", "with open(\"oura_article_data_chunked_final.json\", \"w\", encoding=\"utf-8\") as f:\n", " json.dump(articles_data, f, indent=4, ensure_ascii=False)\n", "\n", "driver.quit()\n", "\n", "print(\"Scraping complete. Data saved to 'oura_articles.json'\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "ouraDemo", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.16" } }, "nbformat": 4, "nbformat_minor": 2 }