""" Cloud Docs Scraper ------------------ Crawls the docs site at BASE_URL and saves each page as a clean .txt file for RAG ingestion. Usage: pip install httpx beautifulsoup4 python scraper.py Output: ./docs/ - one .txt file per page ./docs/index.json - metadata (url, title, filename) for each page """ import json import re import time from pathlib import Path from urllib.parse import urljoin, urlparse import httpx from bs4 import BeautifulSoup BASE_URL = "https://aerospike.com/docs/cloud/" DOCS_DIR = Path("docs") HEADERS = { "User-Agent": "Mozilla/5.0 (compatible; portfolio-demo-scraper/1.0; +https://portfolio.meeses.com)" } DELAY_SECONDS = 1.0 # be polite def slugify(url: str) -> str: """Turn a URL path into a safe filename.""" path = urlparse(url).path.strip("/") slug = re.sub(r"[^a-z0-9]+", "-", path.lower()).strip("-") return slug or "index" def extract_content(soup: BeautifulSoup, url: str) -> dict: """Pull title and main content text from a page.""" # Title title_tag = soup.find("h1") title = title_tag.get_text(strip=True) if title_tag else urlparse(url).path # Main content — docs pages use
or a main content div content_el = ( soup.find("article") or soup.find("main") or soup.find("div", class_=re.compile(r"content|markdown|prose", re.I)) ) if not content_el: content_el = soup.body # Remove noisy elements for tag in content_el.find_all( ["nav", "footer", "header", "aside", "form", "script", "style"] ): tag.decompose() # Remove feedback widget and nav sidebars by class hints for tag in content_el.find_all( class_=re.compile(r"feedback|sidebar|toc|breadcrumb|cookie", re.I) ): tag.decompose() # Clean text lines = [] for el in content_el.find_all(["h1", "h2", "h3", "h4", "p", "li", "td", "th", "code", "pre"]): text = el.get_text(separator=" ", strip=True) if text: lines.append(text) body = "\n\n".join(lines) return {"title": title, "body": body, "url": url} def discover_urls(client: httpx.Client) -> list[str]: """Fetch the index page and collect all /docs/cloud/* links.""" resp = client.get(BASE_URL) resp.raise_for_status() soup = BeautifulSoup(resp.text, "html.parser") urls = set() for a in soup.find_all("a", href=True): href = urljoin(BASE_URL, a["href"]) parsed = urlparse(href) # Keep only paths under /docs/cloud/, on the same host, no anchors if ( parsed.netloc == "aerospike.com" and parsed.path.startswith("/docs/cloud/") and not parsed.fragment ): # Normalize: drop query strings clean = parsed._replace(query="", fragment="").geturl() urls.add(clean) # Always include the base URL itself urls.add(BASE_URL) return sorted(urls) def scrape(): DOCS_DIR.mkdir(exist_ok=True) index = [] with httpx.Client(headers=HEADERS, follow_redirects=True, timeout=15) as client: print("Discovering URLs...") urls = discover_urls(client) print(f"Found {len(urls)} pages under /docs/cloud/\n") for i, url in enumerate(urls, 1): print(f"[{i}/{len(urls)}] {url}") try: resp = client.get(url) resp.raise_for_status() except httpx.HTTPError as e: print(f" !! SKIP — {e}") continue soup = BeautifulSoup(resp.text, "html.parser") page = extract_content(soup, url) if len(page["body"].strip()) < 100: print(f" -- SKIP — too little content") continue filename = slugify(url) + ".txt" filepath = DOCS_DIR / filename # Write: title + url header, then body with open(filepath, "w", encoding="utf-8") as f: f.write(f"TITLE: {page['title']}\n") f.write(f"URL: {page['url']}\n") f.write(f"{'=' * 60}\n\n") f.write(page["body"]) index.append({ "title": page["title"], "url": page["url"], "filename": filename, "char_count": len(page["body"]), }) time.sleep(DELAY_SECONDS) # Write index index_path = DOCS_DIR / "index.json" with open(index_path, "w", encoding="utf-8") as f: json.dump(index, f, indent=2) print(f"\nDone. {len(index)} pages saved to ./{DOCS_DIR}/") print(f"Index written to {index_path}") if __name__ == "__main__": scrape()