tomeese's picture
Claude Opus 5.5
Rebrand as Halyard Cloud ticket analyzer; add triage, replies and personas
d0954af
Raw History Blame Contribute Delete
4.73 kB
"""
Cloud Docs Scraper
------------------
Crawls the docs site at BASE_URL and saves each page as a
clean .txt file for RAG ingestion.
Usage:
pip install httpx beautifulsoup4
python scraper.py
Output:
./docs/ - one .txt file per page
./docs/index.json - metadata (url, title, filename) for each page
"""
import json
import re
import time
from pathlib import Path
from urllib.parse import urljoin, urlparse
import httpx
from bs4 import BeautifulSoup
BASE_URL = "https://aerospike.com/docs/cloud/"
DOCS_DIR = Path("docs")
HEADERS = {
"User-Agent": "Mozilla/5.0 (compatible; portfolio-demo-scraper/1.0; +https://portfolio.meeses.com)"
}
DELAY_SECONDS = 1.0 # be polite
def slugify(url: str) -> str:
"""Turn a URL path into a safe filename."""
path = urlparse(url).path.strip("/")
slug = re.sub(r"[^a-z0-9]+", "-", path.lower()).strip("-")
return slug or "index"
def extract_content(soup: BeautifulSoup, url: str) -> dict:
"""Pull title and main content text from a page."""
# Title
title_tag = soup.find("h1")
title = title_tag.get_text(strip=True) if title_tag else urlparse(url).path
# Main content — docs pages use <article> or a main content div
content_el = (
soup.find("article")
or soup.find("main")
or soup.find("div", class_=re.compile(r"content|markdown|prose", re.I))
)
if not content_el:
content_el = soup.body
# Remove noisy elements
for tag in content_el.find_all(
["nav", "footer", "header", "aside", "form", "script", "style"]
):
tag.decompose()
# Remove feedback widget and nav sidebars by class hints
for tag in content_el.find_all(
class_=re.compile(r"feedback|sidebar|toc|breadcrumb|cookie", re.I)
):
tag.decompose()
# Clean text
lines = []
for el in content_el.find_all(["h1", "h2", "h3", "h4", "p", "li", "td", "th", "code", "pre"]):
text = el.get_text(separator=" ", strip=True)
if text:
lines.append(text)
body = "\n\n".join(lines)
return {"title": title, "body": body, "url": url}
def discover_urls(client: httpx.Client) -> list[str]:
"""Fetch the index page and collect all /docs/cloud/* links."""
resp = client.get(BASE_URL)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, "html.parser")
urls = set()
for a in soup.find_all("a", href=True):
href = urljoin(BASE_URL, a["href"])
parsed = urlparse(href)
# Keep only paths under /docs/cloud/, on the same host, no anchors
if (
parsed.netloc == "aerospike.com"
and parsed.path.startswith("/docs/cloud/")
and not parsed.fragment
):
# Normalize: drop query strings
clean = parsed._replace(query="", fragment="").geturl()
urls.add(clean)
# Always include the base URL itself
urls.add(BASE_URL)
return sorted(urls)
def scrape():
DOCS_DIR.mkdir(exist_ok=True)
index = []
with httpx.Client(headers=HEADERS, follow_redirects=True, timeout=15) as client:
print("Discovering URLs...")
urls = discover_urls(client)
print(f"Found {len(urls)} pages under /docs/cloud/\n")
for i, url in enumerate(urls, 1):
print(f"[{i}/{len(urls)}] {url}")
try:
resp = client.get(url)
resp.raise_for_status()
except httpx.HTTPError as e:
print(f" !! SKIP — {e}")
continue
soup = BeautifulSoup(resp.text, "html.parser")
page = extract_content(soup, url)
if len(page["body"].strip()) < 100:
print(f" -- SKIP — too little content")
continue
filename = slugify(url) + ".txt"
filepath = DOCS_DIR / filename
# Write: title + url header, then body
with open(filepath, "w", encoding="utf-8") as f:
f.write(f"TITLE: {page['title']}\n")
f.write(f"URL: {page['url']}\n")
f.write(f"{'=' * 60}\n\n")
f.write(page["body"])
index.append({
"title": page["title"],
"url": page["url"],
"filename": filename,
"char_count": len(page["body"]),
})
time.sleep(DELAY_SECONDS)
# Write index
index_path = DOCS_DIR / "index.json"
with open(index_path, "w", encoding="utf-8") as f:
json.dump(index, f, indent=2)
print(f"\nDone. {len(index)} pages saved to ./{DOCS_DIR}/")
print(f"Index written to {index_path}")
if __name__ == "__main__":
scrape()