Spaces:
Running
Running
Download backend/src/scraper.py from tomeese/support-ticket-analyzer: direct link, hf CLI and curl.
- Browser
- Download file 4.73 kB
-
https://huggingface.co/spaces/tomeese/support-ticket-analyzer/resolve/main/backend/src/scraper.py
- Command line
-
hf download hf://spaces/tomeese/support-ticket-analyzer/backend/src/scraper.py
-
curl -L -o scraper.py https://huggingface.co/spaces/tomeese/support-ticket-analyzer/resolve/main/backend/src/scraper.py
4.73 kB
| """ | |
| Cloud Docs Scraper | |
| ------------------ | |
| Crawls the docs site at BASE_URL and saves each page as a | |
| clean .txt file for RAG ingestion. | |
| Usage: | |
| pip install httpx beautifulsoup4 | |
| python scraper.py | |
| Output: | |
| ./docs/ - one .txt file per page | |
| ./docs/index.json - metadata (url, title, filename) for each page | |
| """ | |
| import json | |
| import re | |
| import time | |
| from pathlib import Path | |
| from urllib.parse import urljoin, urlparse | |
| import httpx | |
| from bs4 import BeautifulSoup | |
| BASE_URL = "https://aerospike.com/docs/cloud/" | |
| DOCS_DIR = Path("docs") | |
| HEADERS = { | |
| "User-Agent": "Mozilla/5.0 (compatible; portfolio-demo-scraper/1.0; +https://portfolio.meeses.com)" | |
| } | |
| DELAY_SECONDS = 1.0 # be polite | |
| def slugify(url: str) -> str: | |
| """Turn a URL path into a safe filename.""" | |
| path = urlparse(url).path.strip("/") | |
| slug = re.sub(r"[^a-z0-9]+", "-", path.lower()).strip("-") | |
| return slug or "index" | |
| def extract_content(soup: BeautifulSoup, url: str) -> dict: | |
| """Pull title and main content text from a page.""" | |
| # Title | |
| title_tag = soup.find("h1") | |
| title = title_tag.get_text(strip=True) if title_tag else urlparse(url).path | |
| # Main content — docs pages use <article> or a main content div | |
| content_el = ( | |
| soup.find("article") | |
| or soup.find("main") | |
| or soup.find("div", class_=re.compile(r"content|markdown|prose", re.I)) | |
| ) | |
| if not content_el: | |
| content_el = soup.body | |
| # Remove noisy elements | |
| for tag in content_el.find_all( | |
| ["nav", "footer", "header", "aside", "form", "script", "style"] | |
| ): | |
| tag.decompose() | |
| # Remove feedback widget and nav sidebars by class hints | |
| for tag in content_el.find_all( | |
| class_=re.compile(r"feedback|sidebar|toc|breadcrumb|cookie", re.I) | |
| ): | |
| tag.decompose() | |
| # Clean text | |
| lines = [] | |
| for el in content_el.find_all(["h1", "h2", "h3", "h4", "p", "li", "td", "th", "code", "pre"]): | |
| text = el.get_text(separator=" ", strip=True) | |
| if text: | |
| lines.append(text) | |
| body = "\n\n".join(lines) | |
| return {"title": title, "body": body, "url": url} | |
| def discover_urls(client: httpx.Client) -> list[str]: | |
| """Fetch the index page and collect all /docs/cloud/* links.""" | |
| resp = client.get(BASE_URL) | |
| resp.raise_for_status() | |
| soup = BeautifulSoup(resp.text, "html.parser") | |
| urls = set() | |
| for a in soup.find_all("a", href=True): | |
| href = urljoin(BASE_URL, a["href"]) | |
| parsed = urlparse(href) | |
| # Keep only paths under /docs/cloud/, on the same host, no anchors | |
| if ( | |
| parsed.netloc == "aerospike.com" | |
| and parsed.path.startswith("/docs/cloud/") | |
| and not parsed.fragment | |
| ): | |
| # Normalize: drop query strings | |
| clean = parsed._replace(query="", fragment="").geturl() | |
| urls.add(clean) | |
| # Always include the base URL itself | |
| urls.add(BASE_URL) | |
| return sorted(urls) | |
| def scrape(): | |
| DOCS_DIR.mkdir(exist_ok=True) | |
| index = [] | |
| with httpx.Client(headers=HEADERS, follow_redirects=True, timeout=15) as client: | |
| print("Discovering URLs...") | |
| urls = discover_urls(client) | |
| print(f"Found {len(urls)} pages under /docs/cloud/\n") | |
| for i, url in enumerate(urls, 1): | |
| print(f"[{i}/{len(urls)}] {url}") | |
| try: | |
| resp = client.get(url) | |
| resp.raise_for_status() | |
| except httpx.HTTPError as e: | |
| print(f" !! SKIP — {e}") | |
| continue | |
| soup = BeautifulSoup(resp.text, "html.parser") | |
| page = extract_content(soup, url) | |
| if len(page["body"].strip()) < 100: | |
| print(f" -- SKIP — too little content") | |
| continue | |
| filename = slugify(url) + ".txt" | |
| filepath = DOCS_DIR / filename | |
| # Write: title + url header, then body | |
| with open(filepath, "w", encoding="utf-8") as f: | |
| f.write(f"TITLE: {page['title']}\n") | |
| f.write(f"URL: {page['url']}\n") | |
| f.write(f"{'=' * 60}\n\n") | |
| f.write(page["body"]) | |
| index.append({ | |
| "title": page["title"], | |
| "url": page["url"], | |
| "filename": filename, | |
| "char_count": len(page["body"]), | |
| }) | |
| time.sleep(DELAY_SECONDS) | |
| # Write index | |
| index_path = DOCS_DIR / "index.json" | |
| with open(index_path, "w", encoding="utf-8") as f: | |
| json.dump(index, f, indent=2) | |
| print(f"\nDone. {len(index)} pages saved to ./{DOCS_DIR}/") | |
| print(f"Index written to {index_path}") | |
| if __name__ == "__main__": | |
| scrape() |