Spaces:
Running
Running
Download tool.py from bugrahans/whitepages-scraper: direct link, hf CLI and curl.
- Browser
- Download file 17.5 kB
-
https://huggingface.co/spaces/bugrahans/whitepages-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/whitepages-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/whitepages-scraper/resolve/main/tool.py
17.5 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| from dataclasses import asdict, dataclass, field | |
| from typing import Any, Dict, Iterable, List, Optional | |
| from urllib.parse import quote | |
| import requests | |
| from bs4 import BeautifulSoup | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=whitepages-scraper" | |
| API_URL = "https://api.scrape.do/" | |
| # Whitepages is US-only and sits behind Cloudflare. Every page goes through a US | |
| # residential IP (super=true, geoCode=us): 10 Scrape.do credits. | |
| CREDITS_PER_REQUEST = 10 | |
| DEFAULT_NAME = "John Smith" | |
| DEFAULT_CITY = "Seattle" | |
| DEFAULT_STATE = "WA" | |
| US_STATES = [ | |
| "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY", | |
| "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", | |
| "OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY", | |
| ] | |
| BLOCK_RE = re.compile(r"just a moment\.\.\.|cf-chl-|attention required|cf-turnstile", re.I) | |
| # ------------------------------------------------------------------ parsing | |
| # (same parser as `pip install whitepages-scraper`) | |
| BASE_URL = "https://www.whitepages.com" | |
| PROFILE_RE = re.compile(r"^(?:https?://(?:www\.)?whitepages\.com)?/name/[^/?#]+/[^/?#]+/[A-Za-z0-9]{8,14}/?(?:[?#].*)?$") | |
| AGE_TEXT_RE = re.compile(r"\bin (?:their|his|her) (\d{2}s|\d{2}\+)", re.I) | |
| AGE_RANGE_RE = re.compile(r"^\d{2}(?:s|\+)$") | |
| CITY_LINE_RE = re.compile(r"^\s*(.+?),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?\s*$") | |
| class Person: | |
| """One person from a Whitepages search result or person page.""" | |
| name: str = "" | |
| first_name: str = "" | |
| middle_name: str = "" | |
| last_name: str = "" | |
| age_range: str = "" | |
| address: str = "" | |
| city: str = "" | |
| state: str = "" | |
| zip_code: str = "" | |
| neighborhood: str = "" | |
| phones: List[str] = field(default_factory=list) | |
| aliases: List[str] = field(default_factory=list) | |
| relatives: List[str] = field(default_factory=list) | |
| associates: List[str] = field(default_factory=list) | |
| previous_locations: List[str] = field(default_factory=list) | |
| url: str = "" | |
| def phone(self) -> str: | |
| return self.phones[0] if self.phones else "" | |
| def profile_id(self) -> str: | |
| match = re.search(r"/([A-Za-z0-9]{8,14})/?$", self.url.split("?")[0]) | |
| return match.group(1) if match else "" | |
| def to_dict(self) -> Dict[str, Any]: | |
| return asdict(self) | |
| def to_row(self, separator: str = " | ") -> Dict[str, Any]: | |
| row = self.to_dict() | |
| for key in ("phones", "aliases", "relatives", "associates", "previous_locations"): | |
| row[key] = separator.join(row[key]) | |
| return row | |
| CSV_FIELDS = [ | |
| "name", "first_name", "middle_name", "last_name", "age_range", "address", "city", "state", "zip_code", | |
| "neighborhood", "phones", "aliases", "relatives", "associates", "previous_locations", "url", | |
| ] | |
| # --------------------------------------------------------------------------- helpers | |
| def _squash(value: Any) -> str: | |
| return " ".join(str(value).split()) if value is not None else "" | |
| def _as_list(value: Any) -> List[Any]: | |
| if value is None: | |
| return [] | |
| return value if isinstance(value, list) else [value] | |
| def _unique(values: Iterable[str]) -> List[str]: | |
| seen, out = set(), [] | |
| for value in values: | |
| value = _squash(value) | |
| if value and value not in seen: | |
| seen.add(value) | |
| out.append(value) | |
| return out | |
| def _names(values: Any) -> List[str]: | |
| return _unique(v.get("name", "") if isinstance(v, dict) else str(v) for v in _as_list(values)) | |
| def format_phone(value: str) -> str: | |
| """Normalize ``+1-206-555-0142`` and similar to ``(206) 555-0142``; masked numbers return "".""" | |
| value = _squash(value) | |
| if not value or "*" in value: | |
| return "" | |
| digits = re.sub(r"\D", "", value) | |
| if len(digits) == 11 and digits.startswith("1"): | |
| digits = digits[1:] | |
| if len(digits) != 10: | |
| return value | |
| return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}" | |
| def absolute_url(url: str) -> str: | |
| url = _squash(url) | |
| if url.startswith("/"): | |
| url = BASE_URL + url | |
| return url.split("#")[0] | |
| def _slug(text: str) -> str: | |
| words = re.findall(r"[A-Za-z0-9'.]+", text or "") | |
| return "-".join(w.replace("'", "").replace(".", "") for w in words if w.strip("'.")) | |
| def name_search_url(name: str, city: str = "", state: str = "", page: int = 1) -> str: | |
| """Build a name search URL: ``/name/John-Smith``, ``/name/John-Smith/WA`` or ``/name/John-Smith/Seattle-WA``. | |
| ``page`` adds ``?page=N`` (25 people per page). | |
| """ | |
| slug = _slug(name) | |
| if not slug or not re.search(r"[A-Za-z]", slug): | |
| raise ValueError(f"Not a name: {name!r}") | |
| state = (state or "").strip().upper() | |
| if state and not re.fullmatch(r"[A-Z]{2}", state): | |
| raise ValueError(f"State must be a two-letter code like WA, got {state!r}") | |
| if city and not state: | |
| raise ValueError("A city needs a state, e.g. city='Seattle', state='WA'") | |
| url = f"{BASE_URL}/name/{quote(slug)}" | |
| if city: | |
| url += f"/{quote(_slug(city))}-{state}" | |
| elif state: | |
| url += f"/{state}" | |
| if page and int(page) > 1: | |
| url += f"?page={int(page)}" | |
| return url | |
| def iter_jsonld(soup: BeautifulSoup) -> Iterable[Dict[str, Any]]: | |
| for script in soup.find_all("script", type="application/ld+json"): | |
| try: | |
| data = json.loads(script.string or "") | |
| except (TypeError, ValueError): | |
| continue | |
| for item in _as_list(data): | |
| if isinstance(item, dict): | |
| yield item | |
| def _apply_address(person: Person, address: Any) -> None: | |
| for addr in _as_list(address): | |
| if not isinstance(addr, dict): | |
| continue | |
| person.address = _squash(addr.get("streetAddress")) | |
| person.city = _squash(addr.get("addressLocality")) | |
| person.state = _squash(addr.get("addressRegion")) | |
| person.zip_code = _squash(addr.get("postalCode")) | |
| if person.address or person.city: | |
| return | |
| def _person_from_jsonld(item: Dict[str, Any]) -> Person: | |
| person = Person( | |
| name=_squash(item.get("name")), | |
| first_name=_squash(item.get("givenName")), | |
| middle_name=" ".join(_unique(_as_list(item.get("additionalName")))), | |
| last_name=_squash(item.get("familyName")), | |
| url=absolute_url(item.get("url") or item.get("@id") or ""), | |
| ) | |
| _apply_address(person, item.get("address")) | |
| phones = [format_phone(p) for p in _as_list(item.get("telephone"))] | |
| for cp in _as_list(item.get("contactPoint")): | |
| if isinstance(cp, dict): | |
| phones.append(format_phone(cp.get("telephone", ""))) | |
| person.phones = _unique(p for p in phones if p) | |
| person.aliases = [a for a in _unique(_as_list(item.get("alternateName"))) if a != person.name] | |
| person.relatives = _names(item.get("relatedTo")) | |
| person.associates = _names(item.get("knows")) | |
| for prop in _as_list(item.get("additionalProperty")): | |
| if not isinstance(prop, dict): | |
| continue | |
| key, value = _squash(prop.get("name")).lower(), _squash(prop.get("value")) | |
| if key == "age range" and AGE_RANGE_RE.match(value): | |
| person.age_range = value | |
| elif key == "neighborhood": | |
| person.neighborhood = value | |
| elif key == "previous location" and value: | |
| person.previous_locations.append(value) | |
| person.previous_locations = _unique(person.previous_locations) | |
| if not person.age_range: | |
| match = AGE_TEXT_RE.search(_squash(item.get("description"))) | |
| if match: | |
| person.age_range = match.group(1) | |
| return person | |
| # --------------------------------------------------------------------------- pages | |
| def parse_search_results(html: str) -> List[Person]: | |
| """People on a name search page, from its ``ItemList`` JSON-LD (up to 25).""" | |
| soup = BeautifulSoup(html, "html.parser") | |
| people: List[Person] = [] | |
| for block in iter_jsonld(soup): | |
| if block.get("@type") != "ItemList": | |
| continue | |
| for element in _as_list(block.get("itemListElement")): | |
| item = element.get("item") if isinstance(element, dict) else None | |
| if isinstance(item, dict) and item.get("@type") == "Person": | |
| people.append(_person_from_jsonld(item)) | |
| # Ages missing from the JSON-LD are often on the visible card | |
| cards = {} | |
| for card in soup.select(".serp-card"): | |
| link = card.select_one("a.name-link-text, [data-qa-selector=organic-card-person-name]") | |
| age = card.select_one(".person-age") | |
| if link and link.get("href") and age: | |
| cards[absolute_url(link["href"])] = _squash(age.get_text()) | |
| for person in people: | |
| if not person.age_range and AGE_RANGE_RE.match(cards.get(person.url, "")): | |
| person.age_range = cards[person.url] | |
| return people | |
| def parse_profile(html: str, url: str = "") -> Person: | |
| """A person page: JSON-LD ``Person`` plus the address and age from the HTML.""" | |
| soup = BeautifulSoup(html, "html.parser") | |
| item = next((b for b in iter_jsonld(soup) if b.get("@type") == "Person"), None) | |
| if item is None: | |
| raise ValueError("No Person JSON-LD on this page; it is not a Whitepages person page") | |
| person = _person_from_jsonld(item) | |
| if url: | |
| person.url = absolute_url(url) | |
| if not person.address: | |
| line1 = soup.select_one(".address-line1") | |
| line2 = soup.select_one(".address-line2") | |
| if line1: | |
| person.address = _squash(line1.get_text(" ")) | |
| if line2: | |
| match = CITY_LINE_RE.match(_squash(line2.get_text(" "))) | |
| if match: | |
| person.city, person.state, person.zip_code = match.group(1), match.group(2), match.group(3) or "" | |
| if not person.age_range: | |
| for el in soup.select("[data-qa-selector=person-age-mobile], .person-age"): | |
| match = re.search(r"\d{2}(?:s|\+)", el.get_text(" ")) | |
| if match: | |
| person.age_range = match.group(0) | |
| break | |
| html_phones = [format_phone(a.get_text()) for a in soup.select("[data-qa-selector=phone-number-link]")] | |
| person.phones = _unique(person.phones + [p for p in html_phones if p]) | |
| return person | |
| def is_profile_url(url: str) -> bool: | |
| return bool(PROFILE_RE.match(_squash(url))) | |
| # --------------------------------------------------------------------- tool | |
| class ScrapeDoWhitepagesTool(Tool): | |
| name = "whitepages_scraper" | |
| description = """ | |
| Look up public people-search records on Whitepages.com (US only). Two modes: | |
| - 'search': a full name (e.g. 'John Smith'), optionally with a two-letter state code in `state` | |
| and a city in `city` (a city needs a state). Returns up to 25 people per page: name, age range, | |
| current address, phone numbers, neighborhood, previous locations, relatives and the person page URL. | |
| - 'profile': a Whitepages person page URL (whitepages.com/name/<Name>/<City-ST>/<id>). Adds associates | |
| and fills address, age range and phones from the page. | |
| Each page is one request through a US residential IP via Scrape.do (10 credits). | |
| The data is about real people: do not use it for FCRA-regulated decisions (employment, housing, credit, insurance). | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": {"type": "string", "description": "'search' or 'profile'."}, | |
| "query": {"type": "string", "description": "A full name for 'search', or a person page URL for 'profile'."}, | |
| "state": {"type": "string", "description": "Optional two-letter US state code for 'search', e.g. 'WA'.", | |
| "nullable": True}, | |
| "city": {"type": "string", "description": "Optional city for 'search' (needs a state), e.g. 'Seattle'.", | |
| "nullable": True}, | |
| "page": {"type": "integer", "description": "Result page for 'search' (1, 2, ...), 25 people each. Default 1.", | |
| "nullable": True}, | |
| } | |
| super().__init__() | |
| def _resolve_token(self) -> Optional[str]: | |
| token = (self._api_token or os.environ.get("SCRAPE_DO_API_TOKEN") or "").strip() | |
| return token or None | |
| def _missing_token() -> Dict[str, Any]: | |
| return {"status": "error", "errorMessage": "A Scrape.do API token is required.", | |
| "hint": f"Get a free one at {SCRAPE_DO_SIGNUP_URL}"} | |
| def _fetch(self, token: str, url: str, valid) -> Dict[str, Any]: | |
| credits = 0 | |
| last: Dict[str, Any] = {"_error": "No response."} | |
| for attempt in range(3): | |
| started = time.time() | |
| try: | |
| response = requests.get(API_URL, params={"token": token, "url": url, "super": "true", "geoCode": "us"}, | |
| timeout=90) | |
| except requests.RequestException as exc: | |
| last = {"_error": f"Request failed: {exc.__class__.__name__}"} | |
| print(f"[fetch] attempt {attempt + 1}: {last['_error']}", flush=True) | |
| continue | |
| cost = response.headers.get("Scrape.do-Request-Cost") | |
| try: | |
| credits += int(float(cost)) if cost else CREDITS_PER_REQUEST | |
| except ValueError: | |
| credits += CREDITS_PER_REQUEST | |
| from_target = response.headers.get("Scrape.do-Initial-Status-Code") is not None | |
| text = response.text or "" | |
| print(f"[fetch] HTTP {response.status_code}, {len(text)} chars, {time.time() - started:.1f}s", flush=True) | |
| if response.status_code == 401 and not from_target: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True, "credits": credits} | |
| if response.status_code in (404, 410): | |
| return {"_error": "Whitepages has no page at that address (404).", "credits": credits} | |
| if response.status_code == 200 and (valid(text) or not BLOCK_RE.search(text[:5000])): | |
| return {"html": text, "credits": credits} | |
| last = {"_error": f"Whitepages didn't return the page (HTTP {response.status_code})."} | |
| if response.status_code not in (200, 429, 500, 502, 503, 504): | |
| break | |
| last["credits"] = credits | |
| return last | |
| def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: | |
| result = {"status": "error", "errorMessage": data.get("_error", "Unknown error"), | |
| "credits_used": data.get("credits", 0)} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| def search(self, name: str, state: Optional[str] = None, city: Optional[str] = None, page: int = 1) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| try: | |
| url = name_search_url(name or "", (city or "").strip(), (state or "").strip(), int(page or 1)) | |
| except ValueError as exc: | |
| return {"status": "error", "errorMessage": str(exc)} | |
| res = self._fetch(token, url, lambda html: bool(parse_search_results(html))) | |
| if "_error" in res: | |
| return self._as_error(res) | |
| people = [p.to_dict() for p in parse_search_results(res["html"])] | |
| return {"status": "success", "query": (name or "").strip(), "state": (state or "").upper(), | |
| "city": (city or "").strip(), "page": int(page or 1), "search_url": url, | |
| "results": people, "credits_used": res["credits"]} | |
| def profile(self, url: str) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| url = absolute_url((url or "").strip()).split("?")[0] | |
| if not is_profile_url(url): | |
| return {"status": "error", | |
| "errorMessage": "Paste a Whitepages person page: whitepages.com/name/<Name>/<City-ST>/<id>."} | |
| def ok(html: str) -> bool: | |
| try: | |
| parse_profile(html) | |
| return True | |
| except ValueError: | |
| return False | |
| res = self._fetch(token, url, ok) | |
| if "_error" in res: | |
| return self._as_error(res) | |
| try: | |
| person = parse_profile(res["html"], url=url) | |
| except ValueError: | |
| return {"status": "error", "errorMessage": "That page has no person data (removed or not a person page).", | |
| "credits_used": res["credits"]} | |
| return {"status": "success", "person": person.to_dict(), "credits_used": res["credits"]} | |
| def forward(self, mode: str, query: str, state: Optional[str] = None, city: Optional[str] = None, | |
| page: Optional[int] = None) -> str: | |
| key = (mode or "").strip().lower() | |
| if key == "search": | |
| result = self.search(query, state, city, page or 1) | |
| elif key == "profile": | |
| result = self.profile(query) | |
| else: | |
| result = {"status": "error", "errorMessage": "mode must be 'search' or 'profile'."} | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |