from __future__ import annotations import json import os import re import time from dataclasses import asdict, dataclass, field from typing import Any, Dict, Iterable, List, Optional from urllib.parse import quote import requests from bs4 import BeautifulSoup from smolagents.tools import Tool SCRAPE_DO_SIGNUP_URL = "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=whitepages-scraper" API_URL = "https://api.scrape.do/" # Whitepages is US-only and sits behind Cloudflare. Every page goes through a US # residential IP (super=true, geoCode=us): 10 Scrape.do credits. CREDITS_PER_REQUEST = 10 DEFAULT_NAME = "John Smith" DEFAULT_CITY = "Seattle" DEFAULT_STATE = "WA" US_STATES = [ "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY", ] BLOCK_RE = re.compile(r"just a moment\.\.\.|cf-chl-|attention required|cf-turnstile", re.I) # ------------------------------------------------------------------ parsing # (same parser as `pip install whitepages-scraper`) BASE_URL = "https://www.whitepages.com" PROFILE_RE = re.compile(r"^(?:https?://(?:www\.)?whitepages\.com)?/name/[^/?#]+/[^/?#]+/[A-Za-z0-9]{8,14}/?(?:[?#].*)?$") AGE_TEXT_RE = re.compile(r"\bin (?:their|his|her) (\d{2}s|\d{2}\+)", re.I) AGE_RANGE_RE = re.compile(r"^\d{2}(?:s|\+)$") CITY_LINE_RE = re.compile(r"^\s*(.+?),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?\s*$") @dataclass class Person: """One person from a Whitepages search result or person page.""" name: str = "" first_name: str = "" middle_name: str = "" last_name: str = "" age_range: str = "" address: str = "" city: str = "" state: str = "" zip_code: str = "" neighborhood: str = "" phones: List[str] = field(default_factory=list) aliases: List[str] = field(default_factory=list) relatives: List[str] = field(default_factory=list) associates: List[str] = field(default_factory=list) previous_locations: List[str] = field(default_factory=list) url: str = "" @property def phone(self) -> str: return self.phones[0] if self.phones else "" @property def profile_id(self) -> str: match = re.search(r"/([A-Za-z0-9]{8,14})/?$", self.url.split("?")[0]) return match.group(1) if match else "" def to_dict(self) -> Dict[str, Any]: return asdict(self) def to_row(self, separator: str = " | ") -> Dict[str, Any]: row = self.to_dict() for key in ("phones", "aliases", "relatives", "associates", "previous_locations"): row[key] = separator.join(row[key]) return row CSV_FIELDS = [ "name", "first_name", "middle_name", "last_name", "age_range", "address", "city", "state", "zip_code", "neighborhood", "phones", "aliases", "relatives", "associates", "previous_locations", "url", ] # --------------------------------------------------------------------------- helpers def _squash(value: Any) -> str: return " ".join(str(value).split()) if value is not None else "" def _as_list(value: Any) -> List[Any]: if value is None: return [] return value if isinstance(value, list) else [value] def _unique(values: Iterable[str]) -> List[str]: seen, out = set(), [] for value in values: value = _squash(value) if value and value not in seen: seen.add(value) out.append(value) return out def _names(values: Any) -> List[str]: return _unique(v.get("name", "") if isinstance(v, dict) else str(v) for v in _as_list(values)) def format_phone(value: str) -> str: """Normalize ``+1-206-555-0142`` and similar to ``(206) 555-0142``; masked numbers return "".""" value = _squash(value) if not value or "*" in value: return "" digits = re.sub(r"\D", "", value) if len(digits) == 11 and digits.startswith("1"): digits = digits[1:] if len(digits) != 10: return value return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}" def absolute_url(url: str) -> str: url = _squash(url) if url.startswith("/"): url = BASE_URL + url return url.split("#")[0] def _slug(text: str) -> str: words = re.findall(r"[A-Za-z0-9'.]+", text or "") return "-".join(w.replace("'", "").replace(".", "") for w in words if w.strip("'.")) def name_search_url(name: str, city: str = "", state: str = "", page: int = 1) -> str: """Build a name search URL: ``/name/John-Smith``, ``/name/John-Smith/WA`` or ``/name/John-Smith/Seattle-WA``. ``page`` adds ``?page=N`` (25 people per page). """ slug = _slug(name) if not slug or not re.search(r"[A-Za-z]", slug): raise ValueError(f"Not a name: {name!r}") state = (state or "").strip().upper() if state and not re.fullmatch(r"[A-Z]{2}", state): raise ValueError(f"State must be a two-letter code like WA, got {state!r}") if city and not state: raise ValueError("A city needs a state, e.g. city='Seattle', state='WA'") url = f"{BASE_URL}/name/{quote(slug)}" if city: url += f"/{quote(_slug(city))}-{state}" elif state: url += f"/{state}" if page and int(page) > 1: url += f"?page={int(page)}" return url def iter_jsonld(soup: BeautifulSoup) -> Iterable[Dict[str, Any]]: for script in soup.find_all("script", type="application/ld+json"): try: data = json.loads(script.string or "") except (TypeError, ValueError): continue for item in _as_list(data): if isinstance(item, dict): yield item def _apply_address(person: Person, address: Any) -> None: for addr in _as_list(address): if not isinstance(addr, dict): continue person.address = _squash(addr.get("streetAddress")) person.city = _squash(addr.get("addressLocality")) person.state = _squash(addr.get("addressRegion")) person.zip_code = _squash(addr.get("postalCode")) if person.address or person.city: return def _person_from_jsonld(item: Dict[str, Any]) -> Person: person = Person( name=_squash(item.get("name")), first_name=_squash(item.get("givenName")), middle_name=" ".join(_unique(_as_list(item.get("additionalName")))), last_name=_squash(item.get("familyName")), url=absolute_url(item.get("url") or item.get("@id") or ""), ) _apply_address(person, item.get("address")) phones = [format_phone(p) for p in _as_list(item.get("telephone"))] for cp in _as_list(item.get("contactPoint")): if isinstance(cp, dict): phones.append(format_phone(cp.get("telephone", ""))) person.phones = _unique(p for p in phones if p) person.aliases = [a for a in _unique(_as_list(item.get("alternateName"))) if a != person.name] person.relatives = _names(item.get("relatedTo")) person.associates = _names(item.get("knows")) for prop in _as_list(item.get("additionalProperty")): if not isinstance(prop, dict): continue key, value = _squash(prop.get("name")).lower(), _squash(prop.get("value")) if key == "age range" and AGE_RANGE_RE.match(value): person.age_range = value elif key == "neighborhood": person.neighborhood = value elif key == "previous location" and value: person.previous_locations.append(value) person.previous_locations = _unique(person.previous_locations) if not person.age_range: match = AGE_TEXT_RE.search(_squash(item.get("description"))) if match: person.age_range = match.group(1) return person # --------------------------------------------------------------------------- pages def parse_search_results(html: str) -> List[Person]: """People on a name search page, from its ``ItemList`` JSON-LD (up to 25).""" soup = BeautifulSoup(html, "html.parser") people: List[Person] = [] for block in iter_jsonld(soup): if block.get("@type") != "ItemList": continue for element in _as_list(block.get("itemListElement")): item = element.get("item") if isinstance(element, dict) else None if isinstance(item, dict) and item.get("@type") == "Person": people.append(_person_from_jsonld(item)) # Ages missing from the JSON-LD are often on the visible card cards = {} for card in soup.select(".serp-card"): link = card.select_one("a.name-link-text, [data-qa-selector=organic-card-person-name]") age = card.select_one(".person-age") if link and link.get("href") and age: cards[absolute_url(link["href"])] = _squash(age.get_text()) for person in people: if not person.age_range and AGE_RANGE_RE.match(cards.get(person.url, "")): person.age_range = cards[person.url] return people def parse_profile(html: str, url: str = "") -> Person: """A person page: JSON-LD ``Person`` plus the address and age from the HTML.""" soup = BeautifulSoup(html, "html.parser") item = next((b for b in iter_jsonld(soup) if b.get("@type") == "Person"), None) if item is None: raise ValueError("No Person JSON-LD on this page; it is not a Whitepages person page") person = _person_from_jsonld(item) if url: person.url = absolute_url(url) if not person.address: line1 = soup.select_one(".address-line1") line2 = soup.select_one(".address-line2") if line1: person.address = _squash(line1.get_text(" ")) if line2: match = CITY_LINE_RE.match(_squash(line2.get_text(" "))) if match: person.city, person.state, person.zip_code = match.group(1), match.group(2), match.group(3) or "" if not person.age_range: for el in soup.select("[data-qa-selector=person-age-mobile], .person-age"): match = re.search(r"\d{2}(?:s|\+)", el.get_text(" ")) if match: person.age_range = match.group(0) break html_phones = [format_phone(a.get_text()) for a in soup.select("[data-qa-selector=phone-number-link]")] person.phones = _unique(person.phones + [p for p in html_phones if p]) return person def is_profile_url(url: str) -> bool: return bool(PROFILE_RE.match(_squash(url))) # --------------------------------------------------------------------- tool class ScrapeDoWhitepagesTool(Tool): name = "whitepages_scraper" description = """ Look up public people-search records on Whitepages.com (US only). Two modes: - 'search': a full name (e.g. 'John Smith'), optionally with a two-letter state code in `state` and a city in `city` (a city needs a state). Returns up to 25 people per page: name, age range, current address, phone numbers, neighborhood, previous locations, relatives and the person page URL. - 'profile': a Whitepages person page URL (whitepages.com/name///). Adds associates and fills address, age range and phones from the page. Each page is one request through a US residential IP via Scrape.do (10 credits). The data is about real people: do not use it for FCRA-regulated decisions (employment, housing, credit, insurance). """ output_type = "string" def __init__(self, api_token: Optional[str] = None) -> None: self._api_token = api_token self.inputs = { "mode": {"type": "string", "description": "'search' or 'profile'."}, "query": {"type": "string", "description": "A full name for 'search', or a person page URL for 'profile'."}, "state": {"type": "string", "description": "Optional two-letter US state code for 'search', e.g. 'WA'.", "nullable": True}, "city": {"type": "string", "description": "Optional city for 'search' (needs a state), e.g. 'Seattle'.", "nullable": True}, "page": {"type": "integer", "description": "Result page for 'search' (1, 2, ...), 25 people each. Default 1.", "nullable": True}, } super().__init__() def _resolve_token(self) -> Optional[str]: token = (self._api_token or os.environ.get("SCRAPE_DO_API_TOKEN") or "").strip() return token or None @staticmethod def _missing_token() -> Dict[str, Any]: return {"status": "error", "errorMessage": "A Scrape.do API token is required.", "hint": f"Get a free one at {SCRAPE_DO_SIGNUP_URL}"} def _fetch(self, token: str, url: str, valid) -> Dict[str, Any]: credits = 0 last: Dict[str, Any] = {"_error": "No response."} for attempt in range(3): started = time.time() try: response = requests.get(API_URL, params={"token": token, "url": url, "super": "true", "geoCode": "us"}, timeout=90) except requests.RequestException as exc: last = {"_error": f"Request failed: {exc.__class__.__name__}"} print(f"[fetch] attempt {attempt + 1}: {last['_error']}", flush=True) continue cost = response.headers.get("Scrape.do-Request-Cost") try: credits += int(float(cost)) if cost else CREDITS_PER_REQUEST except ValueError: credits += CREDITS_PER_REQUEST from_target = response.headers.get("Scrape.do-Initial-Status-Code") is not None text = response.text or "" print(f"[fetch] HTTP {response.status_code}, {len(text)} chars, {time.time() - started:.1f}s", flush=True) if response.status_code == 401 and not from_target: return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True, "credits": credits} if response.status_code in (404, 410): return {"_error": "Whitepages has no page at that address (404).", "credits": credits} if response.status_code == 200 and (valid(text) or not BLOCK_RE.search(text[:5000])): return {"html": text, "credits": credits} last = {"_error": f"Whitepages didn't return the page (HTTP {response.status_code})."} if response.status_code not in (200, 429, 500, 502, 503, 504): break last["credits"] = credits return last @staticmethod def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: result = {"status": "error", "errorMessage": data.get("_error", "Unknown error"), "credits_used": data.get("credits", 0)} if data.get("_auth"): result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" return result def search(self, name: str, state: Optional[str] = None, city: Optional[str] = None, page: int = 1) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() try: url = name_search_url(name or "", (city or "").strip(), (state or "").strip(), int(page or 1)) except ValueError as exc: return {"status": "error", "errorMessage": str(exc)} res = self._fetch(token, url, lambda html: bool(parse_search_results(html))) if "_error" in res: return self._as_error(res) people = [p.to_dict() for p in parse_search_results(res["html"])] return {"status": "success", "query": (name or "").strip(), "state": (state or "").upper(), "city": (city or "").strip(), "page": int(page or 1), "search_url": url, "results": people, "credits_used": res["credits"]} def profile(self, url: str) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() url = absolute_url((url or "").strip()).split("?")[0] if not is_profile_url(url): return {"status": "error", "errorMessage": "Paste a Whitepages person page: whitepages.com/name///."} def ok(html: str) -> bool: try: parse_profile(html) return True except ValueError: return False res = self._fetch(token, url, ok) if "_error" in res: return self._as_error(res) try: person = parse_profile(res["html"], url=url) except ValueError: return {"status": "error", "errorMessage": "That page has no person data (removed or not a person page).", "credits_used": res["credits"]} return {"status": "success", "person": person.to_dict(), "credits_used": res["credits"]} def forward(self, mode: str, query: str, state: Optional[str] = None, city: Optional[str] = None, page: Optional[int] = None) -> str: key = (mode or "").strip().lower() if key == "search": result = self.search(query, state, city, page or 1) elif key == "profile": result = self.profile(query) else: result = {"status": "error", "errorMessage": "mode must be 'search' or 'profile'."} return json.dumps(result, ensure_ascii=False, indent=2)