bugrahans's picture
Whitepages Scraper Space
4d97166 verified
Raw History Blame Contribute Delete
17.5 kB
from __future__ import annotations
import json
import os
import re
import time
from dataclasses import asdict, dataclass, field
from typing import Any, Dict, Iterable, List, Optional
from urllib.parse import quote
import requests
from bs4 import BeautifulSoup
from smolagents.tools import Tool
SCRAPE_DO_SIGNUP_URL = "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=whitepages-scraper"
API_URL = "https://api.scrape.do/"
# Whitepages is US-only and sits behind Cloudflare. Every page goes through a US
# residential IP (super=true, geoCode=us): 10 Scrape.do credits.
CREDITS_PER_REQUEST = 10
DEFAULT_NAME = "John Smith"
DEFAULT_CITY = "Seattle"
DEFAULT_STATE = "WA"
US_STATES = [
"AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY",
"LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH",
"OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY",
]
BLOCK_RE = re.compile(r"just a moment\.\.\.|cf-chl-|attention required|cf-turnstile", re.I)
# ------------------------------------------------------------------ parsing
# (same parser as `pip install whitepages-scraper`)
BASE_URL = "https://www.whitepages.com"
PROFILE_RE = re.compile(r"^(?:https?://(?:www\.)?whitepages\.com)?/name/[^/?#]+/[^/?#]+/[A-Za-z0-9]{8,14}/?(?:[?#].*)?$")
AGE_TEXT_RE = re.compile(r"\bin (?:their|his|her) (\d{2}s|\d{2}\+)", re.I)
AGE_RANGE_RE = re.compile(r"^\d{2}(?:s|\+)$")
CITY_LINE_RE = re.compile(r"^\s*(.+?),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?\s*$")
@dataclass
class Person:
"""One person from a Whitepages search result or person page."""
name: str = ""
first_name: str = ""
middle_name: str = ""
last_name: str = ""
age_range: str = ""
address: str = ""
city: str = ""
state: str = ""
zip_code: str = ""
neighborhood: str = ""
phones: List[str] = field(default_factory=list)
aliases: List[str] = field(default_factory=list)
relatives: List[str] = field(default_factory=list)
associates: List[str] = field(default_factory=list)
previous_locations: List[str] = field(default_factory=list)
url: str = ""
@property
def phone(self) -> str:
return self.phones[0] if self.phones else ""
@property
def profile_id(self) -> str:
match = re.search(r"/([A-Za-z0-9]{8,14})/?$", self.url.split("?")[0])
return match.group(1) if match else ""
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
def to_row(self, separator: str = " | ") -> Dict[str, Any]:
row = self.to_dict()
for key in ("phones", "aliases", "relatives", "associates", "previous_locations"):
row[key] = separator.join(row[key])
return row
CSV_FIELDS = [
"name", "first_name", "middle_name", "last_name", "age_range", "address", "city", "state", "zip_code",
"neighborhood", "phones", "aliases", "relatives", "associates", "previous_locations", "url",
]
# --------------------------------------------------------------------------- helpers
def _squash(value: Any) -> str:
return " ".join(str(value).split()) if value is not None else ""
def _as_list(value: Any) -> List[Any]:
if value is None:
return []
return value if isinstance(value, list) else [value]
def _unique(values: Iterable[str]) -> List[str]:
seen, out = set(), []
for value in values:
value = _squash(value)
if value and value not in seen:
seen.add(value)
out.append(value)
return out
def _names(values: Any) -> List[str]:
return _unique(v.get("name", "") if isinstance(v, dict) else str(v) for v in _as_list(values))
def format_phone(value: str) -> str:
"""Normalize ``+1-206-555-0142`` and similar to ``(206) 555-0142``; masked numbers return ""."""
value = _squash(value)
if not value or "*" in value:
return ""
digits = re.sub(r"\D", "", value)
if len(digits) == 11 and digits.startswith("1"):
digits = digits[1:]
if len(digits) != 10:
return value
return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}"
def absolute_url(url: str) -> str:
url = _squash(url)
if url.startswith("/"):
url = BASE_URL + url
return url.split("#")[0]
def _slug(text: str) -> str:
words = re.findall(r"[A-Za-z0-9'.]+", text or "")
return "-".join(w.replace("'", "").replace(".", "") for w in words if w.strip("'."))
def name_search_url(name: str, city: str = "", state: str = "", page: int = 1) -> str:
"""Build a name search URL: ``/name/John-Smith``, ``/name/John-Smith/WA`` or ``/name/John-Smith/Seattle-WA``.
``page`` adds ``?page=N`` (25 people per page).
"""
slug = _slug(name)
if not slug or not re.search(r"[A-Za-z]", slug):
raise ValueError(f"Not a name: {name!r}")
state = (state or "").strip().upper()
if state and not re.fullmatch(r"[A-Z]{2}", state):
raise ValueError(f"State must be a two-letter code like WA, got {state!r}")
if city and not state:
raise ValueError("A city needs a state, e.g. city='Seattle', state='WA'")
url = f"{BASE_URL}/name/{quote(slug)}"
if city:
url += f"/{quote(_slug(city))}-{state}"
elif state:
url += f"/{state}"
if page and int(page) > 1:
url += f"?page={int(page)}"
return url
def iter_jsonld(soup: BeautifulSoup) -> Iterable[Dict[str, Any]]:
for script in soup.find_all("script", type="application/ld+json"):
try:
data = json.loads(script.string or "")
except (TypeError, ValueError):
continue
for item in _as_list(data):
if isinstance(item, dict):
yield item
def _apply_address(person: Person, address: Any) -> None:
for addr in _as_list(address):
if not isinstance(addr, dict):
continue
person.address = _squash(addr.get("streetAddress"))
person.city = _squash(addr.get("addressLocality"))
person.state = _squash(addr.get("addressRegion"))
person.zip_code = _squash(addr.get("postalCode"))
if person.address or person.city:
return
def _person_from_jsonld(item: Dict[str, Any]) -> Person:
person = Person(
name=_squash(item.get("name")),
first_name=_squash(item.get("givenName")),
middle_name=" ".join(_unique(_as_list(item.get("additionalName")))),
last_name=_squash(item.get("familyName")),
url=absolute_url(item.get("url") or item.get("@id") or ""),
)
_apply_address(person, item.get("address"))
phones = [format_phone(p) for p in _as_list(item.get("telephone"))]
for cp in _as_list(item.get("contactPoint")):
if isinstance(cp, dict):
phones.append(format_phone(cp.get("telephone", "")))
person.phones = _unique(p for p in phones if p)
person.aliases = [a for a in _unique(_as_list(item.get("alternateName"))) if a != person.name]
person.relatives = _names(item.get("relatedTo"))
person.associates = _names(item.get("knows"))
for prop in _as_list(item.get("additionalProperty")):
if not isinstance(prop, dict):
continue
key, value = _squash(prop.get("name")).lower(), _squash(prop.get("value"))
if key == "age range" and AGE_RANGE_RE.match(value):
person.age_range = value
elif key == "neighborhood":
person.neighborhood = value
elif key == "previous location" and value:
person.previous_locations.append(value)
person.previous_locations = _unique(person.previous_locations)
if not person.age_range:
match = AGE_TEXT_RE.search(_squash(item.get("description")))
if match:
person.age_range = match.group(1)
return person
# --------------------------------------------------------------------------- pages
def parse_search_results(html: str) -> List[Person]:
"""People on a name search page, from its ``ItemList`` JSON-LD (up to 25)."""
soup = BeautifulSoup(html, "html.parser")
people: List[Person] = []
for block in iter_jsonld(soup):
if block.get("@type") != "ItemList":
continue
for element in _as_list(block.get("itemListElement")):
item = element.get("item") if isinstance(element, dict) else None
if isinstance(item, dict) and item.get("@type") == "Person":
people.append(_person_from_jsonld(item))
# Ages missing from the JSON-LD are often on the visible card
cards = {}
for card in soup.select(".serp-card"):
link = card.select_one("a.name-link-text, [data-qa-selector=organic-card-person-name]")
age = card.select_one(".person-age")
if link and link.get("href") and age:
cards[absolute_url(link["href"])] = _squash(age.get_text())
for person in people:
if not person.age_range and AGE_RANGE_RE.match(cards.get(person.url, "")):
person.age_range = cards[person.url]
return people
def parse_profile(html: str, url: str = "") -> Person:
"""A person page: JSON-LD ``Person`` plus the address and age from the HTML."""
soup = BeautifulSoup(html, "html.parser")
item = next((b for b in iter_jsonld(soup) if b.get("@type") == "Person"), None)
if item is None:
raise ValueError("No Person JSON-LD on this page; it is not a Whitepages person page")
person = _person_from_jsonld(item)
if url:
person.url = absolute_url(url)
if not person.address:
line1 = soup.select_one(".address-line1")
line2 = soup.select_one(".address-line2")
if line1:
person.address = _squash(line1.get_text(" "))
if line2:
match = CITY_LINE_RE.match(_squash(line2.get_text(" ")))
if match:
person.city, person.state, person.zip_code = match.group(1), match.group(2), match.group(3) or ""
if not person.age_range:
for el in soup.select("[data-qa-selector=person-age-mobile], .person-age"):
match = re.search(r"\d{2}(?:s|\+)", el.get_text(" "))
if match:
person.age_range = match.group(0)
break
html_phones = [format_phone(a.get_text()) for a in soup.select("[data-qa-selector=phone-number-link]")]
person.phones = _unique(person.phones + [p for p in html_phones if p])
return person
def is_profile_url(url: str) -> bool:
return bool(PROFILE_RE.match(_squash(url)))
# --------------------------------------------------------------------- tool
class ScrapeDoWhitepagesTool(Tool):
name = "whitepages_scraper"
description = """
Look up public people-search records on Whitepages.com (US only). Two modes:
- 'search': a full name (e.g. 'John Smith'), optionally with a two-letter state code in `state`
and a city in `city` (a city needs a state). Returns up to 25 people per page: name, age range,
current address, phone numbers, neighborhood, previous locations, relatives and the person page URL.
- 'profile': a Whitepages person page URL (whitepages.com/name/<Name>/<City-ST>/<id>). Adds associates
and fills address, age range and phones from the page.
Each page is one request through a US residential IP via Scrape.do (10 credits).
The data is about real people: do not use it for FCRA-regulated decisions (employment, housing, credit, insurance).
"""
output_type = "string"
def __init__(self, api_token: Optional[str] = None) -> None:
self._api_token = api_token
self.inputs = {
"mode": {"type": "string", "description": "'search' or 'profile'."},
"query": {"type": "string", "description": "A full name for 'search', or a person page URL for 'profile'."},
"state": {"type": "string", "description": "Optional two-letter US state code for 'search', e.g. 'WA'.",
"nullable": True},
"city": {"type": "string", "description": "Optional city for 'search' (needs a state), e.g. 'Seattle'.",
"nullable": True},
"page": {"type": "integer", "description": "Result page for 'search' (1, 2, ...), 25 people each. Default 1.",
"nullable": True},
}
super().__init__()
def _resolve_token(self) -> Optional[str]:
token = (self._api_token or os.environ.get("SCRAPE_DO_API_TOKEN") or "").strip()
return token or None
@staticmethod
def _missing_token() -> Dict[str, Any]:
return {"status": "error", "errorMessage": "A Scrape.do API token is required.",
"hint": f"Get a free one at {SCRAPE_DO_SIGNUP_URL}"}
def _fetch(self, token: str, url: str, valid) -> Dict[str, Any]:
credits = 0
last: Dict[str, Any] = {"_error": "No response."}
for attempt in range(3):
started = time.time()
try:
response = requests.get(API_URL, params={"token": token, "url": url, "super": "true", "geoCode": "us"},
timeout=90)
except requests.RequestException as exc:
last = {"_error": f"Request failed: {exc.__class__.__name__}"}
print(f"[fetch] attempt {attempt + 1}: {last['_error']}", flush=True)
continue
cost = response.headers.get("Scrape.do-Request-Cost")
try:
credits += int(float(cost)) if cost else CREDITS_PER_REQUEST
except ValueError:
credits += CREDITS_PER_REQUEST
from_target = response.headers.get("Scrape.do-Initial-Status-Code") is not None
text = response.text or ""
print(f"[fetch] HTTP {response.status_code}, {len(text)} chars, {time.time() - started:.1f}s", flush=True)
if response.status_code == 401 and not from_target:
return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True, "credits": credits}
if response.status_code in (404, 410):
return {"_error": "Whitepages has no page at that address (404).", "credits": credits}
if response.status_code == 200 and (valid(text) or not BLOCK_RE.search(text[:5000])):
return {"html": text, "credits": credits}
last = {"_error": f"Whitepages didn't return the page (HTTP {response.status_code})."}
if response.status_code not in (200, 429, 500, 502, 503, 504):
break
last["credits"] = credits
return last
@staticmethod
def _as_error(data: Dict[str, Any]) -> Dict[str, Any]:
result = {"status": "error", "errorMessage": data.get("_error", "Unknown error"),
"credits_used": data.get("credits", 0)}
if data.get("_auth"):
result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
return result
def search(self, name: str, state: Optional[str] = None, city: Optional[str] = None, page: int = 1) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
try:
url = name_search_url(name or "", (city or "").strip(), (state or "").strip(), int(page or 1))
except ValueError as exc:
return {"status": "error", "errorMessage": str(exc)}
res = self._fetch(token, url, lambda html: bool(parse_search_results(html)))
if "_error" in res:
return self._as_error(res)
people = [p.to_dict() for p in parse_search_results(res["html"])]
return {"status": "success", "query": (name or "").strip(), "state": (state or "").upper(),
"city": (city or "").strip(), "page": int(page or 1), "search_url": url,
"results": people, "credits_used": res["credits"]}
def profile(self, url: str) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
url = absolute_url((url or "").strip()).split("?")[0]
if not is_profile_url(url):
return {"status": "error",
"errorMessage": "Paste a Whitepages person page: whitepages.com/name/<Name>/<City-ST>/<id>."}
def ok(html: str) -> bool:
try:
parse_profile(html)
return True
except ValueError:
return False
res = self._fetch(token, url, ok)
if "_error" in res:
return self._as_error(res)
try:
person = parse_profile(res["html"], url=url)
except ValueError:
return {"status": "error", "errorMessage": "That page has no person data (removed or not a person page).",
"credits_used": res["credits"]}
return {"status": "success", "person": person.to_dict(), "credits_used": res["credits"]}
def forward(self, mode: str, query: str, state: Optional[str] = None, city: Optional[str] = None,
page: Optional[int] = None) -> str:
key = (mode or "").strip().lower()
if key == "search":
result = self.search(query, state, city, page or 1)
elif key == "profile":
result = self.profile(query)
else:
result = {"status": "error", "errorMessage": "mode must be 'search' or 'profile'."}
return json.dumps(result, ensure_ascii=False, indent=2)