bugrahans's picture
Upload 4 files
10fcf7e verified
Raw History Blame Contribute Delete
18 kB
from __future__ import annotations
import os
import time
import requests
from smolagents.tools import Tool
SCRAPE_DO_SIGNUP_URL = "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=domain-com-au-scraper"
API_URL = "https://api.scrape.do/"
# Domain sits behind Akamai (403 Access Denied for scripts): super=true (residential) is needed, 10 credits per page.
CREDITS_PER_REQUEST = 10
DEFAULT_SUBURB = "Bondi Beach NSW 2026"
# ------------------------------------------------------------------ parsing
# (same parser as `pip install domain-com-au-scraper`)
import json
import re
from dataclasses import asdict, dataclass, field
from typing import Any, Dict, List, Optional, Tuple
BASE_URL = "https://www.domain.com.au"
NEXT_RE = re.compile(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.S)
LISTING_URL_RE = re.compile(r"^(?:https?://(?:www\.)?domain\.com\.au)?/([a-z0-9-]+-(\d{6,}))/?(?:[?#].*)?$")
SLUG_RE = re.compile(r"^[a-z0-9-]+-(?:nsw|vic|qld|wa|sa|tas|act|nt)-\d{4}$")
MODES = {"sale": "sale", "buy": "sale", "rent": "rent", "sold": "sold-listings", "sold-listings": "sold-listings"}
def next_data(html: str) -> Dict[str, Any]:
"""The ``__NEXT_DATA__`` JSON of a Domain page."""
match = NEXT_RE.search(html or "")
if not match:
raise ValueError("No __NEXT_DATA__ on this page (blocked or not a Domain page)")
return json.loads(match.group(1))
def _props(html: str) -> Dict[str, Any]:
return next_data(html).get("props", {}).get("pageProps", {}).get("componentProps", {}) or {}
def suburb_slug(location: str) -> str:
"""``Bondi Beach NSW 2026`` or ``bondi-beach-nsw-2026`` -> ``bondi-beach-nsw-2026``."""
slug = re.sub(r"[^a-z0-9]+", "-", (location or "").strip().lower()).strip("-")
if not SLUG_RE.match(slug):
raise ValueError(f"Use a suburb with state and postcode, e.g. 'Bondi Beach NSW 2026' (got {location!r})")
return slug
def search_url(location: str, mode: str = "sale", page: int = 1) -> str:
"""Search URL for a suburb: ``mode`` is ``sale``, ``rent`` or ``sold``."""
path = MODES.get((mode or "sale").lower())
if not path:
raise ValueError("mode must be 'sale', 'rent' or 'sold'")
url = f"{BASE_URL}/{path}/{suburb_slug(location)}/"
return url + (f"?page={page}" if page > 1 else "")
def listing_url(value: str) -> str:
match = LISTING_URL_RE.match((value or "").strip())
if not match:
raise ValueError(f"Not a Domain listing URL: {value}")
return f"{BASE_URL}/{match.group(1)}"
def is_listing_url(value: str) -> bool:
return bool(LISTING_URL_RE.match((value or "").strip()))
def parse_price(text: str) -> Tuple[Optional[int], Optional[int]]:
"""Numbers in a price text: ``$675,000 to $725,000`` -> (675000, 725000); ``$1,100,000`` -> (1100000, 1100000)."""
nums = []
for raw, suffix in re.findall(r"\$\s?([\d,]+(?:\.\d+)?)\s*([mk])?\b", text or "", re.I):
try:
value = float(raw.replace(",", ""))
except ValueError:
continue
value *= {"m": 1_000_000, "k": 1_000}.get(suffix.lower(), 1) if suffix else 1
nums.append(int(value))
if not nums:
return None, None
return min(nums), max(nums)
@dataclass
class Listing:
"""One listing from a search page (and, after ``get_listing``, its page)."""
listing_id: int = 0
url: str = ""
price_text: str = ""
price_min: Optional[int] = None
price_max: Optional[int] = None
status: str = ""
property_type: str = ""
beds: Optional[int] = None
baths: Optional[int] = None
parking: Optional[int] = None
land_size: Optional[float] = None
land_unit: str = ""
street: str = ""
suburb: str = ""
state: str = ""
postcode: str = ""
latitude: Optional[float] = None
longitude: Optional[float] = None
agency: str = ""
agency_id: Optional[int] = None
agents: List[str] = field(default_factory=list)
auction: str = ""
inspection_open: str = ""
inspection_close: str = ""
images: List[str] = field(default_factory=list)
promo_type: str = ""
@property
def address(self) -> str:
parts = [self.street, self.suburb, f"{self.state} {self.postcode}".strip()]
return ", ".join(p for p in parts if p)
def to_dict(self) -> Dict[str, Any]:
out = asdict(self)
out["address"] = self.address
return out
def to_row(self) -> Dict[str, Any]:
row = self.to_dict()
row["agents"] = " | ".join(self.agents)
row["images"] = len(self.images)
return row
@dataclass
class SearchPage:
listings: List[Listing]
total_listings: int = 0
total_pages: int = 0
current_page: int = 1
def _int(value: Any) -> Optional[int]:
try:
return int(value) if value not in (None, "") else None
except (TypeError, ValueError):
return None
def _listing_id(url: str) -> int:
match = re.search(r"-(\d{6,})/?$", url or "")
return int(match.group(1)) if match else 0
def listing_from_model(model: Dict[str, Any]) -> Listing:
features = model.get("features") or {}
address = model.get("address") or {}
branding = model.get("branding") or {}
inspection = model.get("inspection") or {}
tags = model.get("tags") or {}
url = model.get("url") or ""
listing = Listing(
listing_id=_listing_id(url),
url=BASE_URL + url if url.startswith("/") else url,
price_text=(model.get("price") or "").strip(),
status=(tags.get("tagText") or "").strip(),
property_type=features.get("propertyTypeFormatted") or features.get("propertyType") or "",
beds=_int(features.get("beds")),
baths=_int(features.get("baths")),
parking=_int(features.get("parking")),
land_size=features.get("landSize") or None,
land_unit=features.get("landUnit") or "" if features.get("landSize") else "",
street=address.get("street") or "",
suburb=address.get("suburb") or "",
state=address.get("state") or "",
postcode=str(address.get("postcode") or ""),
latitude=address.get("lat"),
longitude=address.get("lng"),
agency=branding.get("brandName") or "",
agency_id=_int(branding.get("agencyId")),
agents=[a.get("agentName", "") for a in branding.get("agents") or [] if a.get("agentName")],
auction=model.get("auction") or "",
inspection_open=inspection.get("openTime") or "",
inspection_close=inspection.get("closeTime") or "",
images=list(model.get("images") or []),
promo_type=model.get("promoType") or "",
)
listing.price_min, listing.price_max = parse_price(listing.price_text)
return listing
def parse_search(html: str) -> SearchPage:
"""Listings on a search page, in the order Domain shows them (projects skipped)."""
props = _props(html)
listings_map = props.get("listingsMap") or {}
order = [str(i) for i in props.get("listingSearchResultIds") or []] or list(listings_map)
listings = []
for key in order:
item = listings_map.get(key) or listings_map.get(_int(key))
if not item or item.get("listingType") != "listing":
continue
listings.append(listing_from_model(item.get("listingModel") or {}))
return SearchPage(
listings=listings,
total_listings=_int(props.get("totalListings")) or 0,
total_pages=_int(props.get("totalPages")) or 0,
current_page=_int(props.get("currentPage")) or 1,
)
@dataclass
class Agent:
name: str = ""
phone: str = ""
mobile: str = ""
profile_url: str = ""
@dataclass
class ListingDetail:
"""Everything on a listing page."""
listing_id: int = 0
url: str = ""
title: str = ""
headline: str = ""
listing_type: str = ""
status: str = ""
method: str = ""
property_type: str = ""
beds: Optional[int] = None
baths: Optional[int] = None
parking: Optional[int] = None
land_area: Optional[float] = None
address: str = ""
unit_number: str = ""
street_number: str = ""
street: str = ""
suburb: str = ""
state: str = ""
postcode: str = ""
latitude: Optional[float] = None
longitude: Optional[float] = None
description: str = ""
features: List[str] = field(default_factory=list)
agency: str = ""
agents: List[Agent] = field(default_factory=list)
inspection: str = ""
created_on: str = ""
modified_on: str = ""
images: List[str] = field(default_factory=list)
suburb_median_price: Optional[int] = None
suburb_median_rent: Optional[int] = None
suburb_days_on_market: Optional[int] = None
suburb_auction_clearance: Optional[int] = None
suburb_sold_this_year: Optional[int] = None
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
def to_row(self) -> Dict[str, Any]:
row = self.to_dict()
row["features"] = " | ".join(self.features)
row["agents"] = " | ".join(a.name for a in self.agents)
row["images"] = len(self.images)
return row
def parse_listing(html: str, url: str = "") -> ListingDetail:
"""Parse a listing page."""
cp = _props(html)
if not cp.get("listingId"):
raise ValueError("No listing on this page")
summary = cp.get("listingSummary") or {}
insights = cp.get("suburbInsights") or {}
mapdata = cp.get("map") or {}
inspection = cp.get("inspection") or {}
stats = {s.get("key"): s.get("value") for s in summary.get("stats") or [] if isinstance(s, dict)}
description = cp.get("description") or ""
if isinstance(description, list):
description = "\n".join(p for p in description if p)
images = []
for slide in (cp.get("gallery") or {}).get("slides") or []:
original = (slide.get("images") or {}).get("original") or {}
src = original.get("url") if isinstance(original, dict) else original
if src and slide.get("mediaType", "image") == "image":
images.append(src)
features = [f.get("name") for f in cp.get("structuredFeatures") or [] if f.get("name")] or list(cp.get("features") or [])
agents = []
for a in (cp.get("priceGuide") or {}).get("agents") or cp.get("agents") or []:
agents.append(Agent(name=a.get("name", ""), phone=a.get("phone") or "", mobile=a.get("mobile") or "",
profile_url=a.get("agentProfileUrl") or ""))
return ListingDetail(
listing_id=_int(cp.get("listingId")) or 0,
url=url or cp.get("listingUrl") or "",
title=(summary.get("title") or "").strip(),
headline=(cp.get("headline") or "").strip(),
listing_type={"buy": "sale"}.get(summary.get("mode") or "", summary.get("mode") or ""),
status=summary.get("status") or "",
method=summary.get("method") or "",
property_type=summary.get("propertyType") or cp.get("propertyType") or "",
beds=_int(summary.get("beds")),
baths=_int(summary.get("baths")),
parking=_int(summary.get("parking")),
land_area=stats.get("landArea"),
address=summary.get("address") or cp.get("address") or "",
unit_number=str(cp.get("unitNumber") or ""),
street_number=str(cp.get("streetNumber") or ""),
street=cp.get("street") or "",
suburb=cp.get("suburb") or "",
state=(cp.get("stateAbbreviation") or "").upper(),
postcode=str(cp.get("postcode") or ""),
latitude=mapdata.get("latitude"),
longitude=mapdata.get("longitude"),
description=description.strip(),
features=features,
agency=(cp.get("priceGuide") or {}).get("agencyName") or cp.get("agencyName") or "",
agents=agents,
inspection=(inspection.get("inspectionText") or "").strip(),
created_on=cp.get("createdOn") or "",
modified_on=cp.get("modifiedOn") or "",
images=images,
suburb_median_price=_int(insights.get("medianPrice")),
suburb_median_rent=_int(insights.get("medianRentPrice")),
suburb_days_on_market=_int(insights.get("avgDaysOnMarket")),
suburb_auction_clearance=_int(insights.get("auctionClearance")),
suburb_sold_this_year=_int(insights.get("nrSoldThisYear")),
)
LISTING_FIELDS = list(Listing().to_dict().keys())
DETAIL_FIELDS = list(ListingDetail().to_dict().keys())
# --------------------------------------------------------------------- tool
class ScrapeDoDomainTool(Tool):
name = "domain_com_au_scraper"
description = """
Read Australian property listings from Domain.com.au. Two modes:
- 'search': a suburb with state and postcode (e.g. 'Bondi Beach NSW 2026') plus `listing_mode` 'sale', 'rent' or
'sold'. Returns about 20 listings per page: price text and parsed range, sale status and date, beds, baths,
parking, land size, property type, address, coordinates, agency, agents.
- 'listing': a Domain listing URL. Returns description, features, agents, listing dates and suburb medians.
Each page is one Scrape.do request with super=true (10 credits).
"""
output_type = "string"
def __init__(self, api_token: Optional[str] = None) -> None:
self._api_token = api_token
self.inputs = {
"mode": {"type": "string", "description": "'search' or 'listing'."},
"query": {"type": "string", "description": "Suburb with state and postcode, or a listing URL."},
"listing_mode": {"type": "string", "description": "'sale', 'rent' or 'sold'. Default 'sale'.", "nullable": True},
"page": {"type": "integer", "description": "Result page (1-50). Default 1.", "nullable": True},
}
super().__init__()
def _token(self) -> Optional[str]:
return (self._api_token or os.environ.get("SCRAPE_DO_API_TOKEN") or "").strip() or None
@staticmethod
def _missing() -> Dict[str, Any]:
return {"status": "error", "errorMessage": "A Scrape.do API token is required.",
"hint": f"Get a free one at {SCRAPE_DO_SIGNUP_URL}"}
def _fetch(self, token: str, url: str) -> Dict[str, Any]:
credits = 0
last: Dict[str, Any] = {"_error": "No response."}
for attempt in range(3):
started = time.time()
try:
r = requests.get(API_URL, params={"token": token, "url": url, "super": "true"}, timeout=90)
except requests.RequestException as exc:
last = {"_error": f"Request failed: {exc.__class__.__name__}"}
continue
cost = r.headers.get("Scrape.do-Request-Cost")
try:
credits += int(float(cost)) if cost else CREDITS_PER_REQUEST
except ValueError:
credits += CREDITS_PER_REQUEST
print(f"[fetch] HTTP {r.status_code}, {len(r.text or '')} chars, {time.time() - started:.1f}s", flush=True)
if r.status_code == 401 and r.headers.get("Scrape.do-Initial-Status-Code") is None:
return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True, "credits": credits}
if r.status_code in (404, 410):
return {"_error": "Domain has no page at that address (404).", "credits": credits}
if r.status_code == 200 and "__NEXT_DATA__" in (r.text or ""):
return {"html": r.text, "credits": credits}
last = {"_error": f"Domain didn't return the page (HTTP {r.status_code})."}
time.sleep(1 + attempt)
last["credits"] = credits
return last
@staticmethod
def _err(d: Dict[str, Any]) -> Dict[str, Any]:
out = {"status": "error", "errorMessage": d.get("_error", "Unknown error"), "credits_used": d.get("credits", 0)}
if d.get("_auth"):
out["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
return out
def search(self, suburb: str, listing_mode: str = "sale", page: int = 1) -> Dict[str, Any]:
token = self._token()
if not token:
return self._missing()
try:
url = search_url(suburb, listing_mode or "sale", min(max(1, int(page or 1)), 50))
except ValueError as exc:
return {"status": "error", "errorMessage": str(exc)}
res = self._fetch(token, url)
if "_error" in res:
return self._err(res)
result = parse_search(res["html"])
return {"status": "success", "suburb": suburb, "mode": listing_mode, "page": result.current_page,
"total_listings": result.total_listings, "total_pages": result.total_pages,
"listings": [l.to_dict() for l in result.listings], "credits_used": res["credits"]}
def listing(self, url: str) -> Dict[str, Any]:
token = self._token()
if not token:
return self._missing()
try:
url = listing_url(url)
except ValueError as exc:
return {"status": "error", "errorMessage": str(exc)}
res = self._fetch(token, url)
if "_error" in res:
return self._err(res)
try:
detail = parse_listing(res["html"], url=url)
except ValueError:
return {"status": "error", "errorMessage": "That page has no listing.", "credits_used": res["credits"]}
return {"status": "success", "listing": detail.to_dict(), "credits_used": res["credits"]}
def forward(self, mode: str, query: str, listing_mode: Optional[str] = None, page: Optional[int] = None) -> str:
key = (mode or "").strip().lower()
if key == "search":
result = self.search(query, listing_mode or "sale", page or 1)
elif key == "listing":
result = self.listing(query)
else:
result = {"status": "error", "errorMessage": "mode must be 'search' or 'listing'."}
return json.dumps(result, ensure_ascii=False, indent=2)