from typing import Any, Optional from smolagents.tools import Tool import json import re import urllib import unicodedata import time class GtmApiProviderTool(Tool): name = "gtm_api_provider" description = "Gives each API one provider sells and its rank on each on Datacircle's benchmark (https://datacircle.dev). Returns a dict with the keys provider, slug, website and apis: a list of dicts, each with api, slug, rank, bestPage, plans (how many) and best (the provider's best plan there: plan, score, perThousand, page)." inputs = {'slug': {'type': 'string', 'description': "The provider's slug, like 'bright-data', or its name, like 'Bright Data': the name is made a slug."}} output_type = "object" def forward(self, slug: str) -> dict: import re import unicodedata from urllib.parse import quote if not isinstance(slug, str): raise TypeError("slug is one provider's slug, a string like 'bright-data', or its name, like 'Bright Data'") # The API's rule for a provider's slug: its name in lowercase letters and digits, the rest a hyphen. name = unicodedata.normalize("NFKD", slug).encode("ascii", "ignore").decode() answer = self._get(f"/providers/{quote(re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-'), safe='')}", {}) for api in answer.get("apis", []): plans = api.pop("plans", []) api.pop("sheetPage", None) api["plans"] = len(plans) api["best"] = self._brief(min(plans, key=lambda p: p.get("rank") or 10**9)) if plans else None return answer def _brief(self, plan: dict) -> dict: # A provider may sell 60 APIs: its best plan on each in a few keys, the rest on the plan's page. keys = ["plan", "score", "perThousand", "page"] return {key: plan[key] for key in keys if plan.get(key) is not None} def _get(self, path: str, params: dict): import json import time from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import Request, urlopen query = urlencode({k: v for k, v in params.items() if v not in (None, "")}) url = "https://api.datacircle.dev/v1" + path + (f"?{query}" if query else "") request = Request(url, headers={"Accept": "application/json", "User-Agent": "datacircle-smolagents/0.1.0"}) for attempt in range(2): try: with urlopen(request, timeout=30) as response: return json.load(response) except HTTPError as error: if error.code == 429 and attempt == 0: time.sleep(min(int(error.headers.get("Retry-After") or 30), 60)) continue raise ValueError(f"Datacircle's API answered {error.code}: {error.read().decode('utf-8', 'replace')[:3000]}") from None except URLError: if attempt: raise time.sleep(2) def __init__(self, *args, **kwargs): self.is_initialized = False