waynehamadi2's picture
gtm_api_provider: Datacircle's GET /providers/{slug}
6bc10d0 verified
Raw History Blame Contribute Delete
3.1 kB
from typing import Any, Optional
from smolagents.tools import Tool
import json
import re
import urllib
import unicodedata
import time
class GtmApiProviderTool(Tool):
name = "gtm_api_provider"
description = "Gives each API one provider sells and its rank on each on Datacircle's benchmark (https://datacircle.dev). Returns a dict with the keys provider, slug, website and apis: a list of dicts, each with api, slug, rank, bestPage, plans (how many) and best (the provider's best plan there: plan, score, perThousand, page)."
inputs = {'slug': {'type': 'string', 'description': "The provider's slug, like 'bright-data', or its name, like 'Bright Data': the name is made a slug."}}
output_type = "object"
def forward(self, slug: str) -> dict:
import re
import unicodedata
from urllib.parse import quote
if not isinstance(slug, str):
raise TypeError("slug is one provider's slug, a string like 'bright-data', or its name, like 'Bright Data'")
# The API's rule for a provider's slug: its name in lowercase letters and digits, the rest a hyphen.
name = unicodedata.normalize("NFKD", slug).encode("ascii", "ignore").decode()
answer = self._get(f"/providers/{quote(re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-'), safe='')}", {})
for api in answer.get("apis", []):
plans = api.pop("plans", [])
api.pop("sheetPage", None)
api["plans"] = len(plans)
api["best"] = self._brief(min(plans, key=lambda p: p.get("rank") or 10**9)) if plans else None
return answer
def _brief(self, plan: dict) -> dict:
# A provider may sell 60 APIs: its best plan on each in a few keys, the rest on the plan's page.
keys = ["plan", "score", "perThousand", "page"]
return {key: plan[key] for key in keys if plan.get(key) is not None}
def _get(self, path: str, params: dict):
import json
import time
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
query = urlencode({k: v for k, v in params.items() if v not in (None, "")})
url = "https://api.datacircle.dev/v1" + path + (f"?{query}" if query else "")
request = Request(url, headers={"Accept": "application/json", "User-Agent": "datacircle-smolagents/0.1.0"})
for attempt in range(2):
try:
with urlopen(request, timeout=30) as response:
return json.load(response)
except HTTPError as error:
if error.code == 429 and attempt == 0:
time.sleep(min(int(error.headers.get("Retry-After") or 30), 60))
continue
raise ValueError(f"Datacircle's API answered {error.code}: {error.read().decode('utf-8', 'replace')[:3000]}") from None
except URLError:
if attempt:
raise
time.sleep(2)
def __init__(self, *args, **kwargs):
self.is_initialized = False