Download Modules/_engines.py from alyxsis/Tools: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/spaces/alyxsis/Tools/resolve/main/Modules/_engines.py
- Command line
-
hf download hf://spaces/alyxsis/Tools/Modules/_engines.py
-
curl -L -o _engines.py https://huggingface.co/spaces/alyxsis/Tools/resolve/main/Modules/_engines.py
10.8 kB
| """Text search engines scraped directly: DuckDuckGo (HTML), Bing, Yahoo, plus the Wikipedia API. | |
| ddgs 9.x disables its Bing engine, and its Yahoo parser runs nested result blocks | |
| together (one title made of every title below it). These are small, tested | |
| replacements for text search only; news, images, videos and books still use ddgs. | |
| Each engine returns a list of {"title", "href", "body"} dicts, or raises | |
| NoResults (nothing on the page) or Blocked (a captcha or challenge page). | |
| """ | |
| from __future__ import annotations | |
| import base64 | |
| import re | |
| import time | |
| from typing import Any, Callable | |
| from urllib.parse import parse_qs, unquote, unquote_plus, urlparse | |
| import primp | |
| from lxml import html as lxml_html | |
| class NoResults(Exception): | |
| pass | |
| class Blocked(Exception): | |
| pass | |
| def _client(timeout: float) -> primp.Client: | |
| # A fresh client per request: a new random browser fingerprint each time. | |
| return primp.Client(impersonate="random", impersonate_os="random", timeout=max(1.0, timeout), follow_redirects=True) | |
| def _decode(content: bytes) -> str: | |
| try: | |
| return content.decode("utf-8") | |
| except UnicodeDecodeError: | |
| return content.decode("cp1252", errors="replace") | |
| def _text(node: Any) -> str: | |
| return " ".join(node.text_content().split()) if node is not None else "" | |
| def _first(nodes: list[Any]) -> Any: | |
| return nodes[0] if nodes else None | |
| def _tree(content: bytes) -> Any: | |
| return lxml_html.fromstring(_decode(content)) | |
| def _has_class(name: str) -> str: | |
| return f"contains(concat(' ', normalize-space(@class), ' '), ' {name} ')" | |
| # --- DuckDuckGo (html.duckduckgo.com) ------------------------------------------------- | |
| def _ddg_href(raw: str) -> str: | |
| if raw.startswith("//"): | |
| raw = "https:" + raw | |
| parsed = urlparse(raw) | |
| if parsed.netloc.endswith("duckduckgo.com") and parsed.path.startswith("/l/"): | |
| target = parse_qs(parsed.query).get("uddg", [""])[0] | |
| return unquote(target) | |
| return raw | |
| def duckduckgo(query: str, page: int, timelimit: str | None, deadline: float) -> list[dict[str, str]]: | |
| payload = {"q": query, "b": "", "kl": "us-en"} | |
| if page > 1: | |
| payload["s"] = str(10 + (page - 2) * 15) | |
| payload["dc"] = str(11 + (page - 2) * 15) | |
| if timelimit: | |
| payload["df"] = timelimit | |
| # DuckDuckGo challenges some browser fingerprints (HTTP 202) and not others, | |
| # so a challenge gets one retry with a fresh fingerprint while time allows. | |
| for attempt in range(2): | |
| remaining = deadline - time.monotonic() | |
| if remaining < 0.5: | |
| break | |
| resp = _client(remaining).post("https://html.duckduckgo.com/html/", data=payload) | |
| if resp.status_code == 200: | |
| tree = _tree(resp.content) | |
| results = [] | |
| for item in tree.xpath(f"//div[{_has_class('result')}]"): | |
| if "result--ad" in (item.get("class") or ""): | |
| continue | |
| link = _first(item.xpath(f".//a[{_has_class('result__a')}]")) | |
| if link is None: | |
| continue | |
| href = _ddg_href(link.get("href") or "") | |
| if not href.startswith("http") or "duckduckgo.com/y.js" in href: | |
| continue | |
| snippet = _first(item.xpath(f".//*[{_has_class('result__snippet')}]")) | |
| results.append({"title": _text(link), "href": href, "body": _text(snippet)}) | |
| if results: | |
| return results | |
| if b"anomaly" in resp.content or b"challenge" in resp.content: | |
| continue | |
| raise NoResults("duckduckgo returned no results") | |
| if resp.status_code not in (202, 403, 429): | |
| raise Blocked(f"duckduckgo HTTP {resp.status_code}") | |
| raise Blocked("duckduckgo challenged the request") | |
| # --- Bing ------------------------------------------------------------------------------- | |
| def _bing_href(raw: str) -> str: | |
| parsed = urlparse(raw) | |
| if parsed.netloc.endswith("bing.com") and parsed.path.startswith("/ck/"): | |
| u = parse_qs(parsed.query).get("u", [""])[0] | |
| if len(u) > 2: | |
| b64 = u[2:] | |
| try: | |
| return base64.urlsafe_b64decode(b64 + "=" * (-len(b64) % 4)).decode() | |
| except Exception: | |
| return "" | |
| return raw | |
| _BING_TIME = {"d": 'ex1:"ez1"', "w": 'ex1:"ez2"', "m": 'ex1:"ez3"'} | |
| def bing(query: str, page: int, timelimit: str | None, deadline: float) -> list[dict[str, str]]: | |
| if page > 1: | |
| # Bing ignores "first=" without its session cookies and repeats page one. | |
| raise NoResults("bing serves only one page of results here") | |
| params = {"q": query, "setlang": "en", "cc": "us", "mkt": "en-US"} | |
| if timelimit == "y": | |
| day = int(time.time() // 86400) | |
| params["filters"] = f'ex1:"ez5_{day - 365}_{day}"' | |
| elif timelimit in _BING_TIME: | |
| params["filters"] = _BING_TIME[timelimit] | |
| resp = _client(deadline - time.monotonic()).get("https://www.bing.com/search", params=params) | |
| if resp.status_code != 200: | |
| raise Blocked(f"bing HTTP {resp.status_code}") | |
| tree = _tree(resp.content) | |
| results = [] | |
| for item in tree.xpath(f"//li[{_has_class('b_algo')}]"): | |
| title = _text(_first(item.xpath(".//h2"))) | |
| href = "" | |
| # The title link wraps the h2 in some layouts and sits beside it in others; | |
| # the site-icon link (a.tilk) always points at the page. | |
| for candidate in item.xpath(".//h2//a/@href | .//h2/ancestor::a[1]/@href | .//a[contains(@class, 'tilk')]/@href"): | |
| href = _bing_href(candidate) | |
| if href.startswith("http") and "bing.com/" not in href.split("?")[0]: | |
| break | |
| href = "" | |
| if not title or not href: | |
| continue | |
| body = _text(_first(item.xpath(f".//p | .//div[{_has_class('b_caption')}]"))) | |
| results.append({"title": title.rstrip("…").strip(), "href": href, "body": body}) | |
| if results: | |
| return results | |
| if b"captcha" in resp.content.lower() or b"challenge" in resp.content.lower(): | |
| raise Blocked("bing challenged the request") | |
| raise NoResults("bing returned no results") | |
| # --- Yahoo ------------------------------------------------------------------------------ | |
| def _yahoo_href(raw: str) -> str: | |
| if "/RU=" in raw: | |
| target = raw.split("/RU=", 1)[1] | |
| return unquote_plus(re.split(r"/R[KS]=", target, maxsplit=1)[0]) | |
| return raw | |
| def yahoo(query: str, page: int, timelimit: str | None, deadline: float) -> list[dict[str, str]]: | |
| params = {"p": query, "ei": "UTF-8"} | |
| if page > 1: | |
| params["b"] = str((page - 1) * 7 + 1) | |
| if timelimit: | |
| params["btf"] = timelimit | |
| resp = _client(deadline - time.monotonic()).get("https://search.yahoo.com/search", params=params) | |
| if resp.status_code != 200: | |
| raise Blocked(f"yahoo HTTP {resp.status_code}") | |
| tree = _tree(resp.content) | |
| results = [] | |
| # Yahoo's markup nests later results inside earlier ones, so take only the | |
| # first title, link and text of each block. | |
| for item in tree.xpath(f"//div[{_has_class('algo-sr')}]"): | |
| title_box = _first(item.xpath(".//div[contains(@class, 'compTitle')]")) | |
| if title_box is None: | |
| continue | |
| title = _text(_first(title_box.xpath(".//h3"))) | |
| href = _yahoo_href(_first(title_box.xpath(".//a/@href")) or "") | |
| if not title or not href.startswith("http") or "bing.com/aclick" in href: | |
| continue | |
| body = _text(_first(item.xpath(".//div[contains(@class, 'compText')]"))) | |
| results.append({"title": title, "href": href, "body": body}) | |
| if results: | |
| return results | |
| raise NoResults("yahoo returned no results") | |
| # --- Wikipedia (API) --------------------------------------------------------------------- | |
| def wikipedia(query: str, page: int, timelimit: str | None, deadline: float) -> list[dict[str, str]]: # noqa: ARG001 | |
| params = { | |
| "action": "query", "list": "search", "srsearch": query, "format": "json", | |
| "srlimit": "10", "sroffset": str((page - 1) * 10), "utf8": "1", | |
| } | |
| resp = _client(deadline - time.monotonic()).get("https://en.wikipedia.org/w/api.php", params=params) | |
| if resp.status_code != 200: | |
| raise Blocked(f"wikipedia HTTP {resp.status_code}") | |
| hits = (resp.json().get("query") or {}).get("search") or [] | |
| results = [] | |
| for hit in hits: | |
| title = str(hit.get("title") or "") | |
| snippet = re.sub(r"<[^>]+>", "", str(hit.get("snippet") or "")) | |
| results.append({ | |
| "title": title, | |
| "href": "https://en.wikipedia.org/wiki/" + title.replace(" ", "_"), | |
| "body": " ".join(lxml_html.fromstring(f"<p>{snippet}</p>").text_content().split()) if snippet else "", | |
| }) | |
| if results: | |
| return results | |
| raise NoResults("wikipedia returned no results") | |
| TEXT_ENGINES: dict[str, Callable[[str, int, str | None, float], list[dict[str, str]]]] = { | |
| "duckduckgo": duckduckgo, | |
| "bing": bing, | |
| "yahoo": yahoo, | |
| "wikipedia": wikipedia, | |
| } | |
| # --- Relevance ---------------------------------------------------------------------------- | |
| _STOP = { | |
| "a", "an", "the", "in", "on", "of", "for", "to", "and", "with", "at", "by", "from", "is", "are", | |
| "what", "how", "why", "when", "where", "who", "which", "does", "do", "can", "near", "about", "best", | |
| "vs", "or", "my", "your", "i", "you", "it", "be", "this", "that", | |
| } | |
| def query_terms(query: str) -> list[str]: | |
| words = [] | |
| for word in query.lower().split(): | |
| if re.match(r"^-|^\w+:", word): | |
| continue | |
| word = re.sub(r"[^\w]", "", word) | |
| if len(word) >= 3 and word not in _STOP: | |
| words.append(word) | |
| return words | |
| def looks_relevant(query: str, results: list[dict[str, Any]], sample: int = 5) -> bool: | |
| """False when the first results do not look like answers to the query. | |
| Throttled engines sometimes answer with a page of unrelated results (a | |
| makeup search returning bank login pages, a wig search returning Best Buy). | |
| A query word matches on its first five letters, so "walking" matches "walk". | |
| A result counts when it mentions one query word (two, for queries of three | |
| or more words), and at least two of the first results must count. Queries | |
| with no usable words pass. | |
| """ | |
| stems = {t[:5] for t in query_terms(query)} | |
| if not stems: | |
| return True | |
| need = 1 if len(stems) <= 2 else 2 | |
| top = results[:sample] | |
| good = 0 | |
| for item in top: | |
| hay = " ".join(str(item.get(k) or "") for k in ("title", "body", "href", "url")).lower() | |
| if sum(1 for stem in stems if stem in hay) >= need: | |
| good += 1 | |
| return good >= min(2, len(top)) | |