dheraingoud's picture
Copilot App
Sync upstream v6.19.35 and preserve custom addons
a9e27c0
Raw History Blame Contribute Delete
8.73 kB
from __future__ import annotations
"""Outbound HTTP for web_search / web_fetch with operation-scoped resources."""
import asyncio
import codecs
import socket
from collections.abc import AsyncIterator
from urllib.parse import urljoin, urlparse
import aiohttp
import httpx
from aiohttp import ClientSession, ClientTimeout, TCPConnector
from aiohttp.abc import AbstractResolver, ResolveResult
from free_claude_code.application.web_tools.ports import (
WebFetchEgressPolicy,
WebFetchEgressViolation,
)
from free_claude_code.core.web_tools import WebFetchResult, WebSearchResult
from . import constants
from .constants import (
_MAX_FETCH_CHARS,
_MAX_SEARCH_RESULTS,
_REDIRECT_RESPONSE_BODY_CAP_BYTES,
_REQUEST_TIMEOUT_S,
_WEB_FETCH_REDIRECT_STATUSES,
_WEB_TOOL_HTTP_HEADERS,
)
from .egress import (
get_validated_stream_addrinfos_for_egress,
)
from .parsers import HTMLTextParser, SearchResultParser
async def _iter_response_body_under_cap(
response: httpx.Response, max_bytes: int
) -> AsyncIterator[bytes]:
if max_bytes <= 0:
return
received = 0
async for chunk in response.aiter_bytes(chunk_size=65_536):
if received >= max_bytes:
break
remaining = max_bytes - received
if len(chunk) <= remaining:
received += len(chunk)
yield chunk
if received >= max_bytes:
break
else:
yield chunk[:remaining]
break
async def _drain_response_body_capped(response: httpx.Response, max_bytes: int) -> None:
async for _ in _iter_response_body_under_cap(response, max_bytes):
pass
async def _read_response_body_capped(response: httpx.Response, max_bytes: int) -> bytes:
return b"".join(
[piece async for piece in _iter_response_body_under_cap(response, max_bytes)]
)
_NUMERIC_RESOLVE_FLAGS = socket.AI_NUMERICHOST | socket.AI_NUMERICSERV
_NAME_RESOLVE_FLAGS = socket.NI_NUMERICHOST | socket.NI_NUMERICSERV
def getaddrinfo_rows_to_resolve_results(
host: str, addrinfos: list[tuple]
) -> list[ResolveResult]:
"""Map :func:`socket.getaddrinfo` rows to aiohttp :class:`ResolveResult` (ThreadedResolver logic)."""
out: list[ResolveResult] = []
for family, _type, proto, _canon, sockaddr in addrinfos:
if family == socket.AF_INET6:
if len(sockaddr) < 3:
continue
if sockaddr[3]:
resolved_host, port = socket.getnameinfo(sockaddr, _NAME_RESOLVE_FLAGS)
else:
resolved_host, port = sockaddr[:2]
else:
assert family == socket.AF_INET, family
resolved_host, port = sockaddr[0], sockaddr[1]
resolved_host = str(resolved_host)
port = int(port)
out.append(
ResolveResult(
hostname=host,
host=resolved_host,
port=int(port),
family=family,
proto=proto,
flags=_NUMERIC_RESOLVE_FLAGS,
)
)
return out
class _PinnedEgressStaticResolver(AbstractResolver):
"""Return only pre-validated :class:`ResolveResult` for the outbound request."""
def __init__(self, results: list[ResolveResult]) -> None:
self._results = results
async def resolve(
self, host: str, port: int = 0, family: int = socket.AF_INET
) -> list[ResolveResult]:
return self._results
async def close(self) -> None: # pragma: no cover - aiohttp contract
return
async def _read_aiohttp_body_capped(
response: aiohttp.ClientResponse, max_bytes: int
) -> bytes:
received = 0
parts: list[bytes] = []
async for chunk in response.content.iter_chunked(65_536):
if received >= max_bytes:
break
remaining = max_bytes - received
if len(chunk) <= remaining:
received += len(chunk)
parts.append(chunk)
else:
parts.append(chunk[:remaining])
break
return b"".join(parts)
async def _drain_aiohttp_body_capped(
response: aiohttp.ClientResponse, max_bytes: int
) -> None:
if max_bytes <= 0:
return
received = 0
async for chunk in response.content.iter_chunked(65_536):
received += len(chunk)
if received >= max_bytes:
break
def _text_encoding(charset: str | None) -> str:
"""Return a usable codec name from a Content-Type charset, defaulting to utf-8."""
if isinstance(charset, str) and charset:
try:
return codecs.lookup(charset).name
except (LookupError, ValueError):
pass
return "utf-8"
class HTTPWebToolsClient:
"""Stateless adapter; each call owns its HTTP resources until completion."""
async def search(self, query: str) -> list[WebSearchResult]:
async with (
httpx.AsyncClient(
timeout=_REQUEST_TIMEOUT_S,
follow_redirects=True,
headers=_WEB_TOOL_HTTP_HEADERS,
) as client,
client.stream(
"GET",
"https://lite.duckduckgo.com/lite/",
params={"q": query},
) as response,
):
response.raise_for_status()
body_bytes = await _read_response_body_capped(
response, constants._MAX_WEB_FETCH_RESPONSE_BYTES
)
text = body_bytes.decode("utf-8", errors="replace")
parser = SearchResultParser()
parser.feed(text)
return parser.results[:_MAX_SEARCH_RESULTS]
async def fetch(self, url: str, *, egress: WebFetchEgressPolicy) -> WebFetchResult:
"""Fetch URL with manual redirects; each hop is DNS-pinned to validated addresses."""
current_url = url
redirect_hops = 0
timeout = ClientTimeout(total=_REQUEST_TIMEOUT_S)
while True:
addr_infos = await asyncio.to_thread(
get_validated_stream_addrinfos_for_egress, current_url, egress
)
host = urlparse(current_url).hostname or ""
results = getaddrinfo_rows_to_resolve_results(host, addr_infos)
resolver = _PinnedEgressStaticResolver(results)
connector = TCPConnector(
resolver=resolver,
force_close=True,
)
try:
async with (
ClientSession(
timeout=timeout,
headers=_WEB_TOOL_HTTP_HEADERS,
connector=connector,
) as session,
session.get(current_url, allow_redirects=False) as response,
):
if response.status in _WEB_FETCH_REDIRECT_STATUSES:
await _drain_aiohttp_body_capped(
response, _REDIRECT_RESPONSE_BODY_CAP_BYTES
)
if redirect_hops >= constants._MAX_WEB_FETCH_REDIRECTS:
raise WebFetchEgressViolation(
"web_fetch exceeded maximum redirects "
f"({constants._MAX_WEB_FETCH_REDIRECTS})"
)
location = response.headers.get("location")
if not location or not location.strip():
raise WebFetchEgressViolation(
"web_fetch redirect response missing Location header"
)
current_url = urljoin(str(response.url), location.strip())
redirect_hops += 1
continue
response.raise_for_status()
content_type = response.headers.get("content-type", "text/plain")
final_url = str(response.url)
encoding = _text_encoding(response.charset)
body_bytes = await _read_aiohttp_body_capped(
response, constants._MAX_WEB_FETCH_RESPONSE_BYTES
)
finally:
await connector.close()
break
text = body_bytes.decode(encoding, errors="replace")
title = final_url
data = text
if "html" in content_type.lower():
parser = HTMLTextParser()
parser.feed(text)
title = parser.title or final_url
data = "\n".join(parser.text_parts)
return WebFetchResult(
url=final_url,
title=title,
media_type="text/plain",
data=data[:_MAX_FETCH_CHARS],
)