Spaces:
Sleeping
Sleeping
File size: 6,333 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 | from typing import Dict, Any, Union, List, TypeVar, Type
from ...types import (
SearchRequest,
SearchData,
Document,
SearchResultWeb,
SearchResultNews,
SearchResultImages,
)
from ...utils.http_client_async import AsyncHttpClient
from ...utils.error_handler import handle_response_error
from ...utils.normalize import normalize_document_input
from ...utils.validation import validate_scrape_options, prepare_scrape_options
T = TypeVar("T")
async def search(
client: AsyncHttpClient,
request: SearchRequest
) -> SearchData:
"""
Async search for documents.
Args:
client: Async HTTP client instance
request: Search request
Returns:
SearchData with search results grouped by source type
Raises:
FirecrawlError: If the search operation fails
"""
request_data = _prepare_search_request(request)
try:
response = await client.post("/v2/search", request_data)
if response.status_code != 200:
handle_response_error(response, "search")
response_data = response.json()
if not response_data.get("success"):
handle_response_error(response, "search")
data = response_data.get("data", {}) or {}
out = SearchData()
if "web" in data:
out.web = _transform_array(data["web"], SearchResultWeb)
if "news" in data:
out.news = _transform_array(data["news"], SearchResultNews)
if "images" in data:
out.images = _transform_array(data["images"], SearchResultImages)
return out
except Exception as err:
if hasattr(err, "response"):
handle_response_error(getattr(err, "response"), "search")
raise err
def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, Document]]:
"""
Transforms an array of items into a list of result_type or Document.
If the item dict contains any of the special keys, it is treated as a Document.
Otherwise, it is treated as result_type.
If the item is not a dict, it is wrapped as result_type with url=item.
"""
results: List[Union[T, Document]] = []
for item in arr:
if item and isinstance(item, dict):
if (
"markdown" in item or
"html" in item or
"rawHtml" in item or
"links" in item or
"screenshot" in item or
"changeTracking" in item or
"summary" in item or
"json" in item
):
results.append(Document(**normalize_document_input(item)))
else:
results.append(result_type(**item))
else:
results.append(result_type(url=item))
return results
def _validate_search_request(request: SearchRequest) -> SearchRequest:
"""
Validate and normalize search request.
Args:
request: Search request to validate
Returns:
Validated request
Raises:
ValueError: If request is invalid
"""
if not request.query or not request.query.strip():
raise ValueError("Query cannot be empty")
if request.limit is not None:
if request.limit <= 0:
raise ValueError("Limit must be positive")
if request.limit > 100:
raise ValueError("Limit cannot exceed 100")
if request.timeout is not None:
if request.timeout <= 0:
raise ValueError("Timeout must be positive")
if request.timeout > 300000:
raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
if request.sources is not None:
valid_sources = {"web", "news", "images"}
for source in request.sources:
if isinstance(source, str):
if source not in valid_sources:
raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
elif hasattr(source, 'type'):
if source.type not in valid_sources:
raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
if request.include_domains and request.exclude_domains:
raise ValueError(
"include_domains and exclude_domains cannot both be specified"
)
if request.location is not None:
if not isinstance(request.location, str) or len(request.location.strip()) == 0:
raise ValueError("Location must be a non-empty string")
if request.tbs is not None:
if not isinstance(request.tbs, str) or len(request.tbs.strip()) == 0:
raise ValueError("tbs must be a non-empty string")
if request.scrape_options is not None:
validate_scrape_options(request.scrape_options)
return request
def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
"""
Prepare a search request payload.
Args:
request: Search request
Returns:
Request payload dictionary
"""
validated_request = _validate_search_request(request)
data = validated_request.model_dump(exclude_none=True, by_alias=True)
if "limit" not in data and validated_request.limit is not None:
data["limit"] = validated_request.limit
if "timeout" not in data and validated_request.timeout is not None:
data["timeout"] = validated_request.timeout
if validated_request.ignore_invalid_urls is not None:
data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
data.pop("ignore_invalid_urls", None)
if validated_request.include_domains is not None:
data["includeDomains"] = validated_request.include_domains
data.pop("include_domains", None)
if validated_request.exclude_domains is not None:
data["excludeDomains"] = validated_request.exclude_domains
data.pop("exclude_domains", None)
if validated_request.scrape_options is not None:
scrape_data = prepare_scrape_options(validated_request.scrape_options)
if scrape_data:
data["scrapeOptions"] = scrape_data
data.pop("scrape_options", None)
if (v := getattr(validated_request, "integration", None)) is not None and str(v).strip():
data["integration"] = str(validated_request.integration).strip()
return data
|