Spaces:
Sleeping
Sleeping
Download apps/python-sdk/firecrawl/v2/utils/normalize.py from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 4.17 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/utils/normalize.py
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/python-sdk/firecrawl/v2/utils/normalize.py
-
curl -L -o normalize.py https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/utils/normalize.py
4.17 kB
| """ | |
| Normalization helpers for v2 API payloads to avoid relying on Pydantic aliases. | |
| """ | |
| from typing import Any, Dict, List | |
| from ..types import DocumentMetadata | |
| def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]: | |
| """ | |
| Convert API v2 camelCase metadata keys to snake_case expected by DocumentMetadata. | |
| Leaves unknown keys as-is. | |
| """ | |
| mapping = { | |
| # OpenGraph | |
| "ogTitle": "og_title", | |
| "ogDescription": "og_description", | |
| "ogUrl": "og_url", | |
| "ogImage": "og_image", | |
| "ogAudio": "og_audio", | |
| "ogDeterminer": "og_determiner", | |
| "ogLocale": "og_locale", | |
| "ogLocaleAlternate": "og_locale_alternate", | |
| "ogSiteName": "og_site_name", | |
| "ogVideo": "og_video", | |
| # Dublin Core and misc | |
| "dcTermsCreated": "dc_terms_created", | |
| "dcDateCreated": "dc_date_created", | |
| "dcDate": "dc_date", | |
| "dcTermsType": "dc_terms_type", | |
| "dcType": "dc_type", | |
| "dcTermsAudience": "dc_terms_audience", | |
| "dcTermsSubject": "dc_terms_subject", | |
| "dcSubject": "dc_subject", | |
| "dcDescription": "dc_description", | |
| "dcTermsKeywords": "dc_terms_keywords", | |
| "modifiedTime": "modified_time", | |
| "publishedTime": "published_time", | |
| "articleTag": "article_tag", | |
| "articleSection": "article_section", | |
| # Response-level | |
| "sourceURL": "source_url", | |
| "statusCode": "status_code", | |
| "scrapeId": "scrape_id", | |
| "numPages": "num_pages", | |
| "contentType": "content_type", | |
| "proxyUsed": "proxy_used", | |
| "cacheState": "cache_state", | |
| "cachedAt": "cached_at", | |
| "creditsUsed": "credits_used", | |
| "concurrencyLimited": "concurrency_limited", | |
| "concurrencyQueueDurationMs": "concurrency_queue_duration_ms", | |
| } | |
| out: Dict[str, Any] = {} | |
| for k, v in md.items(): | |
| snake = mapping.get(k, k) | |
| out[snake] = v | |
| # Light coercions where server may send strings/lists | |
| if isinstance(out.get("status_code"), str): | |
| try: | |
| out["status_code"] = int(out["status_code"]) # type: ignore | |
| except ValueError: | |
| pass | |
| # Preserve list values for unknown keys; only lightweight coercions above | |
| return out | |
| def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]: | |
| """ | |
| Normalize a raw Document dict from the API into the Python SDK's expected shape: | |
| - Convert top-level keys rawHtml->raw_html, changeTracking->change_tracking | |
| - Convert metadata keys from camelCase to snake_case | |
| - Convert branding.colorScheme to branding.color_scheme | |
| """ | |
| normalized = dict(doc) | |
| if "rawHtml" in normalized and "raw_html" not in normalized: | |
| normalized["raw_html"] = normalized.pop("rawHtml") | |
| if "changeTracking" in normalized and "change_tracking" not in normalized: | |
| normalized["change_tracking"] = normalized.pop("changeTracking") | |
| md = normalized.get("metadata") | |
| if isinstance(md, dict): | |
| mapped = _map_metadata_keys(md) | |
| # Construct a typed DocumentMetadata; extras allowed/preserved | |
| try: | |
| normalized["metadata"] = DocumentMetadata.model_validate(mapped) | |
| except Exception: | |
| normalized["metadata"] = mapped | |
| # Normalize branding top-level camelCase keys | |
| branding = normalized.get("branding") | |
| if isinstance(branding, dict): | |
| if "colorScheme" in branding and "color_scheme" not in branding: | |
| branding["color_scheme"] = branding.pop("colorScheme") | |
| return normalized | |
| def _map_search_result_keys(result: Dict[str, Any], result_type: str) -> Dict[str, Any]: | |
| if result_type == "images": | |
| mapping = { | |
| "imageUrl": "image_url", | |
| "imageWidth": "image_width", | |
| "imageHeight": "image_height", | |
| } | |
| elif result_type == "news": | |
| mapping = { | |
| "imageUrl": "image_url", | |
| } | |
| elif result_type == "web": | |
| mapping = {} | |
| else: | |
| mapping = {} | |
| out: Dict[str, Any] = {} | |
| for k, v in result.items(): | |
| snake = mapping.get(k, k) | |
| out[snake] = v | |
| return out | |