File size: 4,173 Bytes
097eaf0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
"""
Normalization helpers for v2 API payloads to avoid relying on Pydantic aliases.
"""

from typing import Any, Dict, List
from ..types import DocumentMetadata


def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:
    """
    Convert API v2 camelCase metadata keys to snake_case expected by DocumentMetadata.
    Leaves unknown keys as-is.
    """
    mapping = {
        # OpenGraph
        "ogTitle": "og_title",
        "ogDescription": "og_description",
        "ogUrl": "og_url",
        "ogImage": "og_image",
        "ogAudio": "og_audio",
        "ogDeterminer": "og_determiner",
        "ogLocale": "og_locale",
        "ogLocaleAlternate": "og_locale_alternate",
        "ogSiteName": "og_site_name",
        "ogVideo": "og_video",
        # Dublin Core and misc
        "dcTermsCreated": "dc_terms_created",
        "dcDateCreated": "dc_date_created",
        "dcDate": "dc_date",
        "dcTermsType": "dc_terms_type",
        "dcType": "dc_type",
        "dcTermsAudience": "dc_terms_audience",
        "dcTermsSubject": "dc_terms_subject",
        "dcSubject": "dc_subject",
        "dcDescription": "dc_description",
        "dcTermsKeywords": "dc_terms_keywords",
        "modifiedTime": "modified_time",
        "publishedTime": "published_time",
        "articleTag": "article_tag",
        "articleSection": "article_section",
        # Response-level
        "sourceURL": "source_url",
        "statusCode": "status_code",
        "scrapeId": "scrape_id",
        "numPages": "num_pages",
        "contentType": "content_type",
        "proxyUsed": "proxy_used",
        "cacheState": "cache_state",
        "cachedAt": "cached_at",
        "creditsUsed": "credits_used",
        "concurrencyLimited": "concurrency_limited",
        "concurrencyQueueDurationMs": "concurrency_queue_duration_ms",
    }

    out: Dict[str, Any] = {}
    for k, v in md.items():
        snake = mapping.get(k, k)
        out[snake] = v

    # Light coercions where server may send strings/lists
    if isinstance(out.get("status_code"), str):
        try:
            out["status_code"] = int(out["status_code"])  # type: ignore
        except ValueError:
            pass

    # Preserve list values for unknown keys; only lightweight coercions above
    return out


def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:
    """
    Normalize a raw Document dict from the API into the Python SDK's expected shape:
    - Convert top-level keys rawHtml->raw_html, changeTracking->change_tracking
    - Convert metadata keys from camelCase to snake_case
    - Convert branding.colorScheme to branding.color_scheme
    """
    normalized = dict(doc)

    if "rawHtml" in normalized and "raw_html" not in normalized:
        normalized["raw_html"] = normalized.pop("rawHtml")

    if "changeTracking" in normalized and "change_tracking" not in normalized:
        normalized["change_tracking"] = normalized.pop("changeTracking")

    md = normalized.get("metadata")
    if isinstance(md, dict):
        mapped = _map_metadata_keys(md)
        # Construct a typed DocumentMetadata; extras allowed/preserved
        try:
            normalized["metadata"] = DocumentMetadata.model_validate(mapped)
        except Exception:
            normalized["metadata"] = mapped

    # Normalize branding top-level camelCase keys
    branding = normalized.get("branding")
    if isinstance(branding, dict):
        if "colorScheme" in branding and "color_scheme" not in branding:
            branding["color_scheme"] = branding.pop("colorScheme")

    return normalized


def _map_search_result_keys(result: Dict[str, Any], result_type: str) -> Dict[str, Any]:
    if result_type == "images":
        mapping = {
            "imageUrl": "image_url",
            "imageWidth": "image_width",
            "imageHeight": "image_height",
        }
    elif result_type == "news":
        mapping = {
            "imageUrl": "image_url",
        }
    elif result_type == "web":
        mapping = {}
    else:
        mapping = {}

    out: Dict[str, Any] = {}
    for k, v in result.items():
        snake = mapping.get(k, k)
        out[snake] = v

    return out