Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 11,773 Bytes
8729c1d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 | """Tenant-store listing changes shared by signed notifications and daily snapshots."""
from __future__ import annotations
import copy
import hashlib
import json
from contextlib import closing
from dataclasses import dataclass
from datetime import date, datetime, timezone
KEY = "amzn_listing_changes"
_ATTRIBUTE_NAMES = {"title": "item_name", "bullet_points": "bullet_point",
"description": "product_description"}
class _Replay(Exception):
pass
def normalize_attribute(attribute):
"""Give both writers the same Amazon attribute identity."""
if not isinstance(attribute, str) or not attribute.strip():
raise ValueError("A listing attribute name is required.")
name = attribute.strip().lower()
return _ATTRIBUTE_NAMES.get(name, name)
def _utc(value):
if isinstance(value, datetime):
stamp = value
elif isinstance(value, date):
stamp = datetime.combine(value, datetime.min.time())
elif isinstance(value, str):
stamp = datetime.fromisoformat(value.replace("Z", "+00:00"))
else:
raise ValueError("A listing change timestamp is required.")
return (stamp.replace(tzinfo=timezone.utc) if stamp.tzinfo is None else stamp).astimezone(timezone.utc)
def _text(value):
return value.strip() if isinstance(value, str) else ""
def _runtime(rt):
if rt is None or not getattr(rt, "key", None):
raise ValueError("A tenant runtime is required.")
return rt
def record(rt, change=None, **fields):
"""Atomically record tenant attributes and replay receipts, returning the inserted row count."""
_runtime(rt)
change = {**(change or {}), **fields}
source = change.get("source")
if source not in {"notification", "snapshot"}:
raise ValueError("A listing change source is required.")
asin, sku = _text(change.get("asin")), _text(change.get("sku"))
marketplace = _text(change.get("marketplace_id") or change.get("marketplaceId"))
if not (asin or sku) or not marketplace or (source == "notification" and not asin):
raise ValueError("A listing identity and marketplace are required.")
notification_id = _text(change.get("notificationId"))
if source == "notification" and not notification_id:
raise ValueError("A notification ID is required.")
detected_at = _utc(change.get("publishedAt") if source == "notification" else change.get("detected_at")).isoformat()
batch = source == "snapshot" and "attributesChanged" in change
attributes = change.get("attributesChanged") if source == "notification" or batch else [change.get("attribute")]
if not isinstance(attributes, list) or not attributes:
raise ValueError("Changed listing attributes are required.")
attributes = list(dict.fromkeys(normalize_attribute(attribute) for attribute in attributes))
values = {}
if batch:
# A snapshot batch expands to the same C4 rows as the single-attribute writer.
for key in ("before_values", "after_values"):
if not isinstance(change.get(key), dict):
raise ValueError("Snapshot attribute values are required.")
values[key] = {normalize_attribute(name): value for name, value in change[key].items()}
if any(attribute not in values[key] for attribute in attributes):
raise ValueError("Every snapshot attribute requires before and after values.")
incoming = [{"sku": sku or None, "asin": asin or None, "marketplace_id": marketplace,
"attribute": attribute, "before_value": (values["before_values"][attribute] if batch else
None if source == "notification" else change.get("before_value")),
"after_value": (values["after_values"][attribute] if batch else
None if source == "notification" else change.get("after_value")),
"detected_at": detected_at, "source": source,
"values_source": "snapshot" if source == "snapshot" else None,
"notificationId": notification_id or None,
"publishedAt": detected_at if source == "notification" else None,
"attributesChanged": attributes if source == "notification" or batch else [attribute]}
for attribute in attributes]
inserted = [0]
def save(bucket):
inserted[0] = 0
receipts = bucket.setdefault("notificationIds", {})
if source == "notification" and notification_id in receipts:
# Abort the transaction so a replay performs no durable write.
raise _Replay
rows = bucket.setdefault("rows", {})
for row in incoming:
# Different notification IDs remain separate changes, even within one day.
identity = [notification_id, row["attribute"]] if source == "notification" else row
row_id = hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
if row_id in rows:
continue
# Snapshots can fill values absent from a notification for the same attribute and UTC day.
counterpart = next((saved for saved in reversed(rows.values())
if saved["source"] != source and asin and saved["asin"] == asin
and saved["marketplace_id"] == marketplace and saved["attribute"] == row["attribute"]
and saved["detected_at"][:10] == detected_at[:10]
and (not sku or not saved.get("sku") or saved["sku"] == sku)
and (source != "snapshot" or saved.get("values_source") is None
or (saved["before_value"], saved["after_value"])
== (row["before_value"], row["after_value"]))), None)
if counterpart is not None:
if source == "snapshot":
counterpart.update({key: row[key] for key in ("sku", "before_value", "after_value", "values_source")})
else:
counterpart.update({key: row[key] for key in
("source", "notificationId", "publishedAt", "attributesChanged")})
continue
rows[row_id] = row
inserted[0] += 1
if source == "notification":
receipts[notification_id] = detected_at
return bucket
try:
rt.update(KEY, save)
except _Replay:
return 0
return inserted[0]
def _matching_changes(rows, asins, skus, since, mapped_asins=(), mapped_skus=()):
lower = _utc(since) if since is not None else None
return sorted((copy.deepcopy(row) for row in rows.values()
if (row.get("asin") in asins or row.get("sku") in skus
or (not row.get("sku") and row.get("asin") in mapped_asins)
or (not row.get("asin") and row.get("sku") in mapped_skus))
and (lower is None or _utc(row["detected_at"]) >= lower)),
key=lambda row: row["detected_at"])
def changes_for(rt, asins=None, since=None, *, skus=None):
"""Read all matching tenant changes by ASIN or SKU, with an inclusive UTC lower bound."""
_runtime(rt)
if asins is None and skus is None:
raise ValueError("An ASIN or SKU selector is required.")
wanted_asins = {_text(value) for value in ([asins] if isinstance(asins, str) else asins or [])}
wanted_skus = {_text(value) for value in ([skus] if isinstance(skus, str) else skus or [])}
rows = (rt.get(KEY, fresh=True) or {}).get("rows", {})
return _matching_changes(rows, wanted_asins, wanted_skus, since)
def _target_selectors(targets):
if targets is None:
raise ValueError("Listing targets are required.")
selected = {"sku": set(), "asin": set(), "parent_asin": set()}
for target in [targets] if isinstance(targets, (str, dict)) else targets:
if not isinstance(target, (str, dict)):
raise ValueError("A listing target requires a grain and key.")
grain, key = ("sku", target) if isinstance(target, str) else (target.get("grain"), target.get("key"))
grain = "parent_asin" if grain == "parent" else grain
key = _text(key)
if grain not in selected or not key:
raise ValueError("A listing target requires a sku, asin or parent_asin grain and key.")
selected[grain].add(key)
return selected
def _resolve_targets(rt, selected):
from harness.amazon_store import _columns_of, _latest_account_snapshot_sql, asin_parents_sql
from harness.runtime import mirror_cursor
asins, skus, mapped_asins = set(selected["asin"]), set(selected["sku"]), set()
mapped_skus, family_asins = set(), set()
parents = selected["parent_asin"]
if not asins and not skus and not parents:
return asins, skus, mapped_asins, mapped_skus
with closing(mirror_cursor(rt, require_ready=False)) as cur:
columns = _columns_of(cur, "amzn_listings")
current_listings = _latest_account_snapshot_sql(columns, "amzn_listings")
if skus and {"seller_sku", "asin1"} <= columns:
result = cur.execute("SELECT DISTINCT TRIM(CAST(asin1 AS VARCHAR)) "
f"FROM ({current_listings}) AS current_listings "
"WHERE TRIM(CAST(seller_sku AS VARCHAR)) IN (SELECT UNNEST(?)) "
"AND NULLIF(TRIM(CAST(asin1 AS VARCHAR)), '') IS NOT NULL",
[sorted(skus)])
while batch := result.fetchmany(512):
mapped_asins.update(row[0] for row in batch)
if parents:
sql = asin_parents_sql(cur)
if sql is not None:
result = cur.execute(f"SELECT asin, parent_asin FROM ({sql}) AS families "
"WHERE parent_asin IN (SELECT UNNEST(?))", [sorted(parents)])
while batch := result.fetchmany(512):
for asin, parent in batch:
family_asins.update((asin, parent))
asins.update(family_asins)
if asins and {"seller_sku", "asin1"} <= columns:
result = cur.execute("SELECT DISTINCT TRIM(CAST(seller_sku AS VARCHAR)), TRIM(CAST(asin1 AS VARCHAR)) "
f"FROM ({current_listings}) AS current_listings "
"WHERE TRIM(CAST(asin1 AS VARCHAR)) IN (SELECT UNNEST(?)) "
"AND NULLIF(TRIM(CAST(seller_sku AS VARCHAR)), '') IS NOT NULL", [sorted(asins)])
while batch := result.fetchmany(512):
for sku, asin in batch:
mapped_skus.add(sku)
if asin in family_asins:
skus.add(sku)
return asins, skus, mapped_asins, mapped_skus
@dataclass(frozen=True)
class _RuntimeReader:
_rt: object
def changes_for(self, targets, since=None):
"""Read project target changes from this tenant, closing the mirror cursor before returning."""
selected = _target_selectors(targets)
rows = (self._rt.get(KEY, fresh=True) or {}).get("rows", {})
if not rows or not any(selected.values()):
return _matching_changes({}, (), (), since)
asins, skus, mapped_asins, mapped_skus = _resolve_targets(self._rt, selected)
return _matching_changes(rows, asins, skus, since, mapped_asins, mapped_skus)
def for_runtime(rt):
"""Bind project change reads to the exact tenant runtime without retaining a mirror cursor."""
return _RuntimeReader(_runtime(rt))
|