runloopable / platform /harness /amazon_listing_changes.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
8729c1d verified
Raw History Blame Contribute Delete
11.8 kB
"""Tenant-store listing changes shared by signed notifications and daily snapshots."""
from __future__ import annotations
import copy
import hashlib
import json
from contextlib import closing
from dataclasses import dataclass
from datetime import date, datetime, timezone
KEY = "amzn_listing_changes"
_ATTRIBUTE_NAMES = {"title": "item_name", "bullet_points": "bullet_point",
"description": "product_description"}
class _Replay(Exception):
pass
def normalize_attribute(attribute):
"""Give both writers the same Amazon attribute identity."""
if not isinstance(attribute, str) or not attribute.strip():
raise ValueError("A listing attribute name is required.")
name = attribute.strip().lower()
return _ATTRIBUTE_NAMES.get(name, name)
def _utc(value):
if isinstance(value, datetime):
stamp = value
elif isinstance(value, date):
stamp = datetime.combine(value, datetime.min.time())
elif isinstance(value, str):
stamp = datetime.fromisoformat(value.replace("Z", "+00:00"))
else:
raise ValueError("A listing change timestamp is required.")
return (stamp.replace(tzinfo=timezone.utc) if stamp.tzinfo is None else stamp).astimezone(timezone.utc)
def _text(value):
return value.strip() if isinstance(value, str) else ""
def _runtime(rt):
if rt is None or not getattr(rt, "key", None):
raise ValueError("A tenant runtime is required.")
return rt
def record(rt, change=None, **fields):
"""Atomically record tenant attributes and replay receipts, returning the inserted row count."""
_runtime(rt)
change = {**(change or {}), **fields}
source = change.get("source")
if source not in {"notification", "snapshot"}:
raise ValueError("A listing change source is required.")
asin, sku = _text(change.get("asin")), _text(change.get("sku"))
marketplace = _text(change.get("marketplace_id") or change.get("marketplaceId"))
if not (asin or sku) or not marketplace or (source == "notification" and not asin):
raise ValueError("A listing identity and marketplace are required.")
notification_id = _text(change.get("notificationId"))
if source == "notification" and not notification_id:
raise ValueError("A notification ID is required.")
detected_at = _utc(change.get("publishedAt") if source == "notification" else change.get("detected_at")).isoformat()
batch = source == "snapshot" and "attributesChanged" in change
attributes = change.get("attributesChanged") if source == "notification" or batch else [change.get("attribute")]
if not isinstance(attributes, list) or not attributes:
raise ValueError("Changed listing attributes are required.")
attributes = list(dict.fromkeys(normalize_attribute(attribute) for attribute in attributes))
values = {}
if batch:
# A snapshot batch expands to the same C4 rows as the single-attribute writer.
for key in ("before_values", "after_values"):
if not isinstance(change.get(key), dict):
raise ValueError("Snapshot attribute values are required.")
values[key] = {normalize_attribute(name): value for name, value in change[key].items()}
if any(attribute not in values[key] for attribute in attributes):
raise ValueError("Every snapshot attribute requires before and after values.")
incoming = [{"sku": sku or None, "asin": asin or None, "marketplace_id": marketplace,
"attribute": attribute, "before_value": (values["before_values"][attribute] if batch else
None if source == "notification" else change.get("before_value")),
"after_value": (values["after_values"][attribute] if batch else
None if source == "notification" else change.get("after_value")),
"detected_at": detected_at, "source": source,
"values_source": "snapshot" if source == "snapshot" else None,
"notificationId": notification_id or None,
"publishedAt": detected_at if source == "notification" else None,
"attributesChanged": attributes if source == "notification" or batch else [attribute]}
for attribute in attributes]
inserted = [0]
def save(bucket):
inserted[0] = 0
receipts = bucket.setdefault("notificationIds", {})
if source == "notification" and notification_id in receipts:
# Abort the transaction so a replay performs no durable write.
raise _Replay
rows = bucket.setdefault("rows", {})
for row in incoming:
# Different notification IDs remain separate changes, even within one day.
identity = [notification_id, row["attribute"]] if source == "notification" else row
row_id = hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
if row_id in rows:
continue
# Snapshots can fill values absent from a notification for the same attribute and UTC day.
counterpart = next((saved for saved in reversed(rows.values())
if saved["source"] != source and asin and saved["asin"] == asin
and saved["marketplace_id"] == marketplace and saved["attribute"] == row["attribute"]
and saved["detected_at"][:10] == detected_at[:10]
and (not sku or not saved.get("sku") or saved["sku"] == sku)
and (source != "snapshot" or saved.get("values_source") is None
or (saved["before_value"], saved["after_value"])
== (row["before_value"], row["after_value"]))), None)
if counterpart is not None:
if source == "snapshot":
counterpart.update({key: row[key] for key in ("sku", "before_value", "after_value", "values_source")})
else:
counterpart.update({key: row[key] for key in
("source", "notificationId", "publishedAt", "attributesChanged")})
continue
rows[row_id] = row
inserted[0] += 1
if source == "notification":
receipts[notification_id] = detected_at
return bucket
try:
rt.update(KEY, save)
except _Replay:
return 0
return inserted[0]
def _matching_changes(rows, asins, skus, since, mapped_asins=(), mapped_skus=()):
lower = _utc(since) if since is not None else None
return sorted((copy.deepcopy(row) for row in rows.values()
if (row.get("asin") in asins or row.get("sku") in skus
or (not row.get("sku") and row.get("asin") in mapped_asins)
or (not row.get("asin") and row.get("sku") in mapped_skus))
and (lower is None or _utc(row["detected_at"]) >= lower)),
key=lambda row: row["detected_at"])
def changes_for(rt, asins=None, since=None, *, skus=None):
"""Read all matching tenant changes by ASIN or SKU, with an inclusive UTC lower bound."""
_runtime(rt)
if asins is None and skus is None:
raise ValueError("An ASIN or SKU selector is required.")
wanted_asins = {_text(value) for value in ([asins] if isinstance(asins, str) else asins or [])}
wanted_skus = {_text(value) for value in ([skus] if isinstance(skus, str) else skus or [])}
rows = (rt.get(KEY, fresh=True) or {}).get("rows", {})
return _matching_changes(rows, wanted_asins, wanted_skus, since)
def _target_selectors(targets):
if targets is None:
raise ValueError("Listing targets are required.")
selected = {"sku": set(), "asin": set(), "parent_asin": set()}
for target in [targets] if isinstance(targets, (str, dict)) else targets:
if not isinstance(target, (str, dict)):
raise ValueError("A listing target requires a grain and key.")
grain, key = ("sku", target) if isinstance(target, str) else (target.get("grain"), target.get("key"))
grain = "parent_asin" if grain == "parent" else grain
key = _text(key)
if grain not in selected or not key:
raise ValueError("A listing target requires a sku, asin or parent_asin grain and key.")
selected[grain].add(key)
return selected
def _resolve_targets(rt, selected):
from harness.amazon_store import _columns_of, _latest_account_snapshot_sql, asin_parents_sql
from harness.runtime import mirror_cursor
asins, skus, mapped_asins = set(selected["asin"]), set(selected["sku"]), set()
mapped_skus, family_asins = set(), set()
parents = selected["parent_asin"]
if not asins and not skus and not parents:
return asins, skus, mapped_asins, mapped_skus
with closing(mirror_cursor(rt, require_ready=False)) as cur:
columns = _columns_of(cur, "amzn_listings")
current_listings = _latest_account_snapshot_sql(columns, "amzn_listings")
if skus and {"seller_sku", "asin1"} <= columns:
result = cur.execute("SELECT DISTINCT TRIM(CAST(asin1 AS VARCHAR)) "
f"FROM ({current_listings}) AS current_listings "
"WHERE TRIM(CAST(seller_sku AS VARCHAR)) IN (SELECT UNNEST(?)) "
"AND NULLIF(TRIM(CAST(asin1 AS VARCHAR)), '') IS NOT NULL",
[sorted(skus)])
while batch := result.fetchmany(512):
mapped_asins.update(row[0] for row in batch)
if parents:
sql = asin_parents_sql(cur)
if sql is not None:
result = cur.execute(f"SELECT asin, parent_asin FROM ({sql}) AS families "
"WHERE parent_asin IN (SELECT UNNEST(?))", [sorted(parents)])
while batch := result.fetchmany(512):
for asin, parent in batch:
family_asins.update((asin, parent))
asins.update(family_asins)
if asins and {"seller_sku", "asin1"} <= columns:
result = cur.execute("SELECT DISTINCT TRIM(CAST(seller_sku AS VARCHAR)), TRIM(CAST(asin1 AS VARCHAR)) "
f"FROM ({current_listings}) AS current_listings "
"WHERE TRIM(CAST(asin1 AS VARCHAR)) IN (SELECT UNNEST(?)) "
"AND NULLIF(TRIM(CAST(seller_sku AS VARCHAR)), '') IS NOT NULL", [sorted(asins)])
while batch := result.fetchmany(512):
for sku, asin in batch:
mapped_skus.add(sku)
if asin in family_asins:
skus.add(sku)
return asins, skus, mapped_asins, mapped_skus
@dataclass(frozen=True)
class _RuntimeReader:
_rt: object
def changes_for(self, targets, since=None):
"""Read project target changes from this tenant, closing the mirror cursor before returning."""
selected = _target_selectors(targets)
rows = (self._rt.get(KEY, fresh=True) or {}).get("rows", {})
if not rows or not any(selected.values()):
return _matching_changes({}, (), (), since)
asins, skus, mapped_asins, mapped_skus = _resolve_targets(self._rt, selected)
return _matching_changes(rows, asins, skus, since, mapped_asins, mapped_skus)
def for_runtime(rt):
"""Bind project change reads to the exact tenant runtime without retaining a mirror cursor."""
return _RuntimeReader(_runtime(rt))