loopable / platform /modules /amazon_part_dates.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
3e19b1e verified
Raw History Blame Contribute Delete
4.95 kB
"""Observe listing parts independently and retain only dates supported by recorded changes."""
import hashlib
import json
from core import amazon_pdp as pdp
TABLE = 'amzn_listing_items'
DATE_KEYS = tuple(part + '_last_updated' for part in (*pdp.OBSERVED_PARTS, 'aplus'))
IMAGE_ATTRIBUTES = {'MAIN': 'main_product_image_locator',
**{f'PT{i:02}': f'other_product_image_locator_{i}' for i in range(1, 9)},
'VIDEO': 'product_video_locator'}
def _decoded(raw, default):
value = json.loads(raw) if isinstance(raw, str) else raw
return value if isinstance(value, type(default)) else default
def _values(attributes, name, marketplace):
rows = attributes.get(name)
return [row.get('value', row.get('media_location')) for row in rows if isinstance(row, dict)
and row.get('marketplace_id', marketplace) == marketplace] if isinstance(rows, list) else []
def listing_parts(attributes, images, marketplace):
"""Select marketplace-specific title, ordered bullets and gallery slots without A+ samples."""
parts = {}
for part, name in (('title', 'item_name'), ('bullets', 'bullet_point')):
if isinstance(attributes.get(name), list):
values = _values(attributes, name, marketplace)
parts[part] = (values[0] if values else None) if part == 'title' else values
gallery = dict.fromkeys(IMAGE_ATTRIBUTES)
images_available = any(isinstance(attributes.get(name), list) for name in IMAGE_ATTRIBUTES.values())
for slot, name in IMAGE_ATTRIBUTES.items():
values = _values(attributes, name, marketplace)
if name in attributes:
gallery[slot] = values[0] if values else None
for group in images:
if not isinstance(group, dict):
continue
if group.get('marketplaceId', marketplace) == marketplace:
for image in group.get('images', [group]):
if not isinstance(image, dict):
continue
slot = image.get('variant')
if slot in IMAGE_ATTRIBUTES and image.get('link'):
gallery[slot] = gallery.get(slot) or image['link']
images_available = True
if images_available:
parts['images'] = gallery
return parts
def _digest(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'),
ensure_ascii=False).encode('utf-8')).hexdigest()
def part_hashes(parts):
"""Hash each part's content without borrowing another part's identity."""
return {part: _digest(value) for part, value in parts.items()}
def snapshots(con):
"""Read every listing and combine its SKU and marketplace identities deterministically per ASIN."""
columns = {row[0] for row in con.execute(
'SELECT column_name FROM information_schema.columns WHERE table_name = ?', [TABLE]).fetchall()}
if not {'sku', 'attributes', 'summaries'} <= columns:
return {}
images_column = 'images' if 'images' in columns else 'NULL'
cursor = con.execute(f'SELECT sku, attributes, summaries, {images_column} FROM {TABLE}')
listings = {}
for batch in iter(lambda: cursor.fetchmany(2048), []):
for sku, raw_attributes, raw_summaries, raw_images in batch:
attributes = _decoded(raw_attributes, {})
images = _decoded(raw_images, [])
for summary in _decoded(raw_summaries, []):
if not isinstance(summary, dict) or not summary.get('asin') or not attributes:
continue
identity = (str(summary['asin']).strip().upper(), str(sku or ''), summary.get('marketplaceId') or '')
parts = listing_parts(attributes, images, summary.get('marketplaceId'))
candidate = (summary.get('lastUpdatedDate') or '', _digest(parts), parts)
if identity not in listings or candidate[:2] > listings[identity][:2]:
listings[identity] = candidate
content = {}
for (asin, sku, marketplace), (_, _, parts) in sorted(listings.items()):
for part, value in parts.items():
entry = [marketplace, value]
content.setdefault(asin, {}).setdefault(part, {})[_digest(entry)] = entry
return {asin: part_hashes({part: [values[key] for key in sorted(values)] for part, values in parts.items()})
for asin, parts in content.items()}
def observe(rt, con, *, day):
"""Persist a daily baseline or changed part, including ASINs absent from profit rows."""
observed = snapshots(con)
return {'asins': len(observed), 'hashes': pdp.observe_part_hashes(rt, observed, day)}
def cells(rt, asins):
"""Project recorded dates and clear unsupported sample dates without observing or writing."""
recorded = pdp.part_dates(rt, asins)
return {asin: {key: recorded.get(asin, {}).get(key) for key in DATE_KEYS} for asin in asins}