File size: 4,953 Bytes
3e19b1e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Observe listing parts independently and retain only dates supported by recorded changes."""
import hashlib
import json
from core import amazon_pdp as pdp
TABLE = 'amzn_listing_items'
DATE_KEYS = tuple(part + '_last_updated' for part in (*pdp.OBSERVED_PARTS, 'aplus'))
IMAGE_ATTRIBUTES = {'MAIN': 'main_product_image_locator',
**{f'PT{i:02}': f'other_product_image_locator_{i}' for i in range(1, 9)},
'VIDEO': 'product_video_locator'}
def _decoded(raw, default):
value = json.loads(raw) if isinstance(raw, str) else raw
return value if isinstance(value, type(default)) else default
def _values(attributes, name, marketplace):
rows = attributes.get(name)
return [row.get('value', row.get('media_location')) for row in rows if isinstance(row, dict)
and row.get('marketplace_id', marketplace) == marketplace] if isinstance(rows, list) else []
def listing_parts(attributes, images, marketplace):
"""Select marketplace-specific title, ordered bullets and gallery slots without A+ samples."""
parts = {}
for part, name in (('title', 'item_name'), ('bullets', 'bullet_point')):
if isinstance(attributes.get(name), list):
values = _values(attributes, name, marketplace)
parts[part] = (values[0] if values else None) if part == 'title' else values
gallery = dict.fromkeys(IMAGE_ATTRIBUTES)
images_available = any(isinstance(attributes.get(name), list) for name in IMAGE_ATTRIBUTES.values())
for slot, name in IMAGE_ATTRIBUTES.items():
values = _values(attributes, name, marketplace)
if name in attributes:
gallery[slot] = values[0] if values else None
for group in images:
if not isinstance(group, dict):
continue
if group.get('marketplaceId', marketplace) == marketplace:
for image in group.get('images', [group]):
if not isinstance(image, dict):
continue
slot = image.get('variant')
if slot in IMAGE_ATTRIBUTES and image.get('link'):
gallery[slot] = gallery.get(slot) or image['link']
images_available = True
if images_available:
parts['images'] = gallery
return parts
def _digest(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'),
ensure_ascii=False).encode('utf-8')).hexdigest()
def part_hashes(parts):
"""Hash each part's content without borrowing another part's identity."""
return {part: _digest(value) for part, value in parts.items()}
def snapshots(con):
"""Read every listing and combine its SKU and marketplace identities deterministically per ASIN."""
columns = {row[0] for row in con.execute(
'SELECT column_name FROM information_schema.columns WHERE table_name = ?', [TABLE]).fetchall()}
if not {'sku', 'attributes', 'summaries'} <= columns:
return {}
images_column = 'images' if 'images' in columns else 'NULL'
cursor = con.execute(f'SELECT sku, attributes, summaries, {images_column} FROM {TABLE}')
listings = {}
for batch in iter(lambda: cursor.fetchmany(2048), []):
for sku, raw_attributes, raw_summaries, raw_images in batch:
attributes = _decoded(raw_attributes, {})
images = _decoded(raw_images, [])
for summary in _decoded(raw_summaries, []):
if not isinstance(summary, dict) or not summary.get('asin') or not attributes:
continue
identity = (str(summary['asin']).strip().upper(), str(sku or ''), summary.get('marketplaceId') or '')
parts = listing_parts(attributes, images, summary.get('marketplaceId'))
candidate = (summary.get('lastUpdatedDate') or '', _digest(parts), parts)
if identity not in listings or candidate[:2] > listings[identity][:2]:
listings[identity] = candidate
content = {}
for (asin, sku, marketplace), (_, _, parts) in sorted(listings.items()):
for part, value in parts.items():
entry = [marketplace, value]
content.setdefault(asin, {}).setdefault(part, {})[_digest(entry)] = entry
return {asin: part_hashes({part: [values[key] for key in sorted(values)] for part, values in parts.items()})
for asin, parts in content.items()}
def observe(rt, con, *, day):
"""Persist a daily baseline or changed part, including ASINs absent from profit rows."""
observed = snapshots(con)
return {'asins': len(observed), 'hashes': pdp.observe_part_hashes(rt, observed, day)}
def cells(rt, asins):
"""Project recorded dates and clear unsupported sample dates without observing or writing."""
recorded = pdp.part_dates(rt, asins)
return {asin: {key: recorded.get(asin, {}).get(key) for key in DATE_KEYS} for asin in asins}
|