"""Observe listing parts independently and retain only dates supported by recorded changes.""" import hashlib import json from core import amazon_pdp as pdp TABLE = 'amzn_listing_items' DATE_KEYS = tuple(part + '_last_updated' for part in (*pdp.OBSERVED_PARTS, 'aplus')) IMAGE_ATTRIBUTES = {'MAIN': 'main_product_image_locator', **{f'PT{i:02}': f'other_product_image_locator_{i}' for i in range(1, 9)}, 'VIDEO': 'product_video_locator'} def _decoded(raw, default): value = json.loads(raw) if isinstance(raw, str) else raw return value if isinstance(value, type(default)) else default def _values(attributes, name, marketplace): rows = attributes.get(name) return [row.get('value', row.get('media_location')) for row in rows if isinstance(row, dict) and row.get('marketplace_id', marketplace) == marketplace] if isinstance(rows, list) else [] def listing_parts(attributes, images, marketplace): """Select marketplace-specific title, ordered bullets and gallery slots without A+ samples.""" parts = {} for part, name in (('title', 'item_name'), ('bullets', 'bullet_point')): if isinstance(attributes.get(name), list): values = _values(attributes, name, marketplace) parts[part] = (values[0] if values else None) if part == 'title' else values gallery = dict.fromkeys(IMAGE_ATTRIBUTES) images_available = any(isinstance(attributes.get(name), list) for name in IMAGE_ATTRIBUTES.values()) for slot, name in IMAGE_ATTRIBUTES.items(): values = _values(attributes, name, marketplace) if name in attributes: gallery[slot] = values[0] if values else None for group in images: if not isinstance(group, dict): continue if group.get('marketplaceId', marketplace) == marketplace: for image in group.get('images', [group]): if not isinstance(image, dict): continue slot = image.get('variant') if slot in IMAGE_ATTRIBUTES and image.get('link'): gallery[slot] = gallery.get(slot) or image['link'] images_available = True if images_available: parts['images'] = gallery return parts def _digest(value): return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode('utf-8')).hexdigest() def part_hashes(parts): """Hash each part's content without borrowing another part's identity.""" return {part: _digest(value) for part, value in parts.items()} def snapshots(con): """Read every listing and combine its SKU and marketplace identities deterministically per ASIN.""" columns = {row[0] for row in con.execute( 'SELECT column_name FROM information_schema.columns WHERE table_name = ?', [TABLE]).fetchall()} if not {'sku', 'attributes', 'summaries'} <= columns: return {} images_column = 'images' if 'images' in columns else 'NULL' cursor = con.execute(f'SELECT sku, attributes, summaries, {images_column} FROM {TABLE}') listings = {} for batch in iter(lambda: cursor.fetchmany(2048), []): for sku, raw_attributes, raw_summaries, raw_images in batch: attributes = _decoded(raw_attributes, {}) images = _decoded(raw_images, []) for summary in _decoded(raw_summaries, []): if not isinstance(summary, dict) or not summary.get('asin') or not attributes: continue identity = (str(summary['asin']).strip().upper(), str(sku or ''), summary.get('marketplaceId') or '') parts = listing_parts(attributes, images, summary.get('marketplaceId')) candidate = (summary.get('lastUpdatedDate') or '', _digest(parts), parts) if identity not in listings or candidate[:2] > listings[identity][:2]: listings[identity] = candidate content = {} for (asin, sku, marketplace), (_, _, parts) in sorted(listings.items()): for part, value in parts.items(): entry = [marketplace, value] content.setdefault(asin, {}).setdefault(part, {})[_digest(entry)] = entry return {asin: part_hashes({part: [values[key] for key in sorted(values)] for part, values in parts.items()}) for asin, parts in content.items()} def observe(rt, con, *, day): """Persist a daily baseline or changed part, including ASINs absent from profit rows.""" observed = snapshots(con) return {'asins': len(observed), 'hashes': pdp.observe_part_hashes(rt, observed, day)} def cells(rt, asins): """Project recorded dates and clear unsupported sample dates without observing or writing.""" recorded = pdp.part_dates(rt, asins) return {asin: {key: recorded.get(asin, {}).get(key) for key in DATE_KEYS} for asin in asins}