File size: 4,953 Bytes
95fb175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"""Observe listing parts independently and retain only dates supported by recorded changes."""

import hashlib
import json

from core import amazon_pdp as pdp

TABLE = 'amzn_listing_items'
DATE_KEYS = tuple(part + '_last_updated' for part in (*pdp.OBSERVED_PARTS, 'aplus'))
IMAGE_ATTRIBUTES = {'MAIN': 'main_product_image_locator',
                    **{f'PT{i:02}': f'other_product_image_locator_{i}' for i in range(1, 9)},
                    'VIDEO': 'product_video_locator'}


def _decoded(raw, default):
    value = json.loads(raw) if isinstance(raw, str) else raw
    return value if isinstance(value, type(default)) else default


def _values(attributes, name, marketplace):
    rows = attributes.get(name)
    return [row.get('value', row.get('media_location')) for row in rows if isinstance(row, dict)
            and row.get('marketplace_id', marketplace) == marketplace] if isinstance(rows, list) else []


def listing_parts(attributes, images, marketplace):
    """Select marketplace-specific title, ordered bullets and gallery slots without A+ samples."""
    parts = {}
    for part, name in (('title', 'item_name'), ('bullets', 'bullet_point')):
        if isinstance(attributes.get(name), list):
            values = _values(attributes, name, marketplace)
            parts[part] = (values[0] if values else None) if part == 'title' else values
    gallery = dict.fromkeys(IMAGE_ATTRIBUTES)
    images_available = any(isinstance(attributes.get(name), list) for name in IMAGE_ATTRIBUTES.values())
    for slot, name in IMAGE_ATTRIBUTES.items():
        values = _values(attributes, name, marketplace)
        if name in attributes:
            gallery[slot] = values[0] if values else None
    for group in images:
        if not isinstance(group, dict):
            continue
        if group.get('marketplaceId', marketplace) == marketplace:
            for image in group.get('images', [group]):
                if not isinstance(image, dict):
                    continue
                slot = image.get('variant')
                if slot in IMAGE_ATTRIBUTES and image.get('link'):
                    gallery[slot] = gallery.get(slot) or image['link']
                    images_available = True
    if images_available:
        parts['images'] = gallery
    return parts


def _digest(value):
    return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'),
                                     ensure_ascii=False).encode('utf-8')).hexdigest()


def part_hashes(parts):
    """Hash each part's content without borrowing another part's identity."""
    return {part: _digest(value) for part, value in parts.items()}


def snapshots(con):
    """Read every listing and combine its SKU and marketplace identities deterministically per ASIN."""
    columns = {row[0] for row in con.execute(
        'SELECT column_name FROM information_schema.columns WHERE table_name = ?', [TABLE]).fetchall()}
    if not {'sku', 'attributes', 'summaries'} <= columns:
        return {}
    images_column = 'images' if 'images' in columns else 'NULL'
    cursor = con.execute(f'SELECT sku, attributes, summaries, {images_column} FROM {TABLE}')
    listings = {}
    for batch in iter(lambda: cursor.fetchmany(2048), []):
        for sku, raw_attributes, raw_summaries, raw_images in batch:
            attributes = _decoded(raw_attributes, {})
            images = _decoded(raw_images, [])
            for summary in _decoded(raw_summaries, []):
                if not isinstance(summary, dict) or not summary.get('asin') or not attributes:
                    continue
                identity = (str(summary['asin']).strip().upper(), str(sku or ''), summary.get('marketplaceId') or '')
                parts = listing_parts(attributes, images, summary.get('marketplaceId'))
                candidate = (summary.get('lastUpdatedDate') or '', _digest(parts), parts)
                if identity not in listings or candidate[:2] > listings[identity][:2]:
                    listings[identity] = candidate
    content = {}
    for (asin, sku, marketplace), (_, _, parts) in sorted(listings.items()):
        for part, value in parts.items():
            entry = [marketplace, value]
            content.setdefault(asin, {}).setdefault(part, {})[_digest(entry)] = entry
    return {asin: part_hashes({part: [values[key] for key in sorted(values)] for part, values in parts.items()})
            for asin, parts in content.items()}


def observe(rt, con, *, day):
    """Persist a daily baseline or changed part, including ASINs absent from profit rows."""
    observed = snapshots(con)
    return {'asins': len(observed), 'hashes': pdp.observe_part_hashes(rt, observed, day)}


def cells(rt, asins):
    """Project recorded dates and clear unsupported sample dates without observing or writing."""
    recorded = pdp.part_dates(rt, asins)
    return {asin: {key: recorded.get(asin, {}).get(key) for key in DATE_KEYS} for asin in asins}