Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- platform/harness/amazon_client.py +115 -4
- platform/harness/amazon_store.py +581 -79
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "ea8618de4",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
ea8618de4
|
platform/harness/amazon_client.py
CHANGED
|
@@ -45,9 +45,13 @@ Ideation record, with every owner ruling: `AutoWiki/research/ideation/amazon-con
|
|
| 45 |
from __future__ import annotations
|
| 46 |
|
| 47 |
import contextvars
|
|
|
|
| 48 |
import json
|
|
|
|
| 49 |
import random
|
| 50 |
import re
|
|
|
|
|
|
|
| 51 |
import threading
|
| 52 |
import time
|
| 53 |
from datetime import date, datetime, timezone, timedelta
|
|
@@ -1994,7 +1998,7 @@ def report_document(fields, report_type, start=None, end=None, options=None,
|
|
| 1994 |
url = str(meta.get("url") or "")
|
| 1995 |
if not url:
|
| 1996 |
raise AmazonError(f"the {report_type} report document carried no download URL")
|
| 1997 |
-
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 1998 |
say(f" {report_type}: {len(text):,} chars")
|
| 1999 |
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
| 2000 |
"dataStartTime": got.get("dataStartTime"),
|
|
@@ -2090,8 +2094,106 @@ def _workbook_as_tsv(raw):
|
|
| 2090 |
return "\n".join(lines)
|
| 2091 |
|
| 2092 |
|
| 2093 |
-
|
| 2094 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2095 |
|
| 2096 |
⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
|
| 2097 |
string, which for an S3 presigned URL is the entire credential.
|
|
@@ -2103,11 +2205,20 @@ def _download_document(url, compression="", timeout=120):
|
|
| 2103 |
# apply OPERATIONS here: its host and signature belong to Amazon's document download service.
|
| 2104 |
req = urllib.request.Request(url, method="GET")
|
| 2105 |
req.add_header("User-Agent", _UA)
|
|
|
|
| 2106 |
try:
|
| 2107 |
with urllib.request.urlopen(req, timeout=timeout) as r:
|
| 2108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2109 |
except Exception as e: # noqa: BLE001
|
|
|
|
|
|
|
| 2110 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
|
|
|
|
|
|
| 2111 |
# S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
|
| 2112 |
# `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
|
| 2113 |
# algorithm the bytes were decoded as text and every economics window failed to parse. The
|
|
|
|
| 45 |
from __future__ import annotations
|
| 46 |
|
| 47 |
import contextvars
|
| 48 |
+
import gzip
|
| 49 |
import json
|
| 50 |
+
import os
|
| 51 |
import random
|
| 52 |
import re
|
| 53 |
+
import shutil
|
| 54 |
+
import tempfile
|
| 55 |
import threading
|
| 56 |
import time
|
| 57 |
from datetime import date, datetime, timezone, timedelta
|
|
|
|
| 1998 |
url = str(meta.get("url") or "")
|
| 1999 |
if not url:
|
| 2000 |
raise AmazonError(f"the {report_type} report document carried no download URL")
|
| 2001 |
+
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True)
|
| 2002 |
say(f" {report_type}: {len(text):,} chars")
|
| 2003 |
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
| 2004 |
"dataStartTime": got.get("dataStartTime"),
|
|
|
|
| 2094 |
return "\n".join(lines)
|
| 2095 |
|
| 2096 |
|
| 2097 |
+
class ReportDocument:
|
| 2098 |
+
"""One downloaded report document spooled to a temporary file, read back as text in pieces.
|
| 2099 |
+
|
| 2100 |
+
⛔ A report has no row cap (standing rule 1), so its text never sits in memory whole: the Brand
|
| 2101 |
+
Analytics month held the body, its decompressed copy and its decoded text at once (D-7081).
|
| 2102 |
+
The file is removed when the object is closed or collected.
|
| 2103 |
+
"""
|
| 2104 |
+
|
| 2105 |
+
def __init__(self, path, encoding, errors, chars, size, compressed):
|
| 2106 |
+
self.path, self.encoding, self.errors = path, encoding, errors
|
| 2107 |
+
self.chars, self.size, self.compressed = chars, size, compressed
|
| 2108 |
+
|
| 2109 |
+
def open_text(self):
|
| 2110 |
+
"""The decoded text as a stream; `newline=""` keeps line ends exactly as `bytes.decode` did."""
|
| 2111 |
+
return open(self.path, "r", encoding=self.encoding, errors=self.errors, newline="")
|
| 2112 |
+
|
| 2113 |
+
def read_text(self):
|
| 2114 |
+
"""The whole text, for a consumer whose document is small."""
|
| 2115 |
+
with self.open_text() as handle:
|
| 2116 |
+
return handle.read()
|
| 2117 |
+
|
| 2118 |
+
def __len__(self):
|
| 2119 |
+
return self.chars
|
| 2120 |
+
|
| 2121 |
+
def close(self):
|
| 2122 |
+
path, self.path = getattr(self, "path", None), None
|
| 2123 |
+
if path:
|
| 2124 |
+
try:
|
| 2125 |
+
os.unlink(path)
|
| 2126 |
+
except Exception: # noqa: BLE001
|
| 2127 |
+
pass
|
| 2128 |
+
|
| 2129 |
+
def __del__(self):
|
| 2130 |
+
self.close()
|
| 2131 |
+
|
| 2132 |
+
|
| 2133 |
+
_SPOOL_CHUNK = 1 << 20
|
| 2134 |
+
|
| 2135 |
+
|
| 2136 |
+
def _spool_file(prefix):
|
| 2137 |
+
handle, path = tempfile.mkstemp(prefix=prefix, suffix=".part")
|
| 2138 |
+
os.close(handle)
|
| 2139 |
+
return path
|
| 2140 |
+
|
| 2141 |
+
|
| 2142 |
+
def _unlink(path):
|
| 2143 |
+
try:
|
| 2144 |
+
os.unlink(path)
|
| 2145 |
+
except OSError:
|
| 2146 |
+
pass
|
| 2147 |
+
|
| 2148 |
+
|
| 2149 |
+
def _decoded_chars(path, encoding, errors="strict"):
|
| 2150 |
+
"""Characters in the file under one encoding; raises UnicodeDecodeError like `bytes.decode`."""
|
| 2151 |
+
chars = 0
|
| 2152 |
+
with open(path, "r", encoding=encoding, errors=errors, newline="") as handle:
|
| 2153 |
+
while True:
|
| 2154 |
+
piece = handle.read(_SPOOL_CHUNK)
|
| 2155 |
+
if not piece:
|
| 2156 |
+
return chars
|
| 2157 |
+
chars += len(piece)
|
| 2158 |
+
|
| 2159 |
+
|
| 2160 |
+
def _spooled_document(body, compression):
|
| 2161 |
+
"""The spooled body as a `ReportDocument`: gunzip and decode stream through disk, never memory."""
|
| 2162 |
+
compressed = os.path.getsize(body)
|
| 2163 |
+
with open(body, "rb") as handle:
|
| 2164 |
+
head = handle.read(4)
|
| 2165 |
+
if not compression and head[:2] == b"\x1f\x8b":
|
| 2166 |
+
compression = "GZIP"
|
| 2167 |
+
if str(compression).upper() == "GZIP":
|
| 2168 |
+
plain = _spool_file("amazon-document-")
|
| 2169 |
+
try:
|
| 2170 |
+
with gzip.open(body, "rb") as source, open(plain, "wb") as out:
|
| 2171 |
+
shutil.copyfileobj(source, out, _SPOOL_CHUNK)
|
| 2172 |
+
except Exception as e: # noqa: BLE001
|
| 2173 |
+
_unlink(plain)
|
| 2174 |
+
raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
|
| 2175 |
+
finally:
|
| 2176 |
+
_unlink(body)
|
| 2177 |
+
body = plain
|
| 2178 |
+
with open(body, "rb") as handle:
|
| 2179 |
+
head = handle.read(4)
|
| 2180 |
+
if head[:4] == b"PK\x03\x04":
|
| 2181 |
+
try:
|
| 2182 |
+
with open(body, "rb") as handle:
|
| 2183 |
+
return _workbook_as_tsv(handle.read())
|
| 2184 |
+
finally:
|
| 2185 |
+
_unlink(body)
|
| 2186 |
+
size = os.path.getsize(body)
|
| 2187 |
+
for enc in ("utf-8", "cp1252"):
|
| 2188 |
+
try:
|
| 2189 |
+
return ReportDocument(body, enc, "strict", _decoded_chars(body, enc), size, compressed)
|
| 2190 |
+
except UnicodeDecodeError:
|
| 2191 |
+
continue
|
| 2192 |
+
return ReportDocument(body, "utf-8", "replace", _decoded_chars(body, "utf-8", "replace"), size, compressed)
|
| 2193 |
+
|
| 2194 |
+
|
| 2195 |
+
def _download_document(url, compression="", timeout=120, *, spool=False):
|
| 2196 |
+
"""Fetch one presigned report document and decode it. -> text, or a `ReportDocument` when spooled.
|
| 2197 |
|
| 2198 |
⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
|
| 2199 |
string, which for an S3 presigned URL is the entire credential.
|
|
|
|
| 2205 |
# apply OPERATIONS here: its host and signature belong to Amazon's document download service.
|
| 2206 |
req = urllib.request.Request(url, method="GET")
|
| 2207 |
req.add_header("User-Agent", _UA)
|
| 2208 |
+
body = _spool_file("amazon-document-") if spool else None
|
| 2209 |
try:
|
| 2210 |
with urllib.request.urlopen(req, timeout=timeout) as r:
|
| 2211 |
+
if body:
|
| 2212 |
+
with open(body, "wb") as out:
|
| 2213 |
+
shutil.copyfileobj(r, out, _SPOOL_CHUNK)
|
| 2214 |
+
else:
|
| 2215 |
+
raw = r.read()
|
| 2216 |
except Exception as e: # noqa: BLE001
|
| 2217 |
+
if body:
|
| 2218 |
+
_unlink(body)
|
| 2219 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 2220 |
+
if body:
|
| 2221 |
+
return _spooled_document(body, compression)
|
| 2222 |
# S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
|
| 2223 |
# `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
|
| 2224 |
# algorithm the bytes were decoded as text and every economics window failed to parse. The
|
platform/harness/amazon_store.py
CHANGED
|
@@ -327,6 +327,234 @@ def _without_buyer_data(value):
|
|
| 327 |
return value
|
| 328 |
|
| 329 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
def parse_flat_file(text, *, document_id=None, report_type=None):
|
| 331 |
"""One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
|
| 332 |
|
|
@@ -339,29 +567,15 @@ def parse_flat_file(text, *, document_id=None, report_type=None):
|
|
| 339 |
swallow the rest of the row and silently merge it with the next one, so a catalogue with one
|
| 340 |
inch-mark in a title would lose rows with nothing raising.
|
| 341 |
"""
|
|
|
|
|
|
|
| 342 |
if not str(text or "").strip():
|
| 343 |
return [], []
|
| 344 |
buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
header = next(reader)
|
| 348 |
-
except StopIteration:
|
| 349 |
return [], []
|
| 350 |
-
cols
|
| 351 |
-
width = len(cols)
|
| 352 |
-
rows = []
|
| 353 |
-
# The read-only connectors_amazon report probe measures headers without capture metadata.
|
| 354 |
-
# Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
|
| 355 |
-
# BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
|
| 356 |
-
attributed = document_id is not None or report_type is not None
|
| 357 |
-
for row_number, raw in enumerate(reader, 1):
|
| 358 |
-
if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
|
| 359 |
-
continue
|
| 360 |
-
vals = list(raw[:width]) + [None] * max(0, width - len(raw))
|
| 361 |
-
row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
|
| 362 |
-
if not buyer_column(c)}
|
| 363 |
-
rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
|
| 364 |
-
return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows
|
| 365 |
|
| 366 |
|
| 367 |
def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
@@ -372,6 +586,8 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
| 372 |
with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
|
| 373 |
artifact is the authority; a field the payload omits is NULL.
|
| 374 |
"""
|
|
|
|
|
|
|
| 375 |
try:
|
| 376 |
doc = json.loads(str(text or "") or "{}")
|
| 377 |
except (ValueError, TypeError) as exc:
|
|
@@ -387,12 +603,151 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
| 387 |
body = [body]
|
| 388 |
if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
|
| 389 |
raise ValueError(f"Amazon JSON report has an invalid {root} row list")
|
| 390 |
-
rows =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
attributed = document_id is not None or report_type is not None
|
| 392 |
for row_number, item in enumerate(body, 1):
|
| 393 |
row = _flatten_row(_without_buyer_data(item))
|
| 394 |
-
|
| 395 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 396 |
|
| 397 |
|
| 398 |
def _flatten_row(obj, prefix=""):
|
|
@@ -601,6 +956,8 @@ def _awd_rows(inventory, operation_id=None):
|
|
| 601 |
|
| 602 |
def _tag_fba_inventory(rows):
|
| 603 |
"""Mark report rows so one combined inventory table never conflates FBA with AWD."""
|
|
|
|
|
|
|
| 604 |
for row in rows or ():
|
| 605 |
row["inventory_source"] = "FBA"
|
| 606 |
return rows
|
|
@@ -690,36 +1047,18 @@ def _coltype(name, rows=None):
|
|
| 690 |
return "VARCHAR"
|
| 691 |
if not rows:
|
| 692 |
return "VARCHAR"
|
|
|
|
|
|
|
| 693 |
seen, kinds = 0, set()
|
| 694 |
for r in rows:
|
| 695 |
v = r.get(name)
|
| 696 |
if v is None or v == "":
|
| 697 |
continue
|
| 698 |
-
|
| 699 |
-
kinds.add("str")
|
| 700 |
-
elif isinstance(v, int):
|
| 701 |
-
kinds.add("int")
|
| 702 |
-
elif isinstance(v, float):
|
| 703 |
-
kinds.add("float")
|
| 704 |
-
elif isinstance(v, (list, dict)):
|
| 705 |
-
kinds.add("str")
|
| 706 |
-
else:
|
| 707 |
-
s = str(v).strip()
|
| 708 |
-
try:
|
| 709 |
-
int(s)
|
| 710 |
-
kinds.add("int")
|
| 711 |
-
except ValueError:
|
| 712 |
-
try:
|
| 713 |
-
float(s)
|
| 714 |
-
kinds.add("float")
|
| 715 |
-
except ValueError:
|
| 716 |
-
kinds.add("str")
|
| 717 |
seen += 1
|
| 718 |
if seen >= 500:
|
| 719 |
break
|
| 720 |
-
|
| 721 |
-
return "VARCHAR"
|
| 722 |
-
return "DOUBLE" if "float" in kinds else "BIGINT"
|
| 723 |
|
| 724 |
|
| 725 |
def _cell(value, kind):
|
|
@@ -798,11 +1137,27 @@ def _replace(con, table, cols, types, rows):
|
|
| 798 |
or type(r.get("_report_row_number")) is not int
|
| 799 |
or r["_report_row_number"] < 1 for r in rows):
|
| 800 |
raise ValueError("refusing Amazon rows without complete source attribution")
|
| 801 |
-
ph = ", ".join("?" for _ in cols)
|
| 802 |
-
|
| 803 |
-
|
| 804 |
-
|
| 805 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 806 |
|
| 807 |
|
| 808 |
def replace_keys_for(table):
|
|
@@ -822,28 +1177,26 @@ def replace_window(con, table, cols, types, rows, keys):
|
|
| 822 |
return _replace(con, table, cols, types, rows), None
|
| 823 |
quote = lambda name: '"' + str(name).replace('"', '""') + '"'
|
| 824 |
target = quote(table)
|
| 825 |
-
|
| 826 |
-
|
| 827 |
-
|
| 828 |
-
|
| 829 |
-
for c in keys) for row in rows))
|
| 830 |
con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
|
| 831 |
try:
|
| 832 |
-
con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})",
|
| 833 |
-
incoming)
|
| 834 |
predicate = " AND ".join(
|
| 835 |
f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
|
| 836 |
con.execute("BEGIN TRANSACTION")
|
| 837 |
try:
|
|
|
|
| 838 |
con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
|
| 839 |
-
|
| 840 |
con.execute("COMMIT")
|
| 841 |
except Exception:
|
| 842 |
con.execute("ROLLBACK")
|
| 843 |
raise
|
| 844 |
return count, None
|
| 845 |
finally:
|
| 846 |
-
con.execute(f"DROP TABLE {temporary}")
|
| 847 |
|
| 848 |
|
| 849 |
def _windows(start, end, size=MAX_WINDOW_DAYS):
|
|
@@ -1189,6 +1542,7 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
|
|
| 1189 |
return 0
|
| 1190 |
|
| 1191 |
# Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
|
|
|
|
| 1192 |
doc = json.loads(text)
|
| 1193 |
raw = doc.get(spec["json_root"])
|
| 1194 |
if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
|
|
@@ -2747,6 +3101,16 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
|
|
| 2747 |
|
| 2748 |
def document_row_count(text, roots=()):
|
| 2749 |
"""Count source records before flattening, privacy filtering or mirror transformations."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2750 |
if roots:
|
| 2751 |
doc = json.loads(text)
|
| 2752 |
return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
|
|
@@ -2791,23 +3155,31 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
|
|
| 2791 |
stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
|
| 2792 |
if not stamp and (table == "listings" or not rows):
|
| 2793 |
raise ValueError("Amazon snapshot returned no data date or report creation date")
|
| 2794 |
-
|
| 2795 |
-
for row in rows:
|
| 2796 |
period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
|
| 2797 |
row["_snapshot_date"] = period
|
| 2798 |
if table == "inventory":
|
| 2799 |
row["date"] = period
|
| 2800 |
row["inventory_source"] = "FBA"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2801 |
identity = _snapshot_identity(table, row)
|
| 2802 |
if identity in identities:
|
| 2803 |
raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
|
| 2804 |
identities.add(identity)
|
| 2805 |
counts[period] = counts.get(period, 0) + 1
|
| 2806 |
-
|
|
|
|
| 2807 |
raise ValueError(f"Amazon {table} snapshot row count differs from its document")
|
| 2808 |
if not counts:
|
| 2809 |
counts[_utc(stamp).date().isoformat()] = 0
|
| 2810 |
-
columns = sorted(set(columns) | {"_snapshot_date"}
|
|
|
|
| 2811 |
if table == "inventory":
|
| 2812 |
columns = sorted(set(columns) | {"inventory_source"})
|
| 2813 |
mirror = table_name(table)
|
|
@@ -2954,8 +3326,9 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2954 |
if options:
|
| 2955 |
kwargs["options"] = options
|
| 2956 |
text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
|
|
|
|
| 2957 |
parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
|
| 2958 |
-
if spec.get("json_root") and meta.get("cancelled") and
|
| 2959 |
# ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
|
| 2960 |
# returns an empty document for it, S66-T42), but the JSON parser read "" as a document
|
| 2961 |
# with no row root and REFUSED it. So the two B2B opportunity reports never earned a
|
|
@@ -2965,14 +3338,14 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2965 |
elif spec.get("json_root"):
|
| 2966 |
columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
|
| 2967 |
if spec.get("json_extra_roots"):
|
| 2968 |
-
|
| 2969 |
-
|
| 2970 |
for root in spec["json_extra_roots"]:
|
| 2971 |
extra_columns, extra = parse_json_report(text, root, **parser_args)
|
| 2972 |
-
|
| 2973 |
-
row["_json_root"] = root
|
| 2974 |
columns = merge_columns(columns, extra_columns)
|
| 2975 |
-
|
|
|
|
| 2976 |
else:
|
| 2977 |
columns, rows = parse_flat_file(text, **parser_args)
|
| 2978 |
if table in LEGACY_SNAPSHOT_TABLES:
|
|
@@ -2986,7 +3359,7 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2986 |
period, column = _utc(w0).strftime("%Y-%m"), "month"
|
| 2987 |
else:
|
| 2988 |
period, column = _iso(w0), "_period_start"
|
| 2989 |
-
|
| 2990 |
row[column] = period
|
| 2991 |
if w1 is not None:
|
| 2992 |
row["_period_end"] = _iso(w1)
|
|
@@ -2994,19 +3367,31 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2994 |
row["fulfillment_center"] = (row.get("fulfillment_center") or
|
| 2995 |
row.get("fulfillment_center_id") or
|
| 2996 |
row.get("fulfilment_center") or "all")
|
| 2997 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2998 |
roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
|
| 2999 |
# HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
|
| 3000 |
-
empty = bool(roots) and ((meta.get("cancelled") and
|
| 3001 |
(table in ("b2b_recommended", "b2b_not_yet_on_amazon")
|
| 3002 |
-
and isinstance(json.loads(text or "{}"), dict)
|
| 3003 |
-
and spec["json_root"] not in json.loads(text or "{}")))
|
| 3004 |
marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
|
| 3005 |
"column": column, "count": 0 if empty else document_row_count(text, roots)}
|
| 3006 |
return _replace_period(con, table, rows, columns, column, period, marker=marker,
|
| 3007 |
snapshot=bool(spec.get("snapshot")))
|
| 3008 |
|
| 3009 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3010 |
def _order_metric_totals(records, *, flattened=False):
|
| 3011 |
totals = {"orders": 0, "units": 0, "sales": {}}
|
| 3012 |
for row in records:
|
|
@@ -3542,11 +3927,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3542 |
settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
|
| 3543 |
for item in discovered[issued_index:])
|
| 3544 |
break
|
|
|
|
| 3545 |
operation = "getReportDocument"
|
| 3546 |
documents += 1
|
| 3547 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3548 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 3549 |
text = cx.settlement_document(fields, document_id)
|
|
|
|
| 3550 |
cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
|
| 3551 |
count = 0
|
| 3552 |
if cols and rows:
|
|
@@ -3555,6 +3942,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3555 |
# A newly issued document still replaces its own settlement's lines.
|
| 3556 |
count = write_window(table, cols, types, rows)
|
| 3557 |
written += count
|
|
|
|
| 3558 |
# Record completion after the write, including header-only and blank files.
|
| 3559 |
_record_state(con, entity_key("settlement_doc_" + document_id), "done",
|
| 3560 |
count, issued.get("dataEndTime"))
|
|
@@ -3618,11 +4006,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3618 |
_record_state(con, entity_key("limit_" + table), "done", detail="{}")
|
| 3619 |
for w0, w1 in margin_windows:
|
| 3620 |
# One detail window is one budget task, including its reconciliation query.
|
|
|
|
| 3621 |
documents += 1
|
| 3622 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3623 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 3624 |
-
|
| 3625 |
-
|
|
|
|
|
|
|
| 3626 |
landed_window = True
|
| 3627 |
if table in LEGACY_SNAPSHOT_TABLES:
|
| 3628 |
# The dated replacement already landed; AWD must not clear that
|
|
@@ -3642,6 +4033,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3642 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 3643 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
| 3644 |
documents += 1
|
|
|
|
| 3645 |
with _report_resume(
|
| 3646 |
con, table, rtype, start=_iso(w0),
|
| 3647 |
end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
|
|
@@ -3652,6 +4044,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3652 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3653 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 3654 |
text, meta = cx.report_document(fields, rtype, **resume_kwargs)
|
|
|
|
| 3655 |
landed_window = True
|
| 3656 |
if root:
|
| 3657 |
parsed_cols, rows = parse_json_report(
|
|
@@ -3727,6 +4120,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3727 |
if cx.REPORT_TYPES[table].get("monthly"):
|
| 3728 |
_record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
|
| 3729 |
"count": len(rows)})
|
|
|
|
| 3730 |
if first_load:
|
| 3731 |
for reached in (table, sibling):
|
| 3732 |
if reached:
|
|
@@ -3961,15 +4355,20 @@ def _dedupe_orders(rows):
|
|
| 3961 |
lines by construction, so this drops line detail rather than losing an order. `order_items`
|
| 3962 |
keeps every row, which is why both tables read the same document instead of costing two.
|
| 3963 |
"""
|
| 3964 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3965 |
for r in rows:
|
| 3966 |
oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
|
| 3967 |
if oid and oid in seen:
|
| 3968 |
continue
|
| 3969 |
if oid:
|
| 3970 |
seen.add(oid)
|
| 3971 |
-
|
| 3972 |
-
return out
|
| 3973 |
|
| 3974 |
|
| 3975 |
def validate(tenant_key="royal-imports", rt=None, log=print):
|
|
@@ -6303,10 +6702,113 @@ def selftest(log=print):
|
|
| 6303 |
ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
|
| 6304 |
ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
|
| 6305 |
ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
|
| 6306 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6307 |
return 0 if not fails else 1
|
| 6308 |
|
| 6309 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6310 |
def main(argv=None):
|
| 6311 |
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 6312 |
ap.add_argument("--sync", action="store_true")
|
|
|
|
| 327 |
return value
|
| 328 |
|
| 329 |
|
| 330 |
+
#: Rows per INSERT statement. A batch, not a cap: every row of a document is written (standing rule 1).
|
| 331 |
+
INSERT_BATCH_ROWS = 500
|
| 332 |
+
#: Bound parameters per INSERT statement, so a wide table takes fewer rows per statement.
|
| 333 |
+
INSERT_BATCH_VALUES = 30_000
|
| 334 |
+
#: Characters read per step of a spooled document.
|
| 335 |
+
DOCUMENT_READ_CHARS = 1 << 20
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def _spooled(document):
|
| 339 |
+
"""True for a `ReportDocument` the client spooled to disk; False for in-memory text."""
|
| 340 |
+
return hasattr(document, "open_text")
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def _document_text(document):
|
| 344 |
+
"""The whole text of a document, for a consumer whose documents are small."""
|
| 345 |
+
return document.read_text() if _spooled(document) else document
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
def _blank(document):
|
| 349 |
+
"""`not str(text or "").strip()`, read in pieces when the document is spooled."""
|
| 350 |
+
if not _spooled(document):
|
| 351 |
+
return not str(document or "").strip()
|
| 352 |
+
with document.open_text() as handle:
|
| 353 |
+
while True:
|
| 354 |
+
piece = handle.read(DOCUMENT_READ_CHARS)
|
| 355 |
+
if not piece:
|
| 356 |
+
return True
|
| 357 |
+
if piece.strip():
|
| 358 |
+
return False
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def _value_kind(v):
|
| 362 |
+
"""One non-empty value's sniffed kind: `int`, `float` or `str`."""
|
| 363 |
+
if isinstance(v, bool):
|
| 364 |
+
return "str"
|
| 365 |
+
if isinstance(v, int):
|
| 366 |
+
return "int"
|
| 367 |
+
if isinstance(v, float):
|
| 368 |
+
return "float"
|
| 369 |
+
if isinstance(v, (list, dict)):
|
| 370 |
+
return "str"
|
| 371 |
+
s = str(v).strip()
|
| 372 |
+
try:
|
| 373 |
+
int(s)
|
| 374 |
+
return "int"
|
| 375 |
+
except ValueError:
|
| 376 |
+
try:
|
| 377 |
+
float(s)
|
| 378 |
+
return "float"
|
| 379 |
+
except ValueError:
|
| 380 |
+
return "str"
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def _kinds_type(kinds):
|
| 384 |
+
if not kinds or "str" in kinds:
|
| 385 |
+
return "VARCHAR"
|
| 386 |
+
return "DOUBLE" if "float" in kinds else "BIGINT"
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
class _DocumentRows:
|
| 390 |
+
"""Rows parsed again from a spooled report document on every pass, so no pass holds them all.
|
| 391 |
+
|
| 392 |
+
⛔ D-7081: one list of every row of the Brand Analytics month took the Space to 16 GB. A caller
|
| 393 |
+
iterates, takes `len()` or tests truth; `map` and `pipe` replay a row change on every pass.
|
| 394 |
+
"""
|
| 395 |
+
|
| 396 |
+
def __init__(self, produce, steps=()):
|
| 397 |
+
self._produce, self._steps = produce, tuple(steps)
|
| 398 |
+
self._count, self._sniff = None, None
|
| 399 |
+
|
| 400 |
+
def __iter__(self):
|
| 401 |
+
rows = self._produce()
|
| 402 |
+
for step in self._steps:
|
| 403 |
+
rows = step(rows)
|
| 404 |
+
return iter(rows)
|
| 405 |
+
|
| 406 |
+
def pipe(self, step):
|
| 407 |
+
return _DocumentRows(self._produce, self._steps + (step,))
|
| 408 |
+
|
| 409 |
+
def map(self, change):
|
| 410 |
+
return self.pipe(lambda rows: (change(row) for row in rows))
|
| 411 |
+
|
| 412 |
+
@staticmethod
|
| 413 |
+
def chain(*parts):
|
| 414 |
+
return _DocumentRows(lambda: (row for part in parts for row in part))
|
| 415 |
+
|
| 416 |
+
def _profile(self):
|
| 417 |
+
"""One pass: the row count, every key, and each key's first 500 non-empty kinds."""
|
| 418 |
+
count, sniff = 0, {}
|
| 419 |
+
for row in self:
|
| 420 |
+
count += 1
|
| 421 |
+
for key, value in row.items():
|
| 422 |
+
entry = sniff.get(key)
|
| 423 |
+
if entry is None:
|
| 424 |
+
entry = sniff[key] = [0, set()]
|
| 425 |
+
if entry[0] >= 500 or value is None or value == "":
|
| 426 |
+
continue
|
| 427 |
+
entry[1].add(_value_kind(value))
|
| 428 |
+
entry[0] += 1
|
| 429 |
+
self._count, self._sniff = count, sniff
|
| 430 |
+
|
| 431 |
+
def keys(self):
|
| 432 |
+
if self._sniff is None:
|
| 433 |
+
self._profile()
|
| 434 |
+
return set(self._sniff)
|
| 435 |
+
|
| 436 |
+
def sniffed(self, name):
|
| 437 |
+
"""`_coltype(name, rows)` for these rows, from the one profile pass."""
|
| 438 |
+
if self._sniff is None:
|
| 439 |
+
self._profile()
|
| 440 |
+
return _kinds_type(self._sniff.get(name, (0, ()))[1])
|
| 441 |
+
|
| 442 |
+
def __len__(self):
|
| 443 |
+
if self._count is None:
|
| 444 |
+
self._profile()
|
| 445 |
+
return self._count
|
| 446 |
+
|
| 447 |
+
def __bool__(self):
|
| 448 |
+
if self._count is not None:
|
| 449 |
+
return self._count > 0
|
| 450 |
+
rows = iter(self)
|
| 451 |
+
try:
|
| 452 |
+
return next(rows, None) is not None
|
| 453 |
+
finally:
|
| 454 |
+
getattr(rows, "close", lambda: None)()
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def _rss_gb():
|
| 458 |
+
"""This process's resident memory in GB, or None where /proc is absent."""
|
| 459 |
+
try:
|
| 460 |
+
with open("/proc/self/status", encoding="ascii") as handle:
|
| 461 |
+
for line in handle:
|
| 462 |
+
if line.startswith("VmRSS:"):
|
| 463 |
+
return int(line.split()[1]) / 1048576
|
| 464 |
+
except Exception: # noqa: BLE001
|
| 465 |
+
return None
|
| 466 |
+
return None
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
class _DocumentProbe:
|
| 470 |
+
"""One server-log line per report document: table, bytes, rows, seconds and the memory peak.
|
| 471 |
+
|
| 472 |
+
⚠ `print`, not `log`: the sync desk passes a silent log, so its first load logged nothing until a
|
| 473 |
+
pass ended and the document in flight at an out-of-memory kill was unknown (D-7081).
|
| 474 |
+
"""
|
| 475 |
+
|
| 476 |
+
_lock = threading.Lock()
|
| 477 |
+
_active = None
|
| 478 |
+
|
| 479 |
+
def __init__(self, table, report_type):
|
| 480 |
+
self.table, self.report_type = table, report_type
|
| 481 |
+
self.started, self.size, self.peak = time.monotonic(), None, _rss_gb()
|
| 482 |
+
self._halt = threading.Event()
|
| 483 |
+
with _DocumentProbe._lock:
|
| 484 |
+
previous, _DocumentProbe._active = _DocumentProbe._active, self
|
| 485 |
+
if previous is not None:
|
| 486 |
+
previous._halt.set()
|
| 487 |
+
threading.Thread(target=self._sample, name="amazon-document-probe", daemon=True).start()
|
| 488 |
+
|
| 489 |
+
def _sample(self):
|
| 490 |
+
deadline = time.monotonic() + 6 * 3600
|
| 491 |
+
while not self._halt.wait(0.5) and time.monotonic() < deadline:
|
| 492 |
+
rss = _rss_gb()
|
| 493 |
+
if rss is not None and (self.peak is None or rss > self.peak):
|
| 494 |
+
self.peak = rss
|
| 495 |
+
|
| 496 |
+
@classmethod
|
| 497 |
+
def downloaded(cls, document):
|
| 498 |
+
probe = cls._active
|
| 499 |
+
if probe is not None and not probe._halt.is_set():
|
| 500 |
+
probe.size = (document.size if _spooled(document)
|
| 501 |
+
else len(str(document or "").encode("utf-8", "replace")))
|
| 502 |
+
probe._say("downloaded", None)
|
| 503 |
+
|
| 504 |
+
def finish(self, rows):
|
| 505 |
+
self._halt.set()
|
| 506 |
+
self._say("written", rows)
|
| 507 |
+
|
| 508 |
+
def _say(self, phase, rows):
|
| 509 |
+
rss = _rss_gb()
|
| 510 |
+
peak = max(x for x in (self.peak, rss, 0.0) if x is not None)
|
| 511 |
+
size = "unknown" if self.size is None else f"{self.size:,}"
|
| 512 |
+
print(f"[aios-api] amazon document {phase}: table {self.table}, {self.report_type}, "
|
| 513 |
+
f"{size} bytes, {'-' if rows is None else f'{rows:,}'} rows, "
|
| 514 |
+
f"{time.monotonic() - self.started:.0f}s, rss {rss or 0:.2f} GB, peak {peak:.2f} GB",
|
| 515 |
+
flush=True)
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
def _flat_rows(lines, document_id, report_type):
|
| 519 |
+
"""`(columns, row generator)` over one flat file's lines, or `(None, ())` with no header line."""
|
| 520 |
+
reader = csv.reader(lines, delimiter="\t", quoting=csv.QUOTE_NONE)
|
| 521 |
+
try:
|
| 522 |
+
header = next(reader)
|
| 523 |
+
except StopIteration:
|
| 524 |
+
return None, ()
|
| 525 |
+
cols = _dedupe([_snake(h) for h in header])
|
| 526 |
+
width = len(cols)
|
| 527 |
+
# The read-only connectors_amazon report probe measures headers without capture metadata.
|
| 528 |
+
# Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
|
| 529 |
+
# BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
|
| 530 |
+
attributed = document_id is not None or report_type is not None
|
| 531 |
+
|
| 532 |
+
def rows():
|
| 533 |
+
for row_number, raw in enumerate(reader, 1):
|
| 534 |
+
if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
|
| 535 |
+
continue
|
| 536 |
+
vals = list(raw[:width]) + [None] * max(0, width - len(raw))
|
| 537 |
+
row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
|
| 538 |
+
if not buyer_column(c)}
|
| 539 |
+
yield _origin(row, document_id, row_number, report_type) if attributed else row
|
| 540 |
+
return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows()
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
def _parse_flat_document(document, *, document_id=None, report_type=None):
|
| 544 |
+
"""`parse_flat_file` over a spooled document: the header now, the rows again on every pass."""
|
| 545 |
+
if _blank(document):
|
| 546 |
+
return [], []
|
| 547 |
+
with document.open_text() as handle:
|
| 548 |
+
cols, _rows = _flat_rows(handle, document_id, report_type)
|
| 549 |
+
if cols is None:
|
| 550 |
+
return [], []
|
| 551 |
+
|
| 552 |
+
def produce():
|
| 553 |
+
with document.open_text() as handle:
|
| 554 |
+
yield from _flat_rows(handle, document_id, report_type)[1]
|
| 555 |
+
return cols, _DocumentRows(produce)
|
| 556 |
+
|
| 557 |
+
|
| 558 |
def parse_flat_file(text, *, document_id=None, report_type=None):
|
| 559 |
"""One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
|
| 560 |
|
|
|
|
| 567 |
swallow the rest of the row and silently merge it with the next one, so a catalogue with one
|
| 568 |
inch-mark in a title would lose rows with nothing raising.
|
| 569 |
"""
|
| 570 |
+
if _spooled(text):
|
| 571 |
+
return _parse_flat_document(text, document_id=document_id, report_type=report_type)
|
| 572 |
if not str(text or "").strip():
|
| 573 |
return [], []
|
| 574 |
buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
|
| 575 |
+
cols, rows = _flat_rows(buf, document_id, report_type)
|
| 576 |
+
if cols is None:
|
|
|
|
|
|
|
| 577 |
return [], []
|
| 578 |
+
return cols, list(rows)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 579 |
|
| 580 |
|
| 581 |
def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
|
|
| 586 |
with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
|
| 587 |
artifact is the authority; a field the payload omits is NULL.
|
| 588 |
"""
|
| 589 |
+
if _spooled(text):
|
| 590 |
+
return _parse_json_document(text, root, document_id=document_id, report_type=report_type)
|
| 591 |
try:
|
| 592 |
doc = json.loads(str(text or "") or "{}")
|
| 593 |
except (ValueError, TypeError) as exc:
|
|
|
|
| 603 |
body = [body]
|
| 604 |
if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
|
| 605 |
raise ValueError(f"Amazon JSON report has an invalid {root} row list")
|
| 606 |
+
rows = list(_json_rows(body, document_id, report_type))
|
| 607 |
+
return sorted({k for r in rows for k in r}), rows
|
| 608 |
+
|
| 609 |
+
|
| 610 |
+
def _json_rows(body, document_id, report_type):
|
| 611 |
+
"""Flatten and attribute each row object, numbering from one."""
|
| 612 |
attributed = document_id is not None or report_type is not None
|
| 613 |
for row_number, item in enumerate(body, 1):
|
| 614 |
row = _flatten_row(_without_buyer_data(item))
|
| 615 |
+
yield _origin(row, document_id, row_number, report_type) if attributed else row
|
| 616 |
+
|
| 617 |
+
|
| 618 |
+
class _JsonFallback(Exception):
|
| 619 |
+
"""A shape only the whole-text parser answers exactly: malformed, not an object, a repeated root."""
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
_JSON_WS = re.compile(r"[ \t\n\r]*")
|
| 623 |
+
_JSON_DECODER = json.JSONDecoder()
|
| 624 |
+
#: A single value may span this many characters before the stream hands it to `json.loads`.
|
| 625 |
+
_JSON_VALUE_CHARS = 256 << 20
|
| 626 |
+
|
| 627 |
+
|
| 628 |
+
class _JsonStream:
|
| 629 |
+
"""JSON read from a text stream one value at a time, with `json.loads` value semantics."""
|
| 630 |
+
|
| 631 |
+
def __init__(self, handle):
|
| 632 |
+
self.handle, self.buf, self.pos = handle, "", 0
|
| 633 |
+
|
| 634 |
+
def _fill(self):
|
| 635 |
+
piece = self.handle.read(DOCUMENT_READ_CHARS)
|
| 636 |
+
if not piece:
|
| 637 |
+
return False
|
| 638 |
+
self.buf = self.buf[self.pos:] + piece
|
| 639 |
+
self.pos = 0
|
| 640 |
+
return True
|
| 641 |
+
|
| 642 |
+
def peek(self):
|
| 643 |
+
while True:
|
| 644 |
+
self.pos = _JSON_WS.match(self.buf, self.pos).end()
|
| 645 |
+
if self.pos < len(self.buf) or not self._fill():
|
| 646 |
+
return self.buf[self.pos:self.pos + 1]
|
| 647 |
+
|
| 648 |
+
def take(self, char):
|
| 649 |
+
if self.peek() != char:
|
| 650 |
+
raise _JsonFallback()
|
| 651 |
+
self.pos += 1
|
| 652 |
+
|
| 653 |
+
def value(self):
|
| 654 |
+
self.peek()
|
| 655 |
+
while True:
|
| 656 |
+
try:
|
| 657 |
+
obj, end = _JSON_DECODER.raw_decode(self.buf, self.pos)
|
| 658 |
+
except ValueError:
|
| 659 |
+
if len(self.buf) - self.pos < _JSON_VALUE_CHARS and self._fill():
|
| 660 |
+
continue
|
| 661 |
+
raise _JsonFallback() from None
|
| 662 |
+
# A number ending at the buffer's edge may continue in the next piece.
|
| 663 |
+
if end == len(self.buf) and self._fill():
|
| 664 |
+
continue
|
| 665 |
+
self.pos = end
|
| 666 |
+
return obj
|
| 667 |
+
|
| 668 |
+
def skip(self):
|
| 669 |
+
if self.peek() != "[":
|
| 670 |
+
self.value()
|
| 671 |
+
return
|
| 672 |
+
self.pos += 1
|
| 673 |
+
if self.peek() == "]":
|
| 674 |
+
self.pos += 1
|
| 675 |
+
return
|
| 676 |
+
while True:
|
| 677 |
+
self.skip()
|
| 678 |
+
char = self.peek()
|
| 679 |
+
self.pos += 1
|
| 680 |
+
if char == "]":
|
| 681 |
+
return
|
| 682 |
+
if char != ",":
|
| 683 |
+
raise _JsonFallback()
|
| 684 |
+
|
| 685 |
+
def items(self):
|
| 686 |
+
if self.peek() == "{":
|
| 687 |
+
yield self.value()
|
| 688 |
+
return
|
| 689 |
+
self.take("[")
|
| 690 |
+
if self.peek() == "]":
|
| 691 |
+
self.pos += 1
|
| 692 |
+
return
|
| 693 |
+
while True:
|
| 694 |
+
item = self.value()
|
| 695 |
+
if not isinstance(item, dict):
|
| 696 |
+
raise _JsonFallback()
|
| 697 |
+
yield item
|
| 698 |
+
char = self.peek()
|
| 699 |
+
self.pos += 1
|
| 700 |
+
if char == "]":
|
| 701 |
+
return
|
| 702 |
+
if char != ",":
|
| 703 |
+
raise _JsonFallback()
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
def _json_root_items(document, root):
|
| 707 |
+
"""Each row object under the top-level `root` key of a spooled document, decoded one at a time.
|
| 708 |
+
|
| 709 |
+
Raises `_JsonFallback` for any shape `json.loads` must judge: malformed text, a document that is
|
| 710 |
+
not an object, a missing or repeated root, or a root that is not a list of objects.
|
| 711 |
+
"""
|
| 712 |
+
with document.open_text() as handle:
|
| 713 |
+
stream, found = _JsonStream(handle), False
|
| 714 |
+
stream.take("{")
|
| 715 |
+
if stream.peek() == "}":
|
| 716 |
+
stream.pos += 1
|
| 717 |
+
else:
|
| 718 |
+
while True:
|
| 719 |
+
key = stream.value()
|
| 720 |
+
if not isinstance(key, str):
|
| 721 |
+
raise _JsonFallback()
|
| 722 |
+
stream.take(":")
|
| 723 |
+
if key == root:
|
| 724 |
+
if found:
|
| 725 |
+
raise _JsonFallback()
|
| 726 |
+
found = True
|
| 727 |
+
yield from stream.items()
|
| 728 |
+
else:
|
| 729 |
+
stream.skip()
|
| 730 |
+
char = stream.peek()
|
| 731 |
+
stream.pos += 1
|
| 732 |
+
if char == "}":
|
| 733 |
+
break
|
| 734 |
+
if char != ",":
|
| 735 |
+
raise _JsonFallback()
|
| 736 |
+
if stream.peek() or not found:
|
| 737 |
+
raise _JsonFallback()
|
| 738 |
+
|
| 739 |
+
|
| 740 |
+
def _parse_json_document(document, root, *, document_id=None, report_type=None):
|
| 741 |
+
"""`parse_json_report` over a spooled document; any shape it cannot stream reads the whole text."""
|
| 742 |
+
rows = _DocumentRows(lambda: _json_rows(_json_root_items(document, root), document_id, report_type))
|
| 743 |
+
try:
|
| 744 |
+
keys = rows.keys()
|
| 745 |
+
except _JsonFallback:
|
| 746 |
+
return parse_json_report(document.read_text(), root, document_id=document_id,
|
| 747 |
+
report_type=report_type)
|
| 748 |
+
if not len(rows):
|
| 749 |
+
return [], []
|
| 750 |
+
return sorted(keys), rows
|
| 751 |
|
| 752 |
|
| 753 |
def _flatten_row(obj, prefix=""):
|
|
|
|
| 956 |
|
| 957 |
def _tag_fba_inventory(rows):
|
| 958 |
"""Mark report rows so one combined inventory table never conflates FBA with AWD."""
|
| 959 |
+
if isinstance(rows, _DocumentRows):
|
| 960 |
+
return _stamp_rows(rows, {"inventory_source": "FBA"})
|
| 961 |
for row in rows or ():
|
| 962 |
row["inventory_source"] = "FBA"
|
| 963 |
return rows
|
|
|
|
| 1047 |
return "VARCHAR"
|
| 1048 |
if not rows:
|
| 1049 |
return "VARCHAR"
|
| 1050 |
+
if isinstance(rows, _DocumentRows):
|
| 1051 |
+
return rows.sniffed(name)
|
| 1052 |
seen, kinds = 0, set()
|
| 1053 |
for r in rows:
|
| 1054 |
v = r.get(name)
|
| 1055 |
if v is None or v == "":
|
| 1056 |
continue
|
| 1057 |
+
kinds.add(_value_kind(v))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1058 |
seen += 1
|
| 1059 |
if seen >= 500:
|
| 1060 |
break
|
| 1061 |
+
return _kinds_type(kinds)
|
|
|
|
|
|
|
| 1062 |
|
| 1063 |
|
| 1064 |
def _cell(value, kind):
|
|
|
|
| 1137 |
or type(r.get("_report_row_number")) is not int
|
| 1138 |
or r["_report_row_number"] < 1 for r in rows):
|
| 1139 |
raise ValueError("refusing Amazon rows without complete source attribution")
|
| 1140 |
+
ph = "(" + ", ".join("?" for _ in cols) + ")"
|
| 1141 |
+
insert = f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES '
|
| 1142 |
+
per = max(1, min(INSERT_BATCH_ROWS, INSERT_BATCH_VALUES // len(cols)))
|
| 1143 |
+
full = insert + ", ".join([ph] * per)
|
| 1144 |
+
|
| 1145 |
+
def write(batch):
|
| 1146 |
+
con.execute(full if len(batch) == per else insert + ", ".join([ph] * len(batch)),
|
| 1147 |
+
[value for row in batch for value in row])
|
| 1148 |
+
|
| 1149 |
+
# Multi-row statements in batches: no copy of every row exists at once, the caller's transaction
|
| 1150 |
+
# keeps the window all-or-nothing, and values bind and cast exactly as `executemany` did.
|
| 1151 |
+
count, batch = 0, []
|
| 1152 |
+
for r in rows:
|
| 1153 |
+
batch.append([_cell(r.get(c), types.get(c, "VARCHAR")) for c in cols])
|
| 1154 |
+
if len(batch) == per:
|
| 1155 |
+
write(batch)
|
| 1156 |
+
count, batch = count + len(batch), []
|
| 1157 |
+
if batch:
|
| 1158 |
+
write(batch)
|
| 1159 |
+
count += len(batch)
|
| 1160 |
+
return count
|
| 1161 |
|
| 1162 |
|
| 1163 |
def replace_keys_for(table):
|
|
|
|
| 1177 |
return _replace(con, table, cols, types, rows), None
|
| 1178 |
quote = lambda name: '"' + str(name).replace('"', '""') + '"'
|
| 1179 |
target = quote(table)
|
| 1180 |
+
# The window's rows are staged in batches inside the transaction, typed as the mirror and
|
| 1181 |
+
# converted by the insert's own cells, so NULL and numeric keys agree and no key list exists.
|
| 1182 |
+
temporary = quote("amazon_window_rows_" + uuid.uuid4().hex)
|
| 1183 |
+
columns = ", ".join(quote(c) for c in cols)
|
|
|
|
| 1184 |
con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
|
| 1185 |
try:
|
|
|
|
|
|
|
| 1186 |
predicate = " AND ".join(
|
| 1187 |
f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
|
| 1188 |
con.execute("BEGIN TRANSACTION")
|
| 1189 |
try:
|
| 1190 |
+
count = _replace(con, temporary, cols, types, rows)
|
| 1191 |
con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
|
| 1192 |
+
con.execute(f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {temporary}")
|
| 1193 |
con.execute("COMMIT")
|
| 1194 |
except Exception:
|
| 1195 |
con.execute("ROLLBACK")
|
| 1196 |
raise
|
| 1197 |
return count, None
|
| 1198 |
finally:
|
| 1199 |
+
con.execute(f"DROP TABLE IF EXISTS {temporary}")
|
| 1200 |
|
| 1201 |
|
| 1202 |
def _windows(start, end, size=MAX_WINDOW_DAYS):
|
|
|
|
| 1542 |
return 0
|
| 1543 |
|
| 1544 |
# Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
|
| 1545 |
+
text = _document_text(text)
|
| 1546 |
doc = json.loads(text)
|
| 1547 |
raw = doc.get(spec["json_root"])
|
| 1548 |
if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
|
|
|
|
| 3101 |
|
| 3102 |
def document_row_count(text, roots=()):
|
| 3103 |
"""Count source records before flattening, privacy filtering or mirror transformations."""
|
| 3104 |
+
if _spooled(text):
|
| 3105 |
+
try:
|
| 3106 |
+
if roots:
|
| 3107 |
+
return sum(sum(1 for _item in _json_root_items(text, root)) for root in roots)
|
| 3108 |
+
with text.open_text() as handle:
|
| 3109 |
+
reader = csv.reader(handle, delimiter="\t", quoting=csv.QUOTE_NONE)
|
| 3110 |
+
next(reader, None)
|
| 3111 |
+
return sum(bool(raw) and not (len(raw) == 1 and not str(raw[0]).strip()) for raw in reader)
|
| 3112 |
+
except _JsonFallback:
|
| 3113 |
+
return document_row_count(text.read_text(), roots)
|
| 3114 |
if roots:
|
| 3115 |
doc = json.loads(text)
|
| 3116 |
return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
|
|
|
|
| 3155 |
stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
|
| 3156 |
if not stamp and (table == "listings" or not rows):
|
| 3157 |
raise ValueError("Amazon snapshot returned no data date or report creation date")
|
| 3158 |
+
def stamped(row):
|
|
|
|
| 3159 |
period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
|
| 3160 |
row["_snapshot_date"] = period
|
| 3161 |
if table == "inventory":
|
| 3162 |
row["date"] = period
|
| 3163 |
row["inventory_source"] = "FBA"
|
| 3164 |
+
return row
|
| 3165 |
+
|
| 3166 |
+
lazy = isinstance(rows, _DocumentRows)
|
| 3167 |
+
rows = rows.map(stamped) if lazy else rows
|
| 3168 |
+
counts, identities, total = {}, set(), 0
|
| 3169 |
+
for row in rows:
|
| 3170 |
+
period = (row if lazy else stamped(row))["_snapshot_date"]
|
| 3171 |
identity = _snapshot_identity(table, row)
|
| 3172 |
if identity in identities:
|
| 3173 |
raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
|
| 3174 |
identities.add(identity)
|
| 3175 |
counts[period] = counts.get(period, 0) + 1
|
| 3176 |
+
total += 1
|
| 3177 |
+
if total != source_count:
|
| 3178 |
raise ValueError(f"Amazon {table} snapshot row count differs from its document")
|
| 3179 |
if not counts:
|
| 3180 |
counts[_utc(stamp).date().isoformat()] = 0
|
| 3181 |
+
columns = sorted(set(columns) | {"_snapshot_date"}
|
| 3182 |
+
| (rows.keys() if lazy else {k for row in rows for k in row}))
|
| 3183 |
if table == "inventory":
|
| 3184 |
columns = sorted(set(columns) | {"inventory_source"})
|
| 3185 |
mirror = table_name(table)
|
|
|
|
| 3326 |
if options:
|
| 3327 |
kwargs["options"] = options
|
| 3328 |
text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
|
| 3329 |
+
_DocumentProbe.downloaded(text)
|
| 3330 |
parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
|
| 3331 |
+
if spec.get("json_root") and meta.get("cancelled") and _blank(text):
|
| 3332 |
# ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
|
| 3333 |
# returns an empty document for it, S66-T42), but the JSON parser read "" as a document
|
| 3334 |
# with no row root and REFUSED it. So the two B2B opportunity reports never earned a
|
|
|
|
| 3338 |
elif spec.get("json_root"):
|
| 3339 |
columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
|
| 3340 |
if spec.get("json_extra_roots"):
|
| 3341 |
+
rows = _stamp_rows(rows, {"_json_root": spec["json_root"]})
|
| 3342 |
+
parts = [rows]
|
| 3343 |
for root in spec["json_extra_roots"]:
|
| 3344 |
extra_columns, extra = parse_json_report(text, root, **parser_args)
|
| 3345 |
+
parts.append(_stamp_rows(extra, {"_json_root": root}))
|
|
|
|
| 3346 |
columns = merge_columns(columns, extra_columns)
|
| 3347 |
+
rows = (_DocumentRows.chain(*parts) if any(isinstance(part, _DocumentRows) for part in parts)
|
| 3348 |
+
else [row for part in parts for row in part])
|
| 3349 |
else:
|
| 3350 |
columns, rows = parse_flat_file(text, **parser_args)
|
| 3351 |
if table in LEGACY_SNAPSHOT_TABLES:
|
|
|
|
| 3359 |
period, column = _utc(w0).strftime("%Y-%m"), "month"
|
| 3360 |
else:
|
| 3361 |
period, column = _iso(w0), "_period_start"
|
| 3362 |
+
def stamped(row):
|
| 3363 |
row[column] = period
|
| 3364 |
if w1 is not None:
|
| 3365 |
row["_period_end"] = _iso(w1)
|
|
|
|
| 3367 |
row["fulfillment_center"] = (row.get("fulfillment_center") or
|
| 3368 |
row.get("fulfillment_center_id") or
|
| 3369 |
row.get("fulfilment_center") or "all")
|
| 3370 |
+
return row
|
| 3371 |
+
|
| 3372 |
+
rows = rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
|
| 3373 |
+
columns = sorted(set(columns) | (rows.keys() if isinstance(rows, _DocumentRows)
|
| 3374 |
+
else {k for row in rows for k in row}))
|
| 3375 |
roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
|
| 3376 |
# HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
|
| 3377 |
+
empty = bool(roots) and ((meta.get("cancelled") and _blank(text)) or
|
| 3378 |
(table in ("b2b_recommended", "b2b_not_yet_on_amazon")
|
| 3379 |
+
and isinstance(json.loads(_document_text(text) or "{}"), dict)
|
| 3380 |
+
and spec["json_root"] not in json.loads(_document_text(text) or "{}")))
|
| 3381 |
marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
|
| 3382 |
"column": column, "count": 0 if empty else document_row_count(text, roots)}
|
| 3383 |
return _replace_period(con, table, rows, columns, column, period, marker=marker,
|
| 3384 |
snapshot=bool(spec.get("snapshot")))
|
| 3385 |
|
| 3386 |
|
| 3387 |
+
def _stamp_rows(rows, values):
|
| 3388 |
+
"""Set `values` on every row, on each pass when the rows are lazy."""
|
| 3389 |
+
def stamped(row):
|
| 3390 |
+
row.update(values)
|
| 3391 |
+
return row
|
| 3392 |
+
return rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
|
| 3393 |
+
|
| 3394 |
+
|
| 3395 |
def _order_metric_totals(records, *, flattened=False):
|
| 3396 |
totals = {"orders": 0, "units": 0, "sales": {}}
|
| 3397 |
for row in records:
|
|
|
|
| 3927 |
settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
|
| 3928 |
for item in discovered[issued_index:])
|
| 3929 |
break
|
| 3930 |
+
probe = _DocumentProbe(table, rtype)
|
| 3931 |
operation = "getReportDocument"
|
| 3932 |
documents += 1
|
| 3933 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3934 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 3935 |
text = cx.settlement_document(fields, document_id)
|
| 3936 |
+
_DocumentProbe.downloaded(text)
|
| 3937 |
cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
|
| 3938 |
count = 0
|
| 3939 |
if cols and rows:
|
|
|
|
| 3942 |
# A newly issued document still replaces its own settlement's lines.
|
| 3943 |
count = write_window(table, cols, types, rows)
|
| 3944 |
written += count
|
| 3945 |
+
probe.finish(count)
|
| 3946 |
# Record completion after the write, including header-only and blank files.
|
| 3947 |
_record_state(con, entity_key("settlement_doc_" + document_id), "done",
|
| 3948 |
count, issued.get("dataEndTime"))
|
|
|
|
| 4006 |
_record_state(con, entity_key("limit_" + table), "done", detail="{}")
|
| 4007 |
for w0, w1 in margin_windows:
|
| 4008 |
# One detail window is one budget task, including its reconciliation query.
|
| 4009 |
+
probe = _DocumentProbe(table, spec.get("report") or operation)
|
| 4010 |
documents += 1
|
| 4011 |
_sync_event("amazon.sync", tenant_key, table,
|
| 4012 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 4013 |
+
count = _sync_margin_window(con, table, fields, w0, w1,
|
| 4014 |
+
max_wait=max_wait, log=log)
|
| 4015 |
+
probe.finish(count)
|
| 4016 |
+
written += count
|
| 4017 |
landed_window = True
|
| 4018 |
if table in LEGACY_SNAPSHOT_TABLES:
|
| 4019 |
# The dated replacement already landed; AWD must not clear that
|
|
|
|
| 4033 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 4034 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
| 4035 |
documents += 1
|
| 4036 |
+
probe = _DocumentProbe(table, rtype)
|
| 4037 |
with _report_resume(
|
| 4038 |
con, table, rtype, start=_iso(w0),
|
| 4039 |
end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
|
|
|
|
| 4044 |
_sync_event("amazon.sync", tenant_key, table,
|
| 4045 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 4046 |
text, meta = cx.report_document(fields, rtype, **resume_kwargs)
|
| 4047 |
+
_DocumentProbe.downloaded(text)
|
| 4048 |
landed_window = True
|
| 4049 |
if root:
|
| 4050 |
parsed_cols, rows = parse_json_report(
|
|
|
|
| 4120 |
if cx.REPORT_TYPES[table].get("monthly"):
|
| 4121 |
_record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
|
| 4122 |
"count": len(rows)})
|
| 4123 |
+
probe.finish(len(rows) if cols else 0)
|
| 4124 |
if first_load:
|
| 4125 |
for reached in (table, sibling):
|
| 4126 |
if reached:
|
|
|
|
| 4355 |
lines by construction, so this drops line detail rather than losing an order. `order_items`
|
| 4356 |
keeps every row, which is why both tables read the same document instead of costing two.
|
| 4357 |
"""
|
| 4358 |
+
if isinstance(rows, _DocumentRows):
|
| 4359 |
+
return rows.pipe(_first_row_per_order)
|
| 4360 |
+
return list(_first_row_per_order(rows))
|
| 4361 |
+
|
| 4362 |
+
|
| 4363 |
+
def _first_row_per_order(rows):
|
| 4364 |
+
seen = set()
|
| 4365 |
for r in rows:
|
| 4366 |
oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
|
| 4367 |
if oid and oid in seen:
|
| 4368 |
continue
|
| 4369 |
if oid:
|
| 4370 |
seen.add(oid)
|
| 4371 |
+
yield r
|
|
|
|
| 4372 |
|
| 4373 |
|
| 4374 |
def validate(tenant_key="royal-imports", rt=None, log=print):
|
|
|
|
| 6702 |
ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
|
| 6703 |
ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
|
| 6704 |
ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
|
| 6705 |
+
same, detail = _spooled_parse_selftest()
|
| 6706 |
+
ck("NC-AZ34 a spooled document parses and counts exactly as its text does", same, detail)
|
| 6707 |
+
bounded, detail = _bounded_document_selftest()
|
| 6708 |
+
ck("NC-AZ35 a large synthetic report parses and writes in bounded memory", bounded, detail)
|
| 6709 |
+
log(f"\n{'GREEN' if not fails else 'RED'}: {35 - len(fails)}/35")
|
| 6710 |
return 0 if not fails else 1
|
| 6711 |
|
| 6712 |
|
| 6713 |
+
def _spool_fixture(text, *, gz=False, encoding="utf-8"):
|
| 6714 |
+
"""A `ReportDocument` made by the client's own spool path from local bytes, no network."""
|
| 6715 |
+
import gzip
|
| 6716 |
+
raw = text.encode(encoding) if isinstance(text, str) else text
|
| 6717 |
+
handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
|
| 6718 |
+
with os.fdopen(handle, "wb") as out:
|
| 6719 |
+
out.write(gzip.compress(raw) if gz else raw)
|
| 6720 |
+
return _client()._spooled_document(path, "GZIP" if gz else "")
|
| 6721 |
+
|
| 6722 |
+
|
| 6723 |
+
def _spooled_parse_selftest():
|
| 6724 |
+
"""D-7081: the streaming readers answer every fixture exactly as the whole-text parsers do."""
|
| 6725 |
+
origin = {"document_id": "selftest-document", "report_type": "selftest-report"}
|
| 6726 |
+
|
| 6727 |
+
def outcome(parse):
|
| 6728 |
+
try:
|
| 6729 |
+
cols, rows = parse()
|
| 6730 |
+
return "ok", list(cols), [dict(row) for row in rows]
|
| 6731 |
+
except Exception as exc: # noqa: BLE001
|
| 6732 |
+
return "error", type(exc).__name__, str(exc)
|
| 6733 |
+
|
| 6734 |
+
def count(text, roots):
|
| 6735 |
+
try:
|
| 6736 |
+
return document_row_count(text, roots)
|
| 6737 |
+
except Exception as exc: # noqa: BLE001
|
| 6738 |
+
return type(exc).__name__
|
| 6739 |
+
|
| 6740 |
+
flat = ["", " \n\t", "a\tb\r\n1\t2\r\n\r\n3\t4", "\ufeffsku\tsku\tbuyer-email\nA\tB\tx\n\"q\t1\n",
|
| 6741 |
+
"x\ty\r1\r2\t3\r\r", "sku\tname\nA\tsmart \u201cquote\u201d\n"]
|
| 6742 |
+
blobs = [(t, {}) for t in flat] + [(t, {"gz": True}) for t in flat] + [(flat[-1], {"encoding": "cp1252"})]
|
| 6743 |
+
misses = []
|
| 6744 |
+
for index, (text, how) in enumerate(blobs):
|
| 6745 |
+
expected = text if how.get("encoding") != "cp1252" else text.encode("cp1252").decode("cp1252")
|
| 6746 |
+
got = outcome(lambda: parse_flat_file(_spool_fixture(text, **how), **origin))
|
| 6747 |
+
if got != outcome(lambda: parse_flat_file(expected, **origin)):
|
| 6748 |
+
misses.append(f"flat {index}")
|
| 6749 |
+
if expected.strip() and count(_spool_fixture(text, **how), ()) != count(expected, ()):
|
| 6750 |
+
misses.append(f"flat count {index}")
|
| 6751 |
+
documents = ['', '[]', '{"rows": []}', '{"rows": {"a": 1}}', '{"rows": [1]}', '{"rows": [{"a": 1},]}',
|
| 6752 |
+
'{"rows": [{"a": 1}]} x', '{"rows": [{"a": 1}], "rows": [{"b": 2}]}', '{"other": 1}',
|
| 6753 |
+
' {"spec": {"q": [1, [2]]}, "rows": [{"a": 1, "b": null, "n": {"c": "\u2014"}}, '
|
| 6754 |
+
'{"a": 2.5, "l": [1, 2]}], "tail": [[1], []]} ',
|
| 6755 |
+
'{"rows": [{"_report_document_id": "vendor"}]}', '{"rows": [{"buyerEmail": "e", "a": 123456789012345678901}]}',
|
| 6756 |
+
'{"x": NaN, "rows": [{"v": Infinity, "w": -0.0, "s": "\\ud83d"}]}']
|
| 6757 |
+
for index, text in enumerate(documents):
|
| 6758 |
+
if outcome(lambda: parse_json_report(_spool_fixture(text), "rows", **origin)) != outcome(
|
| 6759 |
+
lambda: parse_json_report(text, "rows", **origin)):
|
| 6760 |
+
misses.append(f"json {index}")
|
| 6761 |
+
if count(_spool_fixture(text), ("rows",)) != count(text, ("rows",)):
|
| 6762 |
+
misses.append(f"json count {index}")
|
| 6763 |
+
return not misses, misses
|
| 6764 |
+
|
| 6765 |
+
|
| 6766 |
+
def _bounded_document_selftest(rows=24_000):
|
| 6767 |
+
"""D-7081: a report larger than the bound parses, sniffs and writes with a bounded Python peak.
|
| 6768 |
+
|
| 6769 |
+
The whole-text path holds the decoded text and every row at once, so it fails this bound; the
|
| 6770 |
+
spooled path holds one read piece and one insert batch.
|
| 6771 |
+
"""
|
| 6772 |
+
import gzip
|
| 6773 |
+
import tracemalloc
|
| 6774 |
+
bound = 10 << 20
|
| 6775 |
+
handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
|
| 6776 |
+
os.close(handle)
|
| 6777 |
+
with gzip.open(path, "wt", encoding="utf-8") as out:
|
| 6778 |
+
out.write('{"reportSpecification": {"reportType": "selftest"}, "dataByDepartmentAndSearchTerm": [')
|
| 6779 |
+
for i in range(rows):
|
| 6780 |
+
out.write(("," if i else "") + json.dumps({
|
| 6781 |
+
"departmentName": "Amazon.com", "searchTerm": f"term {i} \u2014 caf\u00e9",
|
| 6782 |
+
"searchFrequencyRank": i + 1, "clickedAsin": f"B0{i:08d}",
|
| 6783 |
+
"clickedItemName": f"Product title number {i} with several words", "clickShareRank": i % 3 + 1,
|
| 6784 |
+
"clickShare": round(i / 7 % 1, 4), "conversionShare": round(i / 11 % 1, 4)}))
|
| 6785 |
+
out.write("]}")
|
| 6786 |
+
document = _client()._spooled_document(path, "GZIP")
|
| 6787 |
+
text_bytes = 2 * len(document)
|
| 6788 |
+
con = datastore.duckdb.connect(":memory:")
|
| 6789 |
+
table = "amzn_brand_analytics"
|
| 6790 |
+
# A first bound statement imports DuckDB's lazy modules; that one-time cost is not a row's.
|
| 6791 |
+
con.execute("SELECT ?", [1]).fetchall()
|
| 6792 |
+
tracemalloc.start()
|
| 6793 |
+
try:
|
| 6794 |
+
cols, parsed = parse_json_report(document, "dataByDepartmentAndSearchTerm",
|
| 6795 |
+
document_id="selftest-document", report_type="selftest")
|
| 6796 |
+
cols = merge_columns([c for c, _t in amazon_schema.columns_for("brand_analytics")], cols)
|
| 6797 |
+
types = _ensure(con, table, cols, parsed, declared=dict(amazon_schema.columns_for("brand_analytics")))
|
| 6798 |
+
first, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
|
| 6799 |
+
peak = tracemalloc.get_traced_memory()[1]
|
| 6800 |
+
finally:
|
| 6801 |
+
tracemalloc.stop()
|
| 6802 |
+
again, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
|
| 6803 |
+
stored = con.execute(f"SELECT COUNT(*), COUNT(DISTINCT _report_row_number) FROM {table}").fetchone()
|
| 6804 |
+
con.close()
|
| 6805 |
+
document.close()
|
| 6806 |
+
detail = {"rows": len(parsed), "first": first, "again": again, "stored": stored,
|
| 6807 |
+
"text_bytes": text_bytes, "peak": peak, "bound": bound}
|
| 6808 |
+
return (len(parsed) == first == again == rows and stored == (rows, rows)
|
| 6809 |
+
and text_bytes > bound and peak < bound), detail
|
| 6810 |
+
|
| 6811 |
+
|
| 6812 |
def main(argv=None):
|
| 6813 |
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 6814 |
ap.add_argument("--sync", action="store_true")
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
e24ce7c6a61cd78dfd92b798e5cec7eff002f567fe56d2190d77cdc7ebe0e7b1
|