fsanyoto commited on
Commit
e91167d
·
verified ·
1 Parent(s): 45a692d

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "b07ce2572",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "ea8618de4",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- b07ce2572
 
1
+ ea8618de4
platform/harness/amazon_client.py CHANGED
@@ -45,9 +45,13 @@ Ideation record, with every owner ruling: `AutoWiki/research/ideation/amazon-con
45
  from __future__ import annotations
46
 
47
  import contextvars
 
48
  import json
 
49
  import random
50
  import re
 
 
51
  import threading
52
  import time
53
  from datetime import date, datetime, timezone, timedelta
@@ -1994,7 +1998,7 @@ def report_document(fields, report_type, start=None, end=None, options=None,
1994
  url = str(meta.get("url") or "")
1995
  if not url:
1996
  raise AmazonError(f"the {report_type} report document carried no download URL")
1997
- text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
1998
  say(f" {report_type}: {len(text):,} chars")
1999
  return text, {"reportId": rid, "reportDocumentId": doc_id,
2000
  "dataStartTime": got.get("dataStartTime"),
@@ -2090,8 +2094,106 @@ def _workbook_as_tsv(raw):
2090
  return "\n".join(lines)
2091
 
2092
 
2093
- def _download_document(url, compression="", timeout=120):
2094
- """Fetch one presigned report document and decode it. -> text.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2095
 
2096
  ⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
2097
  string, which for an S3 presigned URL is the entire credential.
@@ -2103,11 +2205,20 @@ def _download_document(url, compression="", timeout=120):
2103
  # apply OPERATIONS here: its host and signature belong to Amazon's document download service.
2104
  req = urllib.request.Request(url, method="GET")
2105
  req.add_header("User-Agent", _UA)
 
2106
  try:
2107
  with urllib.request.urlopen(req, timeout=timeout) as r:
2108
- raw = r.read()
 
 
 
 
2109
  except Exception as e: # noqa: BLE001
 
 
2110
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
 
 
2111
  # S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
2112
  # `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
2113
  # algorithm the bytes were decoded as text and every economics window failed to parse. The
 
45
  from __future__ import annotations
46
 
47
  import contextvars
48
+ import gzip
49
  import json
50
+ import os
51
  import random
52
  import re
53
+ import shutil
54
+ import tempfile
55
  import threading
56
  import time
57
  from datetime import date, datetime, timezone, timedelta
 
1998
  url = str(meta.get("url") or "")
1999
  if not url:
2000
  raise AmazonError(f"the {report_type} report document carried no download URL")
2001
+ text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True)
2002
  say(f" {report_type}: {len(text):,} chars")
2003
  return text, {"reportId": rid, "reportDocumentId": doc_id,
2004
  "dataStartTime": got.get("dataStartTime"),
 
2094
  return "\n".join(lines)
2095
 
2096
 
2097
+ class ReportDocument:
2098
+ """One downloaded report document spooled to a temporary file, read back as text in pieces.
2099
+
2100
+ ⛔ A report has no row cap (standing rule 1), so its text never sits in memory whole: the Brand
2101
+ Analytics month held the body, its decompressed copy and its decoded text at once (D-7081).
2102
+ The file is removed when the object is closed or collected.
2103
+ """
2104
+
2105
+ def __init__(self, path, encoding, errors, chars, size, compressed):
2106
+ self.path, self.encoding, self.errors = path, encoding, errors
2107
+ self.chars, self.size, self.compressed = chars, size, compressed
2108
+
2109
+ def open_text(self):
2110
+ """The decoded text as a stream; `newline=""` keeps line ends exactly as `bytes.decode` did."""
2111
+ return open(self.path, "r", encoding=self.encoding, errors=self.errors, newline="")
2112
+
2113
+ def read_text(self):
2114
+ """The whole text, for a consumer whose document is small."""
2115
+ with self.open_text() as handle:
2116
+ return handle.read()
2117
+
2118
+ def __len__(self):
2119
+ return self.chars
2120
+
2121
+ def close(self):
2122
+ path, self.path = getattr(self, "path", None), None
2123
+ if path:
2124
+ try:
2125
+ os.unlink(path)
2126
+ except Exception: # noqa: BLE001
2127
+ pass
2128
+
2129
+ def __del__(self):
2130
+ self.close()
2131
+
2132
+
2133
+ _SPOOL_CHUNK = 1 << 20
2134
+
2135
+
2136
+ def _spool_file(prefix):
2137
+ handle, path = tempfile.mkstemp(prefix=prefix, suffix=".part")
2138
+ os.close(handle)
2139
+ return path
2140
+
2141
+
2142
+ def _unlink(path):
2143
+ try:
2144
+ os.unlink(path)
2145
+ except OSError:
2146
+ pass
2147
+
2148
+
2149
+ def _decoded_chars(path, encoding, errors="strict"):
2150
+ """Characters in the file under one encoding; raises UnicodeDecodeError like `bytes.decode`."""
2151
+ chars = 0
2152
+ with open(path, "r", encoding=encoding, errors=errors, newline="") as handle:
2153
+ while True:
2154
+ piece = handle.read(_SPOOL_CHUNK)
2155
+ if not piece:
2156
+ return chars
2157
+ chars += len(piece)
2158
+
2159
+
2160
+ def _spooled_document(body, compression):
2161
+ """The spooled body as a `ReportDocument`: gunzip and decode stream through disk, never memory."""
2162
+ compressed = os.path.getsize(body)
2163
+ with open(body, "rb") as handle:
2164
+ head = handle.read(4)
2165
+ if not compression and head[:2] == b"\x1f\x8b":
2166
+ compression = "GZIP"
2167
+ if str(compression).upper() == "GZIP":
2168
+ plain = _spool_file("amazon-document-")
2169
+ try:
2170
+ with gzip.open(body, "rb") as source, open(plain, "wb") as out:
2171
+ shutil.copyfileobj(source, out, _SPOOL_CHUNK)
2172
+ except Exception as e: # noqa: BLE001
2173
+ _unlink(plain)
2174
+ raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
2175
+ finally:
2176
+ _unlink(body)
2177
+ body = plain
2178
+ with open(body, "rb") as handle:
2179
+ head = handle.read(4)
2180
+ if head[:4] == b"PK\x03\x04":
2181
+ try:
2182
+ with open(body, "rb") as handle:
2183
+ return _workbook_as_tsv(handle.read())
2184
+ finally:
2185
+ _unlink(body)
2186
+ size = os.path.getsize(body)
2187
+ for enc in ("utf-8", "cp1252"):
2188
+ try:
2189
+ return ReportDocument(body, enc, "strict", _decoded_chars(body, enc), size, compressed)
2190
+ except UnicodeDecodeError:
2191
+ continue
2192
+ return ReportDocument(body, "utf-8", "replace", _decoded_chars(body, "utf-8", "replace"), size, compressed)
2193
+
2194
+
2195
+ def _download_document(url, compression="", timeout=120, *, spool=False):
2196
+ """Fetch one presigned report document and decode it. -> text, or a `ReportDocument` when spooled.
2197
 
2198
  ⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
2199
  string, which for an S3 presigned URL is the entire credential.
 
2205
  # apply OPERATIONS here: its host and signature belong to Amazon's document download service.
2206
  req = urllib.request.Request(url, method="GET")
2207
  req.add_header("User-Agent", _UA)
2208
+ body = _spool_file("amazon-document-") if spool else None
2209
  try:
2210
  with urllib.request.urlopen(req, timeout=timeout) as r:
2211
+ if body:
2212
+ with open(body, "wb") as out:
2213
+ shutil.copyfileobj(r, out, _SPOOL_CHUNK)
2214
+ else:
2215
+ raw = r.read()
2216
  except Exception as e: # noqa: BLE001
2217
+ if body:
2218
+ _unlink(body)
2219
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
2220
+ if body:
2221
+ return _spooled_document(body, compression)
2222
  # S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
2223
  # `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
2224
  # algorithm the bytes were decoded as text and every economics window failed to parse. The
platform/harness/amazon_store.py CHANGED
@@ -327,6 +327,234 @@ def _without_buyer_data(value):
327
  return value
328
 
329
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
330
  def parse_flat_file(text, *, document_id=None, report_type=None):
331
  """One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
332
 
@@ -339,29 +567,15 @@ def parse_flat_file(text, *, document_id=None, report_type=None):
339
  swallow the rest of the row and silently merge it with the next one, so a catalogue with one
340
  inch-mark in a title would lose rows with nothing raising.
341
  """
 
 
342
  if not str(text or "").strip():
343
  return [], []
344
  buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
345
- reader = csv.reader(buf, delimiter="\t", quoting=csv.QUOTE_NONE)
346
- try:
347
- header = next(reader)
348
- except StopIteration:
349
  return [], []
350
- cols = _dedupe([_snake(h) for h in header])
351
- width = len(cols)
352
- rows = []
353
- # The read-only connectors_amazon report probe measures headers without capture metadata.
354
- # Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
355
- # BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
356
- attributed = document_id is not None or report_type is not None
357
- for row_number, raw in enumerate(reader, 1):
358
- if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
359
- continue
360
- vals = list(raw[:width]) + [None] * max(0, width - len(raw))
361
- row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
362
- if not buyer_column(c)}
363
- rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
364
- return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows
365
 
366
 
367
  def parse_json_report(text, root, *, document_id=None, report_type=None):
@@ -372,6 +586,8 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
372
  with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
373
  artifact is the authority; a field the payload omits is NULL.
374
  """
 
 
375
  try:
376
  doc = json.loads(str(text or "") or "{}")
377
  except (ValueError, TypeError) as exc:
@@ -387,12 +603,151 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
387
  body = [body]
388
  if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
389
  raise ValueError(f"Amazon JSON report has an invalid {root} row list")
390
- rows = []
 
 
 
 
 
391
  attributed = document_id is not None or report_type is not None
392
  for row_number, item in enumerate(body, 1):
393
  row = _flatten_row(_without_buyer_data(item))
394
- rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
395
- return sorted({k for r in rows for k in r}), rows
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
396
 
397
 
398
  def _flatten_row(obj, prefix=""):
@@ -601,6 +956,8 @@ def _awd_rows(inventory, operation_id=None):
601
 
602
  def _tag_fba_inventory(rows):
603
  """Mark report rows so one combined inventory table never conflates FBA with AWD."""
 
 
604
  for row in rows or ():
605
  row["inventory_source"] = "FBA"
606
  return rows
@@ -690,36 +1047,18 @@ def _coltype(name, rows=None):
690
  return "VARCHAR"
691
  if not rows:
692
  return "VARCHAR"
 
 
693
  seen, kinds = 0, set()
694
  for r in rows:
695
  v = r.get(name)
696
  if v is None or v == "":
697
  continue
698
- if isinstance(v, bool):
699
- kinds.add("str")
700
- elif isinstance(v, int):
701
- kinds.add("int")
702
- elif isinstance(v, float):
703
- kinds.add("float")
704
- elif isinstance(v, (list, dict)):
705
- kinds.add("str")
706
- else:
707
- s = str(v).strip()
708
- try:
709
- int(s)
710
- kinds.add("int")
711
- except ValueError:
712
- try:
713
- float(s)
714
- kinds.add("float")
715
- except ValueError:
716
- kinds.add("str")
717
  seen += 1
718
  if seen >= 500:
719
  break
720
- if not kinds or "str" in kinds:
721
- return "VARCHAR"
722
- return "DOUBLE" if "float" in kinds else "BIGINT"
723
 
724
 
725
  def _cell(value, kind):
@@ -798,11 +1137,27 @@ def _replace(con, table, cols, types, rows):
798
  or type(r.get("_report_row_number")) is not int
799
  or r["_report_row_number"] < 1 for r in rows):
800
  raise ValueError("refusing Amazon rows without complete source attribution")
801
- ph = ", ".join("?" for _ in cols)
802
- con.executemany(
803
- f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES ({ph})',
804
- [[_cell(r.get(c), types.get(c, "VARCHAR")) for c in cols] for r in rows])
805
- return len(rows)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
806
 
807
 
808
  def replace_keys_for(table):
@@ -822,28 +1177,26 @@ def replace_window(con, table, cols, types, rows, keys):
822
  return _replace(con, table, cols, types, rows), None
823
  quote = lambda name: '"' + str(name).replace('"', '""') + '"'
824
  target = quote(table)
825
- temporary = quote("amazon_window_keys_" + uuid.uuid4().hex)
826
- columns = ", ".join(quote(c) for c in keys)
827
- # Use the mirror's types and the insert's cell conversion so NULL and numeric keys agree.
828
- incoming = list(dict.fromkeys(tuple(_cell(row.get(c), types.get(c, "VARCHAR"))
829
- for c in keys) for row in rows))
830
  con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
831
  try:
832
- con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})",
833
- incoming)
834
  predicate = " AND ".join(
835
  f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
836
  con.execute("BEGIN TRANSACTION")
837
  try:
 
838
  con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
839
- count = _replace(con, table, cols, types, rows)
840
  con.execute("COMMIT")
841
  except Exception:
842
  con.execute("ROLLBACK")
843
  raise
844
  return count, None
845
  finally:
846
- con.execute(f"DROP TABLE {temporary}")
847
 
848
 
849
  def _windows(start, end, size=MAX_WINDOW_DAYS):
@@ -1189,6 +1542,7 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
1189
  return 0
1190
 
1191
  # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
 
1192
  doc = json.loads(text)
1193
  raw = doc.get(spec["json_root"])
1194
  if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
@@ -2747,6 +3101,16 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
2747
 
2748
  def document_row_count(text, roots=()):
2749
  """Count source records before flattening, privacy filtering or mirror transformations."""
 
 
 
 
 
 
 
 
 
 
2750
  if roots:
2751
  doc = json.loads(text)
2752
  return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
@@ -2791,23 +3155,31 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
2791
  stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
2792
  if not stamp and (table == "listings" or not rows):
2793
  raise ValueError("Amazon snapshot returned no data date or report creation date")
2794
- counts, identities = {}, set()
2795
- for row in rows:
2796
  period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
2797
  row["_snapshot_date"] = period
2798
  if table == "inventory":
2799
  row["date"] = period
2800
  row["inventory_source"] = "FBA"
 
 
 
 
 
 
 
2801
  identity = _snapshot_identity(table, row)
2802
  if identity in identities:
2803
  raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
2804
  identities.add(identity)
2805
  counts[period] = counts.get(period, 0) + 1
2806
- if len(rows) != source_count:
 
2807
  raise ValueError(f"Amazon {table} snapshot row count differs from its document")
2808
  if not counts:
2809
  counts[_utc(stamp).date().isoformat()] = 0
2810
- columns = sorted(set(columns) | {"_snapshot_date"} | {k for row in rows for k in row})
 
2811
  if table == "inventory":
2812
  columns = sorted(set(columns) | {"inventory_source"})
2813
  mirror = table_name(table)
@@ -2954,8 +3326,9 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2954
  if options:
2955
  kwargs["options"] = options
2956
  text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
 
2957
  parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
2958
- if spec.get("json_root") and meta.get("cancelled") and not str(text or "").strip():
2959
  # ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
2960
  # returns an empty document for it, S66-T42), but the JSON parser read "" as a document
2961
  # with no row root and REFUSED it. So the two B2B opportunity reports never earned a
@@ -2965,14 +3338,14 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2965
  elif spec.get("json_root"):
2966
  columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
2967
  if spec.get("json_extra_roots"):
2968
- for row in rows:
2969
- row["_json_root"] = spec["json_root"]
2970
  for root in spec["json_extra_roots"]:
2971
  extra_columns, extra = parse_json_report(text, root, **parser_args)
2972
- for row in extra:
2973
- row["_json_root"] = root
2974
  columns = merge_columns(columns, extra_columns)
2975
- rows.extend(extra)
 
2976
  else:
2977
  columns, rows = parse_flat_file(text, **parser_args)
2978
  if table in LEGACY_SNAPSHOT_TABLES:
@@ -2986,7 +3359,7 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2986
  period, column = _utc(w0).strftime("%Y-%m"), "month"
2987
  else:
2988
  period, column = _iso(w0), "_period_start"
2989
- for row in rows:
2990
  row[column] = period
2991
  if w1 is not None:
2992
  row["_period_end"] = _iso(w1)
@@ -2994,19 +3367,31 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2994
  row["fulfillment_center"] = (row.get("fulfillment_center") or
2995
  row.get("fulfillment_center_id") or
2996
  row.get("fulfilment_center") or "all")
2997
- columns = sorted(set(columns) | {k for row in rows for k in row})
 
 
 
 
2998
  roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
2999
  # HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
3000
- empty = bool(roots) and ((meta.get("cancelled") and not str(text or "").strip()) or
3001
  (table in ("b2b_recommended", "b2b_not_yet_on_amazon")
3002
- and isinstance(json.loads(text or "{}"), dict)
3003
- and spec["json_root"] not in json.loads(text or "{}")))
3004
  marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
3005
  "column": column, "count": 0 if empty else document_row_count(text, roots)}
3006
  return _replace_period(con, table, rows, columns, column, period, marker=marker,
3007
  snapshot=bool(spec.get("snapshot")))
3008
 
3009
 
 
 
 
 
 
 
 
 
3010
  def _order_metric_totals(records, *, flattened=False):
3011
  totals = {"orders": 0, "units": 0, "sales": {}}
3012
  for row in records:
@@ -3542,11 +3927,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3542
  settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
3543
  for item in discovered[issued_index:])
3544
  break
 
3545
  operation = "getReportDocument"
3546
  documents += 1
3547
  _sync_event("amazon.sync", tenant_key, table,
3548
  {"state": "access", "operation": operation}, rt, report, log)
3549
  text = cx.settlement_document(fields, document_id)
 
3550
  cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
3551
  count = 0
3552
  if cols and rows:
@@ -3555,6 +3942,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3555
  # A newly issued document still replaces its own settlement's lines.
3556
  count = write_window(table, cols, types, rows)
3557
  written += count
 
3558
  # Record completion after the write, including header-only and blank files.
3559
  _record_state(con, entity_key("settlement_doc_" + document_id), "done",
3560
  count, issued.get("dataEndTime"))
@@ -3618,11 +4006,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3618
  _record_state(con, entity_key("limit_" + table), "done", detail="{}")
3619
  for w0, w1 in margin_windows:
3620
  # One detail window is one budget task, including its reconciliation query.
 
3621
  documents += 1
3622
  _sync_event("amazon.sync", tenant_key, table,
3623
  {"state": "access", "operation": operation}, rt, report, log)
3624
- written += _sync_margin_window(con, table, fields, w0, w1,
3625
- max_wait=max_wait, log=log)
 
 
3626
  landed_window = True
3627
  if table in LEGACY_SNAPSHOT_TABLES:
3628
  # The dated replacement already landed; AWD must not clear that
@@ -3642,6 +4033,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3642
  # vendor attempt as a sync ACCESS phase, distinct from the completion event.
3643
  # Lane F/combine own the vocabulary decision; no credential or row enters C2.
3644
  documents += 1
 
3645
  with _report_resume(
3646
  con, table, rtype, start=_iso(w0),
3647
  end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
@@ -3652,6 +4044,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3652
  _sync_event("amazon.sync", tenant_key, table,
3653
  {"state": "access", "operation": "report_document"}, rt, report, log)
3654
  text, meta = cx.report_document(fields, rtype, **resume_kwargs)
 
3655
  landed_window = True
3656
  if root:
3657
  parsed_cols, rows = parse_json_report(
@@ -3727,6 +4120,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3727
  if cx.REPORT_TYPES[table].get("monthly"):
3728
  _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
3729
  "count": len(rows)})
 
3730
  if first_load:
3731
  for reached in (table, sibling):
3732
  if reached:
@@ -3961,15 +4355,20 @@ def _dedupe_orders(rows):
3961
  lines by construction, so this drops line detail rather than losing an order. `order_items`
3962
  keeps every row, which is why both tables read the same document instead of costing two.
3963
  """
3964
- seen, out = set(), []
 
 
 
 
 
 
3965
  for r in rows:
3966
  oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
3967
  if oid and oid in seen:
3968
  continue
3969
  if oid:
3970
  seen.add(oid)
3971
- out.append(r)
3972
- return out
3973
 
3974
 
3975
  def validate(tenant_key="royal-imports", rt=None, log=print):
@@ -6303,10 +6702,113 @@ def selftest(log=print):
6303
  ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
6304
  ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
6305
  ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
6306
- log(f"\n{'GREEN' if not fails else 'RED'}: {33 - len(fails)}/33")
 
 
 
 
6307
  return 0 if not fails else 1
6308
 
6309
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6310
  def main(argv=None):
6311
  ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
6312
  ap.add_argument("--sync", action="store_true")
 
327
  return value
328
 
329
 
330
+ #: Rows per INSERT statement. A batch, not a cap: every row of a document is written (standing rule 1).
331
+ INSERT_BATCH_ROWS = 500
332
+ #: Bound parameters per INSERT statement, so a wide table takes fewer rows per statement.
333
+ INSERT_BATCH_VALUES = 30_000
334
+ #: Characters read per step of a spooled document.
335
+ DOCUMENT_READ_CHARS = 1 << 20
336
+
337
+
338
+ def _spooled(document):
339
+ """True for a `ReportDocument` the client spooled to disk; False for in-memory text."""
340
+ return hasattr(document, "open_text")
341
+
342
+
343
+ def _document_text(document):
344
+ """The whole text of a document, for a consumer whose documents are small."""
345
+ return document.read_text() if _spooled(document) else document
346
+
347
+
348
+ def _blank(document):
349
+ """`not str(text or "").strip()`, read in pieces when the document is spooled."""
350
+ if not _spooled(document):
351
+ return not str(document or "").strip()
352
+ with document.open_text() as handle:
353
+ while True:
354
+ piece = handle.read(DOCUMENT_READ_CHARS)
355
+ if not piece:
356
+ return True
357
+ if piece.strip():
358
+ return False
359
+
360
+
361
+ def _value_kind(v):
362
+ """One non-empty value's sniffed kind: `int`, `float` or `str`."""
363
+ if isinstance(v, bool):
364
+ return "str"
365
+ if isinstance(v, int):
366
+ return "int"
367
+ if isinstance(v, float):
368
+ return "float"
369
+ if isinstance(v, (list, dict)):
370
+ return "str"
371
+ s = str(v).strip()
372
+ try:
373
+ int(s)
374
+ return "int"
375
+ except ValueError:
376
+ try:
377
+ float(s)
378
+ return "float"
379
+ except ValueError:
380
+ return "str"
381
+
382
+
383
+ def _kinds_type(kinds):
384
+ if not kinds or "str" in kinds:
385
+ return "VARCHAR"
386
+ return "DOUBLE" if "float" in kinds else "BIGINT"
387
+
388
+
389
+ class _DocumentRows:
390
+ """Rows parsed again from a spooled report document on every pass, so no pass holds them all.
391
+
392
+ ⛔ D-7081: one list of every row of the Brand Analytics month took the Space to 16 GB. A caller
393
+ iterates, takes `len()` or tests truth; `map` and `pipe` replay a row change on every pass.
394
+ """
395
+
396
+ def __init__(self, produce, steps=()):
397
+ self._produce, self._steps = produce, tuple(steps)
398
+ self._count, self._sniff = None, None
399
+
400
+ def __iter__(self):
401
+ rows = self._produce()
402
+ for step in self._steps:
403
+ rows = step(rows)
404
+ return iter(rows)
405
+
406
+ def pipe(self, step):
407
+ return _DocumentRows(self._produce, self._steps + (step,))
408
+
409
+ def map(self, change):
410
+ return self.pipe(lambda rows: (change(row) for row in rows))
411
+
412
+ @staticmethod
413
+ def chain(*parts):
414
+ return _DocumentRows(lambda: (row for part in parts for row in part))
415
+
416
+ def _profile(self):
417
+ """One pass: the row count, every key, and each key's first 500 non-empty kinds."""
418
+ count, sniff = 0, {}
419
+ for row in self:
420
+ count += 1
421
+ for key, value in row.items():
422
+ entry = sniff.get(key)
423
+ if entry is None:
424
+ entry = sniff[key] = [0, set()]
425
+ if entry[0] >= 500 or value is None or value == "":
426
+ continue
427
+ entry[1].add(_value_kind(value))
428
+ entry[0] += 1
429
+ self._count, self._sniff = count, sniff
430
+
431
+ def keys(self):
432
+ if self._sniff is None:
433
+ self._profile()
434
+ return set(self._sniff)
435
+
436
+ def sniffed(self, name):
437
+ """`_coltype(name, rows)` for these rows, from the one profile pass."""
438
+ if self._sniff is None:
439
+ self._profile()
440
+ return _kinds_type(self._sniff.get(name, (0, ()))[1])
441
+
442
+ def __len__(self):
443
+ if self._count is None:
444
+ self._profile()
445
+ return self._count
446
+
447
+ def __bool__(self):
448
+ if self._count is not None:
449
+ return self._count > 0
450
+ rows = iter(self)
451
+ try:
452
+ return next(rows, None) is not None
453
+ finally:
454
+ getattr(rows, "close", lambda: None)()
455
+
456
+
457
+ def _rss_gb():
458
+ """This process's resident memory in GB, or None where /proc is absent."""
459
+ try:
460
+ with open("/proc/self/status", encoding="ascii") as handle:
461
+ for line in handle:
462
+ if line.startswith("VmRSS:"):
463
+ return int(line.split()[1]) / 1048576
464
+ except Exception: # noqa: BLE001
465
+ return None
466
+ return None
467
+
468
+
469
+ class _DocumentProbe:
470
+ """One server-log line per report document: table, bytes, rows, seconds and the memory peak.
471
+
472
+ ⚠ `print`, not `log`: the sync desk passes a silent log, so its first load logged nothing until a
473
+ pass ended and the document in flight at an out-of-memory kill was unknown (D-7081).
474
+ """
475
+
476
+ _lock = threading.Lock()
477
+ _active = None
478
+
479
+ def __init__(self, table, report_type):
480
+ self.table, self.report_type = table, report_type
481
+ self.started, self.size, self.peak = time.monotonic(), None, _rss_gb()
482
+ self._halt = threading.Event()
483
+ with _DocumentProbe._lock:
484
+ previous, _DocumentProbe._active = _DocumentProbe._active, self
485
+ if previous is not None:
486
+ previous._halt.set()
487
+ threading.Thread(target=self._sample, name="amazon-document-probe", daemon=True).start()
488
+
489
+ def _sample(self):
490
+ deadline = time.monotonic() + 6 * 3600
491
+ while not self._halt.wait(0.5) and time.monotonic() < deadline:
492
+ rss = _rss_gb()
493
+ if rss is not None and (self.peak is None or rss > self.peak):
494
+ self.peak = rss
495
+
496
+ @classmethod
497
+ def downloaded(cls, document):
498
+ probe = cls._active
499
+ if probe is not None and not probe._halt.is_set():
500
+ probe.size = (document.size if _spooled(document)
501
+ else len(str(document or "").encode("utf-8", "replace")))
502
+ probe._say("downloaded", None)
503
+
504
+ def finish(self, rows):
505
+ self._halt.set()
506
+ self._say("written", rows)
507
+
508
+ def _say(self, phase, rows):
509
+ rss = _rss_gb()
510
+ peak = max(x for x in (self.peak, rss, 0.0) if x is not None)
511
+ size = "unknown" if self.size is None else f"{self.size:,}"
512
+ print(f"[aios-api] amazon document {phase}: table {self.table}, {self.report_type}, "
513
+ f"{size} bytes, {'-' if rows is None else f'{rows:,}'} rows, "
514
+ f"{time.monotonic() - self.started:.0f}s, rss {rss or 0:.2f} GB, peak {peak:.2f} GB",
515
+ flush=True)
516
+
517
+
518
+ def _flat_rows(lines, document_id, report_type):
519
+ """`(columns, row generator)` over one flat file's lines, or `(None, ())` with no header line."""
520
+ reader = csv.reader(lines, delimiter="\t", quoting=csv.QUOTE_NONE)
521
+ try:
522
+ header = next(reader)
523
+ except StopIteration:
524
+ return None, ()
525
+ cols = _dedupe([_snake(h) for h in header])
526
+ width = len(cols)
527
+ # The read-only connectors_amazon report probe measures headers without capture metadata.
528
+ # Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
529
+ # BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
530
+ attributed = document_id is not None or report_type is not None
531
+
532
+ def rows():
533
+ for row_number, raw in enumerate(reader, 1):
534
+ if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
535
+ continue
536
+ vals = list(raw[:width]) + [None] * max(0, width - len(raw))
537
+ row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
538
+ if not buyer_column(c)}
539
+ yield _origin(row, document_id, row_number, report_type) if attributed else row
540
+ return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows()
541
+
542
+
543
+ def _parse_flat_document(document, *, document_id=None, report_type=None):
544
+ """`parse_flat_file` over a spooled document: the header now, the rows again on every pass."""
545
+ if _blank(document):
546
+ return [], []
547
+ with document.open_text() as handle:
548
+ cols, _rows = _flat_rows(handle, document_id, report_type)
549
+ if cols is None:
550
+ return [], []
551
+
552
+ def produce():
553
+ with document.open_text() as handle:
554
+ yield from _flat_rows(handle, document_id, report_type)[1]
555
+ return cols, _DocumentRows(produce)
556
+
557
+
558
  def parse_flat_file(text, *, document_id=None, report_type=None):
559
  """One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
560
 
 
567
  swallow the rest of the row and silently merge it with the next one, so a catalogue with one
568
  inch-mark in a title would lose rows with nothing raising.
569
  """
570
+ if _spooled(text):
571
+ return _parse_flat_document(text, document_id=document_id, report_type=report_type)
572
  if not str(text or "").strip():
573
  return [], []
574
  buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
575
+ cols, rows = _flat_rows(buf, document_id, report_type)
576
+ if cols is None:
 
 
577
  return [], []
578
+ return cols, list(rows)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
579
 
580
 
581
  def parse_json_report(text, root, *, document_id=None, report_type=None):
 
586
  with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
587
  artifact is the authority; a field the payload omits is NULL.
588
  """
589
+ if _spooled(text):
590
+ return _parse_json_document(text, root, document_id=document_id, report_type=report_type)
591
  try:
592
  doc = json.loads(str(text or "") or "{}")
593
  except (ValueError, TypeError) as exc:
 
603
  body = [body]
604
  if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
605
  raise ValueError(f"Amazon JSON report has an invalid {root} row list")
606
+ rows = list(_json_rows(body, document_id, report_type))
607
+ return sorted({k for r in rows for k in r}), rows
608
+
609
+
610
+ def _json_rows(body, document_id, report_type):
611
+ """Flatten and attribute each row object, numbering from one."""
612
  attributed = document_id is not None or report_type is not None
613
  for row_number, item in enumerate(body, 1):
614
  row = _flatten_row(_without_buyer_data(item))
615
+ yield _origin(row, document_id, row_number, report_type) if attributed else row
616
+
617
+
618
+ class _JsonFallback(Exception):
619
+ """A shape only the whole-text parser answers exactly: malformed, not an object, a repeated root."""
620
+
621
+
622
+ _JSON_WS = re.compile(r"[ \t\n\r]*")
623
+ _JSON_DECODER = json.JSONDecoder()
624
+ #: A single value may span this many characters before the stream hands it to `json.loads`.
625
+ _JSON_VALUE_CHARS = 256 << 20
626
+
627
+
628
+ class _JsonStream:
629
+ """JSON read from a text stream one value at a time, with `json.loads` value semantics."""
630
+
631
+ def __init__(self, handle):
632
+ self.handle, self.buf, self.pos = handle, "", 0
633
+
634
+ def _fill(self):
635
+ piece = self.handle.read(DOCUMENT_READ_CHARS)
636
+ if not piece:
637
+ return False
638
+ self.buf = self.buf[self.pos:] + piece
639
+ self.pos = 0
640
+ return True
641
+
642
+ def peek(self):
643
+ while True:
644
+ self.pos = _JSON_WS.match(self.buf, self.pos).end()
645
+ if self.pos < len(self.buf) or not self._fill():
646
+ return self.buf[self.pos:self.pos + 1]
647
+
648
+ def take(self, char):
649
+ if self.peek() != char:
650
+ raise _JsonFallback()
651
+ self.pos += 1
652
+
653
+ def value(self):
654
+ self.peek()
655
+ while True:
656
+ try:
657
+ obj, end = _JSON_DECODER.raw_decode(self.buf, self.pos)
658
+ except ValueError:
659
+ if len(self.buf) - self.pos < _JSON_VALUE_CHARS and self._fill():
660
+ continue
661
+ raise _JsonFallback() from None
662
+ # A number ending at the buffer's edge may continue in the next piece.
663
+ if end == len(self.buf) and self._fill():
664
+ continue
665
+ self.pos = end
666
+ return obj
667
+
668
+ def skip(self):
669
+ if self.peek() != "[":
670
+ self.value()
671
+ return
672
+ self.pos += 1
673
+ if self.peek() == "]":
674
+ self.pos += 1
675
+ return
676
+ while True:
677
+ self.skip()
678
+ char = self.peek()
679
+ self.pos += 1
680
+ if char == "]":
681
+ return
682
+ if char != ",":
683
+ raise _JsonFallback()
684
+
685
+ def items(self):
686
+ if self.peek() == "{":
687
+ yield self.value()
688
+ return
689
+ self.take("[")
690
+ if self.peek() == "]":
691
+ self.pos += 1
692
+ return
693
+ while True:
694
+ item = self.value()
695
+ if not isinstance(item, dict):
696
+ raise _JsonFallback()
697
+ yield item
698
+ char = self.peek()
699
+ self.pos += 1
700
+ if char == "]":
701
+ return
702
+ if char != ",":
703
+ raise _JsonFallback()
704
+
705
+
706
+ def _json_root_items(document, root):
707
+ """Each row object under the top-level `root` key of a spooled document, decoded one at a time.
708
+
709
+ Raises `_JsonFallback` for any shape `json.loads` must judge: malformed text, a document that is
710
+ not an object, a missing or repeated root, or a root that is not a list of objects.
711
+ """
712
+ with document.open_text() as handle:
713
+ stream, found = _JsonStream(handle), False
714
+ stream.take("{")
715
+ if stream.peek() == "}":
716
+ stream.pos += 1
717
+ else:
718
+ while True:
719
+ key = stream.value()
720
+ if not isinstance(key, str):
721
+ raise _JsonFallback()
722
+ stream.take(":")
723
+ if key == root:
724
+ if found:
725
+ raise _JsonFallback()
726
+ found = True
727
+ yield from stream.items()
728
+ else:
729
+ stream.skip()
730
+ char = stream.peek()
731
+ stream.pos += 1
732
+ if char == "}":
733
+ break
734
+ if char != ",":
735
+ raise _JsonFallback()
736
+ if stream.peek() or not found:
737
+ raise _JsonFallback()
738
+
739
+
740
+ def _parse_json_document(document, root, *, document_id=None, report_type=None):
741
+ """`parse_json_report` over a spooled document; any shape it cannot stream reads the whole text."""
742
+ rows = _DocumentRows(lambda: _json_rows(_json_root_items(document, root), document_id, report_type))
743
+ try:
744
+ keys = rows.keys()
745
+ except _JsonFallback:
746
+ return parse_json_report(document.read_text(), root, document_id=document_id,
747
+ report_type=report_type)
748
+ if not len(rows):
749
+ return [], []
750
+ return sorted(keys), rows
751
 
752
 
753
  def _flatten_row(obj, prefix=""):
 
956
 
957
  def _tag_fba_inventory(rows):
958
  """Mark report rows so one combined inventory table never conflates FBA with AWD."""
959
+ if isinstance(rows, _DocumentRows):
960
+ return _stamp_rows(rows, {"inventory_source": "FBA"})
961
  for row in rows or ():
962
  row["inventory_source"] = "FBA"
963
  return rows
 
1047
  return "VARCHAR"
1048
  if not rows:
1049
  return "VARCHAR"
1050
+ if isinstance(rows, _DocumentRows):
1051
+ return rows.sniffed(name)
1052
  seen, kinds = 0, set()
1053
  for r in rows:
1054
  v = r.get(name)
1055
  if v is None or v == "":
1056
  continue
1057
+ kinds.add(_value_kind(v))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1058
  seen += 1
1059
  if seen >= 500:
1060
  break
1061
+ return _kinds_type(kinds)
 
 
1062
 
1063
 
1064
  def _cell(value, kind):
 
1137
  or type(r.get("_report_row_number")) is not int
1138
  or r["_report_row_number"] < 1 for r in rows):
1139
  raise ValueError("refusing Amazon rows without complete source attribution")
1140
+ ph = "(" + ", ".join("?" for _ in cols) + ")"
1141
+ insert = f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES '
1142
+ per = max(1, min(INSERT_BATCH_ROWS, INSERT_BATCH_VALUES // len(cols)))
1143
+ full = insert + ", ".join([ph] * per)
1144
+
1145
+ def write(batch):
1146
+ con.execute(full if len(batch) == per else insert + ", ".join([ph] * len(batch)),
1147
+ [value for row in batch for value in row])
1148
+
1149
+ # Multi-row statements in batches: no copy of every row exists at once, the caller's transaction
1150
+ # keeps the window all-or-nothing, and values bind and cast exactly as `executemany` did.
1151
+ count, batch = 0, []
1152
+ for r in rows:
1153
+ batch.append([_cell(r.get(c), types.get(c, "VARCHAR")) for c in cols])
1154
+ if len(batch) == per:
1155
+ write(batch)
1156
+ count, batch = count + len(batch), []
1157
+ if batch:
1158
+ write(batch)
1159
+ count += len(batch)
1160
+ return count
1161
 
1162
 
1163
  def replace_keys_for(table):
 
1177
  return _replace(con, table, cols, types, rows), None
1178
  quote = lambda name: '"' + str(name).replace('"', '""') + '"'
1179
  target = quote(table)
1180
+ # The window's rows are staged in batches inside the transaction, typed as the mirror and
1181
+ # converted by the insert's own cells, so NULL and numeric keys agree and no key list exists.
1182
+ temporary = quote("amazon_window_rows_" + uuid.uuid4().hex)
1183
+ columns = ", ".join(quote(c) for c in cols)
 
1184
  con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
1185
  try:
 
 
1186
  predicate = " AND ".join(
1187
  f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
1188
  con.execute("BEGIN TRANSACTION")
1189
  try:
1190
+ count = _replace(con, temporary, cols, types, rows)
1191
  con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
1192
+ con.execute(f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {temporary}")
1193
  con.execute("COMMIT")
1194
  except Exception:
1195
  con.execute("ROLLBACK")
1196
  raise
1197
  return count, None
1198
  finally:
1199
+ con.execute(f"DROP TABLE IF EXISTS {temporary}")
1200
 
1201
 
1202
  def _windows(start, end, size=MAX_WINDOW_DAYS):
 
1542
  return 0
1543
 
1544
  # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
1545
+ text = _document_text(text)
1546
  doc = json.loads(text)
1547
  raw = doc.get(spec["json_root"])
1548
  if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
 
3101
 
3102
  def document_row_count(text, roots=()):
3103
  """Count source records before flattening, privacy filtering or mirror transformations."""
3104
+ if _spooled(text):
3105
+ try:
3106
+ if roots:
3107
+ return sum(sum(1 for _item in _json_root_items(text, root)) for root in roots)
3108
+ with text.open_text() as handle:
3109
+ reader = csv.reader(handle, delimiter="\t", quoting=csv.QUOTE_NONE)
3110
+ next(reader, None)
3111
+ return sum(bool(raw) and not (len(raw) == 1 and not str(raw[0]).strip()) for raw in reader)
3112
+ except _JsonFallback:
3113
+ return document_row_count(text.read_text(), roots)
3114
  if roots:
3115
  doc = json.loads(text)
3116
  return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
 
3155
  stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
3156
  if not stamp and (table == "listings" or not rows):
3157
  raise ValueError("Amazon snapshot returned no data date or report creation date")
3158
+ def stamped(row):
 
3159
  period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
3160
  row["_snapshot_date"] = period
3161
  if table == "inventory":
3162
  row["date"] = period
3163
  row["inventory_source"] = "FBA"
3164
+ return row
3165
+
3166
+ lazy = isinstance(rows, _DocumentRows)
3167
+ rows = rows.map(stamped) if lazy else rows
3168
+ counts, identities, total = {}, set(), 0
3169
+ for row in rows:
3170
+ period = (row if lazy else stamped(row))["_snapshot_date"]
3171
  identity = _snapshot_identity(table, row)
3172
  if identity in identities:
3173
  raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
3174
  identities.add(identity)
3175
  counts[period] = counts.get(period, 0) + 1
3176
+ total += 1
3177
+ if total != source_count:
3178
  raise ValueError(f"Amazon {table} snapshot row count differs from its document")
3179
  if not counts:
3180
  counts[_utc(stamp).date().isoformat()] = 0
3181
+ columns = sorted(set(columns) | {"_snapshot_date"}
3182
+ | (rows.keys() if lazy else {k for row in rows for k in row}))
3183
  if table == "inventory":
3184
  columns = sorted(set(columns) | {"inventory_source"})
3185
  mirror = table_name(table)
 
3326
  if options:
3327
  kwargs["options"] = options
3328
  text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
3329
+ _DocumentProbe.downloaded(text)
3330
  parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
3331
+ if spec.get("json_root") and meta.get("cancelled") and _blank(text):
3332
  # ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
3333
  # returns an empty document for it, S66-T42), but the JSON parser read "" as a document
3334
  # with no row root and REFUSED it. So the two B2B opportunity reports never earned a
 
3338
  elif spec.get("json_root"):
3339
  columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
3340
  if spec.get("json_extra_roots"):
3341
+ rows = _stamp_rows(rows, {"_json_root": spec["json_root"]})
3342
+ parts = [rows]
3343
  for root in spec["json_extra_roots"]:
3344
  extra_columns, extra = parse_json_report(text, root, **parser_args)
3345
+ parts.append(_stamp_rows(extra, {"_json_root": root}))
 
3346
  columns = merge_columns(columns, extra_columns)
3347
+ rows = (_DocumentRows.chain(*parts) if any(isinstance(part, _DocumentRows) for part in parts)
3348
+ else [row for part in parts for row in part])
3349
  else:
3350
  columns, rows = parse_flat_file(text, **parser_args)
3351
  if table in LEGACY_SNAPSHOT_TABLES:
 
3359
  period, column = _utc(w0).strftime("%Y-%m"), "month"
3360
  else:
3361
  period, column = _iso(w0), "_period_start"
3362
+ def stamped(row):
3363
  row[column] = period
3364
  if w1 is not None:
3365
  row["_period_end"] = _iso(w1)
 
3367
  row["fulfillment_center"] = (row.get("fulfillment_center") or
3368
  row.get("fulfillment_center_id") or
3369
  row.get("fulfilment_center") or "all")
3370
+ return row
3371
+
3372
+ rows = rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
3373
+ columns = sorted(set(columns) | (rows.keys() if isinstance(rows, _DocumentRows)
3374
+ else {k for row in rows for k in row}))
3375
  roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
3376
  # HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
3377
+ empty = bool(roots) and ((meta.get("cancelled") and _blank(text)) or
3378
  (table in ("b2b_recommended", "b2b_not_yet_on_amazon")
3379
+ and isinstance(json.loads(_document_text(text) or "{}"), dict)
3380
+ and spec["json_root"] not in json.loads(_document_text(text) or "{}")))
3381
  marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
3382
  "column": column, "count": 0 if empty else document_row_count(text, roots)}
3383
  return _replace_period(con, table, rows, columns, column, period, marker=marker,
3384
  snapshot=bool(spec.get("snapshot")))
3385
 
3386
 
3387
+ def _stamp_rows(rows, values):
3388
+ """Set `values` on every row, on each pass when the rows are lazy."""
3389
+ def stamped(row):
3390
+ row.update(values)
3391
+ return row
3392
+ return rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
3393
+
3394
+
3395
  def _order_metric_totals(records, *, flattened=False):
3396
  totals = {"orders": 0, "units": 0, "sales": {}}
3397
  for row in records:
 
3927
  settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
3928
  for item in discovered[issued_index:])
3929
  break
3930
+ probe = _DocumentProbe(table, rtype)
3931
  operation = "getReportDocument"
3932
  documents += 1
3933
  _sync_event("amazon.sync", tenant_key, table,
3934
  {"state": "access", "operation": operation}, rt, report, log)
3935
  text = cx.settlement_document(fields, document_id)
3936
+ _DocumentProbe.downloaded(text)
3937
  cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
3938
  count = 0
3939
  if cols and rows:
 
3942
  # A newly issued document still replaces its own settlement's lines.
3943
  count = write_window(table, cols, types, rows)
3944
  written += count
3945
+ probe.finish(count)
3946
  # Record completion after the write, including header-only and blank files.
3947
  _record_state(con, entity_key("settlement_doc_" + document_id), "done",
3948
  count, issued.get("dataEndTime"))
 
4006
  _record_state(con, entity_key("limit_" + table), "done", detail="{}")
4007
  for w0, w1 in margin_windows:
4008
  # One detail window is one budget task, including its reconciliation query.
4009
+ probe = _DocumentProbe(table, spec.get("report") or operation)
4010
  documents += 1
4011
  _sync_event("amazon.sync", tenant_key, table,
4012
  {"state": "access", "operation": operation}, rt, report, log)
4013
+ count = _sync_margin_window(con, table, fields, w0, w1,
4014
+ max_wait=max_wait, log=log)
4015
+ probe.finish(count)
4016
+ written += count
4017
  landed_window = True
4018
  if table in LEGACY_SNAPSHOT_TABLES:
4019
  # The dated replacement already landed; AWD must not clear that
 
4033
  # vendor attempt as a sync ACCESS phase, distinct from the completion event.
4034
  # Lane F/combine own the vocabulary decision; no credential or row enters C2.
4035
  documents += 1
4036
+ probe = _DocumentProbe(table, rtype)
4037
  with _report_resume(
4038
  con, table, rtype, start=_iso(w0),
4039
  end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
 
4044
  _sync_event("amazon.sync", tenant_key, table,
4045
  {"state": "access", "operation": "report_document"}, rt, report, log)
4046
  text, meta = cx.report_document(fields, rtype, **resume_kwargs)
4047
+ _DocumentProbe.downloaded(text)
4048
  landed_window = True
4049
  if root:
4050
  parsed_cols, rows = parse_json_report(
 
4120
  if cx.REPORT_TYPES[table].get("monthly"):
4121
  _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
4122
  "count": len(rows)})
4123
+ probe.finish(len(rows) if cols else 0)
4124
  if first_load:
4125
  for reached in (table, sibling):
4126
  if reached:
 
4355
  lines by construction, so this drops line detail rather than losing an order. `order_items`
4356
  keeps every row, which is why both tables read the same document instead of costing two.
4357
  """
4358
+ if isinstance(rows, _DocumentRows):
4359
+ return rows.pipe(_first_row_per_order)
4360
+ return list(_first_row_per_order(rows))
4361
+
4362
+
4363
+ def _first_row_per_order(rows):
4364
+ seen = set()
4365
  for r in rows:
4366
  oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
4367
  if oid and oid in seen:
4368
  continue
4369
  if oid:
4370
  seen.add(oid)
4371
+ yield r
 
4372
 
4373
 
4374
  def validate(tenant_key="royal-imports", rt=None, log=print):
 
6702
  ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
6703
  ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
6704
  ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
6705
+ same, detail = _spooled_parse_selftest()
6706
+ ck("NC-AZ34 a spooled document parses and counts exactly as its text does", same, detail)
6707
+ bounded, detail = _bounded_document_selftest()
6708
+ ck("NC-AZ35 a large synthetic report parses and writes in bounded memory", bounded, detail)
6709
+ log(f"\n{'GREEN' if not fails else 'RED'}: {35 - len(fails)}/35")
6710
  return 0 if not fails else 1
6711
 
6712
 
6713
+ def _spool_fixture(text, *, gz=False, encoding="utf-8"):
6714
+ """A `ReportDocument` made by the client's own spool path from local bytes, no network."""
6715
+ import gzip
6716
+ raw = text.encode(encoding) if isinstance(text, str) else text
6717
+ handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
6718
+ with os.fdopen(handle, "wb") as out:
6719
+ out.write(gzip.compress(raw) if gz else raw)
6720
+ return _client()._spooled_document(path, "GZIP" if gz else "")
6721
+
6722
+
6723
+ def _spooled_parse_selftest():
6724
+ """D-7081: the streaming readers answer every fixture exactly as the whole-text parsers do."""
6725
+ origin = {"document_id": "selftest-document", "report_type": "selftest-report"}
6726
+
6727
+ def outcome(parse):
6728
+ try:
6729
+ cols, rows = parse()
6730
+ return "ok", list(cols), [dict(row) for row in rows]
6731
+ except Exception as exc: # noqa: BLE001
6732
+ return "error", type(exc).__name__, str(exc)
6733
+
6734
+ def count(text, roots):
6735
+ try:
6736
+ return document_row_count(text, roots)
6737
+ except Exception as exc: # noqa: BLE001
6738
+ return type(exc).__name__
6739
+
6740
+ flat = ["", " \n\t", "a\tb\r\n1\t2\r\n\r\n3\t4", "\ufeffsku\tsku\tbuyer-email\nA\tB\tx\n\"q\t1\n",
6741
+ "x\ty\r1\r2\t3\r\r", "sku\tname\nA\tsmart \u201cquote\u201d\n"]
6742
+ blobs = [(t, {}) for t in flat] + [(t, {"gz": True}) for t in flat] + [(flat[-1], {"encoding": "cp1252"})]
6743
+ misses = []
6744
+ for index, (text, how) in enumerate(blobs):
6745
+ expected = text if how.get("encoding") != "cp1252" else text.encode("cp1252").decode("cp1252")
6746
+ got = outcome(lambda: parse_flat_file(_spool_fixture(text, **how), **origin))
6747
+ if got != outcome(lambda: parse_flat_file(expected, **origin)):
6748
+ misses.append(f"flat {index}")
6749
+ if expected.strip() and count(_spool_fixture(text, **how), ()) != count(expected, ()):
6750
+ misses.append(f"flat count {index}")
6751
+ documents = ['', '[]', '{"rows": []}', '{"rows": {"a": 1}}', '{"rows": [1]}', '{"rows": [{"a": 1},]}',
6752
+ '{"rows": [{"a": 1}]} x', '{"rows": [{"a": 1}], "rows": [{"b": 2}]}', '{"other": 1}',
6753
+ ' {"spec": {"q": [1, [2]]}, "rows": [{"a": 1, "b": null, "n": {"c": "\u2014"}}, '
6754
+ '{"a": 2.5, "l": [1, 2]}], "tail": [[1], []]} ',
6755
+ '{"rows": [{"_report_document_id": "vendor"}]}', '{"rows": [{"buyerEmail": "e", "a": 123456789012345678901}]}',
6756
+ '{"x": NaN, "rows": [{"v": Infinity, "w": -0.0, "s": "\\ud83d"}]}']
6757
+ for index, text in enumerate(documents):
6758
+ if outcome(lambda: parse_json_report(_spool_fixture(text), "rows", **origin)) != outcome(
6759
+ lambda: parse_json_report(text, "rows", **origin)):
6760
+ misses.append(f"json {index}")
6761
+ if count(_spool_fixture(text), ("rows",)) != count(text, ("rows",)):
6762
+ misses.append(f"json count {index}")
6763
+ return not misses, misses
6764
+
6765
+
6766
+ def _bounded_document_selftest(rows=24_000):
6767
+ """D-7081: a report larger than the bound parses, sniffs and writes with a bounded Python peak.
6768
+
6769
+ The whole-text path holds the decoded text and every row at once, so it fails this bound; the
6770
+ spooled path holds one read piece and one insert batch.
6771
+ """
6772
+ import gzip
6773
+ import tracemalloc
6774
+ bound = 10 << 20
6775
+ handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
6776
+ os.close(handle)
6777
+ with gzip.open(path, "wt", encoding="utf-8") as out:
6778
+ out.write('{"reportSpecification": {"reportType": "selftest"}, "dataByDepartmentAndSearchTerm": [')
6779
+ for i in range(rows):
6780
+ out.write(("," if i else "") + json.dumps({
6781
+ "departmentName": "Amazon.com", "searchTerm": f"term {i} \u2014 caf\u00e9",
6782
+ "searchFrequencyRank": i + 1, "clickedAsin": f"B0{i:08d}",
6783
+ "clickedItemName": f"Product title number {i} with several words", "clickShareRank": i % 3 + 1,
6784
+ "clickShare": round(i / 7 % 1, 4), "conversionShare": round(i / 11 % 1, 4)}))
6785
+ out.write("]}")
6786
+ document = _client()._spooled_document(path, "GZIP")
6787
+ text_bytes = 2 * len(document)
6788
+ con = datastore.duckdb.connect(":memory:")
6789
+ table = "amzn_brand_analytics"
6790
+ # A first bound statement imports DuckDB's lazy modules; that one-time cost is not a row's.
6791
+ con.execute("SELECT ?", [1]).fetchall()
6792
+ tracemalloc.start()
6793
+ try:
6794
+ cols, parsed = parse_json_report(document, "dataByDepartmentAndSearchTerm",
6795
+ document_id="selftest-document", report_type="selftest")
6796
+ cols = merge_columns([c for c, _t in amazon_schema.columns_for("brand_analytics")], cols)
6797
+ types = _ensure(con, table, cols, parsed, declared=dict(amazon_schema.columns_for("brand_analytics")))
6798
+ first, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
6799
+ peak = tracemalloc.get_traced_memory()[1]
6800
+ finally:
6801
+ tracemalloc.stop()
6802
+ again, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
6803
+ stored = con.execute(f"SELECT COUNT(*), COUNT(DISTINCT _report_row_number) FROM {table}").fetchone()
6804
+ con.close()
6805
+ document.close()
6806
+ detail = {"rows": len(parsed), "first": first, "again": again, "stored": stored,
6807
+ "text_bytes": text_bytes, "peak": peak, "bound": bound}
6808
+ return (len(parsed) == first == again == rows and stored == (rows, rows)
6809
+ and text_bytes > bound and peak < bound), detail
6810
+
6811
+
6812
  def main(argv=None):
6813
  ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
6814
  ap.add_argument("--sync", action="store_true")
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- cb8f337e46d2f3be3b566f171494c0f6cd6860be3b800b8d006a62b1e062dfe0
 
1
+ e24ce7c6a61cd78dfd92b798e5cec7eff002f567fe56d2190d77cdc7ebe0e7b1