fsanyoto commited on
Commit
812ec21
·
verified ·
1 Parent(s): 95fb175

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,11 +1,521 @@
1
  {
2
- "current": "v88 (592ef76d1f)",
3
  "releases": [
4
  {
5
- "version": "v88",
6
- "sha": "592ef76d1f",
7
- "date": "2026-11-18",
8
- "subject": "release v88"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  }
10
  ]
11
  }
 
1
  {
2
+ "current": "caef761dd5af969155c5c31f930259757738efe4 (caef761dd5af9)",
3
  "releases": [
4
  {
5
+ "version": "v87",
6
+ "sha": "288b6b8793041",
7
+ "date": "2026-11-12",
8
+ "subject": "release(sdlc85): release lessons and log line"
9
+ },
10
+ {
11
+ "version": "v86",
12
+ "sha": "108bd7510dcd2",
13
+ "date": "2026-10-27",
14
+ "subject": "release v86: SDLC 80 (owner override, 3 booked FAILs to SDLC 81)"
15
+ },
16
+ {
17
+ "version": "v85",
18
+ "sha": "b36dd4992153e",
19
+ "date": "2026-10-25",
20
+ "subject": "Merge remote-tracking branch 'origin/master' into hfstage/sdlc79-release"
21
+ },
22
+ {
23
+ "version": "v84",
24
+ "sha": "7bb3a2e09e138",
25
+ "date": "2026-10-24",
26
+ "subject": "release(sdlc78): v84 QA records the staging build change and what v84 excludes"
27
+ },
28
+ {
29
+ "version": "v83",
30
+ "sha": "9b7471968e79d",
31
+ "date": "2026-10-21",
32
+ "subject": "release v83"
33
+ },
34
+ {
35
+ "version": "v82",
36
+ "sha": "9c61b1ee94d0c",
37
+ "date": "2026-10-17",
38
+ "subject": "release v82"
39
+ },
40
+ {
41
+ "version": "v81",
42
+ "sha": "e141ccc85ef0c",
43
+ "date": "2026-10-15",
44
+ "subject": "release v81"
45
+ },
46
+ {
47
+ "version": "v80",
48
+ "sha": "869b8fcc62ac7",
49
+ "date": "2026-10-13",
50
+ "subject": "release v80"
51
+ },
52
+ {
53
+ "version": "v79",
54
+ "sha": "406417b8ded0e",
55
+ "date": "2026-10-13",
56
+ "subject": "release v79"
57
+ },
58
+ {
59
+ "version": "v78",
60
+ "sha": "aad6561d63e6c",
61
+ "date": "2026-10-09",
62
+ "subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs"
63
+ },
64
+ {
65
+ "version": "v77",
66
+ "sha": "e741e5ca604d4",
67
+ "date": "2026-10-05",
68
+ "subject": "release v77"
69
+ },
70
+ {
71
+ "version": "v76",
72
+ "sha": "7990f03e02544",
73
+ "date": "2026-10-01",
74
+ "subject": "release v76"
75
+ },
76
+ {
77
+ "version": "v75",
78
+ "sha": "2c5146981ccfb",
79
+ "date": "2026-09-19",
80
+ "subject": "release v75"
81
+ },
82
+ {
83
+ "version": "v74",
84
+ "sha": "40dceba9262ec",
85
+ "date": "2026-09-14",
86
+ "subject": "release v74: waves 58+59 incl. the Amazon credential door (W59-T21/22/23) and sync producer (W59-T42); hotfix promote so"
87
+ },
88
+ {
89
+ "version": "v73",
90
+ "sha": "bb94de5665e62",
91
+ "date": "2026-09-13",
92
+ "subject": "release v73"
93
+ },
94
+ {
95
+ "version": "v71",
96
+ "sha": "96143dc6fb656",
97
+ "date": "2026-09-04",
98
+ "subject": "release v71 - wave 51, QA'd 24 PASS / 6 FAIL / 5 UNPROVEN"
99
+ },
100
+ {
101
+ "version": "v70",
102
+ "sha": "eedfb6a00163c",
103
+ "date": "2026-09-03",
104
+ "subject": "release v70 - wave 50 (grid analytics, the Sheet as a spreadsheet, view trash and audit)"
105
+ },
106
+ {
107
+ "version": "v69",
108
+ "sha": "8598f56293a00",
109
+ "date": "2026-09-02",
110
+ "subject": "release v69: wave48-continuation validated + hotfixes v67/v68/d4e3e31 + go-live QA repairs (Supplier option limit, optio"
111
+ },
112
+ {
113
+ "version": "v68",
114
+ "sha": "62b065a2610a4",
115
+ "date": "2026-09-01",
116
+ "subject": "release v68: unified holiday date offsets"
117
+ },
118
+ {
119
+ "version": "v67",
120
+ "sha": "3b2241982cf73",
121
+ "date": "2026-09-01",
122
+ "subject": "release v67"
123
+ },
124
+ {
125
+ "version": "v66",
126
+ "sha": "73c94f0767404",
127
+ "date": "2026-08-31",
128
+ "subject": "release v66"
129
+ },
130
+ {
131
+ "version": "v65",
132
+ "sha": "9f6cfedb8a43a",
133
+ "date": "2026-08-31",
134
+ "subject": "release v65"
135
+ },
136
+ {
137
+ "version": "v64",
138
+ "sha": "bb8985c6e73e1",
139
+ "date": "2026-08-31",
140
+ "subject": "release v64"
141
+ },
142
+ {
143
+ "version": "v63",
144
+ "sha": "ec75cb8f34129",
145
+ "date": "2026-08-31",
146
+ "subject": "release v63"
147
+ },
148
+ {
149
+ "version": "v62",
150
+ "sha": "2c8f702c964fd",
151
+ "date": "2026-08-31",
152
+ "subject": "release v62"
153
+ },
154
+ {
155
+ "version": "v61",
156
+ "sha": "bbc4c0760ab52",
157
+ "date": "2026-08-30",
158
+ "subject": "v61 - wave 46 hotfix: D-590 schema overlay reaches the grid read (all three assemblies), D-599 route to zone association"
159
+ },
160
+ {
161
+ "version": "v60",
162
+ "sha": "dd1b0620210e4",
163
+ "date": "2026-08-30",
164
+ "subject": "v60 - wave 46"
165
+ },
166
+ {
167
+ "version": "v59",
168
+ "sha": "3cb73ce8021ed",
169
+ "date": "2026-08-29",
170
+ "subject": "release v59"
171
+ },
172
+ {
173
+ "version": "v58",
174
+ "sha": "c526cede82f92",
175
+ "date": "2026-08-29",
176
+ "subject": "release v58"
177
+ },
178
+ {
179
+ "version": "v57",
180
+ "sha": "4859c035e6a7d",
181
+ "date": "2026-08-29",
182
+ "subject": "release v57"
183
+ },
184
+ {
185
+ "version": "v56",
186
+ "sha": "2970b729e8f04",
187
+ "date": "2026-08-28",
188
+ "subject": "release v56"
189
+ },
190
+ {
191
+ "version": "v55",
192
+ "sha": "b766b904b8fa5",
193
+ "date": "2026-08-28",
194
+ "subject": "release v55"
195
+ },
196
+ {
197
+ "version": "v54",
198
+ "sha": "6a8e99a064d99",
199
+ "date": "2026-08-27",
200
+ "subject": "release v54"
201
+ },
202
+ {
203
+ "version": "v53",
204
+ "sha": "89ed2cb867da5",
205
+ "date": "2026-08-24",
206
+ "subject": "release v53"
207
+ },
208
+ {
209
+ "version": "v52",
210
+ "sha": "8263493762416",
211
+ "date": "2026-08-23",
212
+ "subject": "release v52"
213
+ },
214
+ {
215
+ "version": "v51",
216
+ "sha": "5740d20fa585e",
217
+ "date": "2026-08-23",
218
+ "subject": "release v51"
219
+ },
220
+ {
221
+ "version": "v50",
222
+ "sha": "8027c83d1da69",
223
+ "date": "2026-08-23",
224
+ "subject": "release v50"
225
+ },
226
+ {
227
+ "version": "v49",
228
+ "sha": "9a14c01a48a40",
229
+ "date": "2026-08-23",
230
+ "subject": "release v49"
231
+ },
232
+ {
233
+ "version": "v48",
234
+ "sha": "3c69fd93da7c4",
235
+ "date": "2026-08-23",
236
+ "subject": "release v48"
237
+ },
238
+ {
239
+ "version": "v47",
240
+ "sha": "917f9a0eb5097",
241
+ "date": "2026-08-23",
242
+ "subject": "release v47"
243
+ },
244
+ {
245
+ "version": "v46",
246
+ "sha": "a8dc531b5078e",
247
+ "date": "2026-08-23",
248
+ "subject": "release v46"
249
+ },
250
+ {
251
+ "version": "v45",
252
+ "sha": "0b2b5010d86f0",
253
+ "date": "2026-08-23",
254
+ "subject": "release v45"
255
+ },
256
+ {
257
+ "version": "v44",
258
+ "sha": "a7877aaf20ffd",
259
+ "date": "2026-08-23",
260
+ "subject": "Hotfix: allow renaming the active Map view"
261
+ },
262
+ {
263
+ "version": "v43",
264
+ "sha": "e2ba4616fa0fd",
265
+ "date": "2026-08-23",
266
+ "subject": "release v43"
267
+ },
268
+ {
269
+ "version": "v42",
270
+ "sha": "25b885e6373a7",
271
+ "date": "2026-08-23",
272
+ "subject": "release v42"
273
+ },
274
+ {
275
+ "version": "v41",
276
+ "sha": "1d6adcc3c3ac5",
277
+ "date": "2026-08-22",
278
+ "subject": "v41 vertical route navigation"
279
+ },
280
+ {
281
+ "version": "v40",
282
+ "sha": "1451837168f3c",
283
+ "date": "2026-08-22",
284
+ "subject": "release v40 route UI correction"
285
+ },
286
+ {
287
+ "version": "v39",
288
+ "sha": "a78bdf00e38e5",
289
+ "date": "2026-08-21",
290
+ "subject": "W39 clean stale migration targets"
291
+ },
292
+ {
293
+ "version": "v38",
294
+ "sha": "a556631cf979d",
295
+ "date": "2026-08-21",
296
+ "subject": "W39 reversible Live store cutover"
297
+ },
298
+ {
299
+ "version": "v37",
300
+ "sha": "1ddd8cf82379b",
301
+ "date": "2026-08-21",
302
+ "subject": "W39: coalesce derived workspace reads"
303
+ },
304
+ {
305
+ "version": "v36",
306
+ "sha": "51db3b7fee2ce",
307
+ "date": "2026-08-21",
308
+ "subject": "W39 share-field oid repair"
309
+ },
310
+ {
311
+ "version": "v35",
312
+ "sha": "0f642129c8f77",
313
+ "date": "2026-08-21",
314
+ "subject": "W39 visual QA fixture"
315
+ },
316
+ {
317
+ "version": "v34",
318
+ "sha": "cb4caa188e05a",
319
+ "date": "2026-08-21",
320
+ "subject": "W39 Staging PostgreSQL egress controls"
321
+ },
322
+ {
323
+ "version": "v33",
324
+ "sha": "2b115e573d6ab",
325
+ "date": "2026-08-21",
326
+ "subject": "W39: stop nonproduction Postgres background egress"
327
+ },
328
+ {
329
+ "version": "v32",
330
+ "sha": "b47b27ca7b2ae",
331
+ "date": "2026-08-21",
332
+ "subject": "W39: repair Staging Postgres projection"
333
+ },
334
+ {
335
+ "version": "v31",
336
+ "sha": "8880a49a244ee",
337
+ "date": "2026-08-21",
338
+ "subject": "v31: Wave 39 staging candidate"
339
+ },
340
+ {
341
+ "version": "v30",
342
+ "sha": "71776ca6c8dfa",
343
+ "date": "2026-08-20",
344
+ "subject": "v30: W39-T29 Member-owned View first-share hotfix"
345
+ },
346
+ {
347
+ "version": "v29",
348
+ "sha": "55ced5018ff26",
349
+ "date": "2026-08-18",
350
+ "subject": "v29: the build staging has been running since 2026-08-18 (55ced50)."
351
+ },
352
+ {
353
+ "version": "v28",
354
+ "sha": "2793e4ba53dc1",
355
+ "date": "2026-08-18",
356
+ "subject": "release v28"
357
+ },
358
+ {
359
+ "version": "v27",
360
+ "sha": "c05a794837fc2",
361
+ "date": "2026-08-18",
362
+ "subject": "release v27"
363
+ },
364
+ {
365
+ "version": "v25",
366
+ "sha": "bc248295ed660",
367
+ "date": "2026-08-14",
368
+ "subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)"
369
+ },
370
+ {
371
+ "version": "v26",
372
+ "sha": "6a642e139cfaa",
373
+ "date": "2026-08-14",
374
+ "subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)"
375
+ },
376
+ {
377
+ "version": "v24",
378
+ "sha": "514d8292ce734",
379
+ "date": "2026-08-09",
380
+ "subject": "release v24"
381
+ },
382
+ {
383
+ "version": "v23",
384
+ "sha": "44cc28380fca6",
385
+ "date": "2026-08-09",
386
+ "subject": "release v23"
387
+ },
388
+ {
389
+ "version": "v22",
390
+ "sha": "f14e199ff32f4",
391
+ "date": "2026-08-09",
392
+ "subject": "release v22"
393
+ },
394
+ {
395
+ "version": "v21",
396
+ "sha": "5fdffce8b5586",
397
+ "date": "2026-08-09",
398
+ "subject": "release v21"
399
+ },
400
+ {
401
+ "version": "v20",
402
+ "sha": "442e4a1430a77",
403
+ "date": "2026-08-09",
404
+ "subject": "release v20"
405
+ },
406
+ {
407
+ "version": "v19",
408
+ "sha": "94cb4a4dfdfef",
409
+ "date": "2026-08-09",
410
+ "subject": "release v19"
411
+ },
412
+ {
413
+ "version": "v18",
414
+ "sha": "0106ef9498056",
415
+ "date": "2026-08-09",
416
+ "subject": "release v18"
417
+ },
418
+ {
419
+ "version": "v17",
420
+ "sha": "308240cc75e87",
421
+ "date": "2026-08-09",
422
+ "subject": "release v17"
423
+ },
424
+ {
425
+ "version": "v16",
426
+ "sha": "a804df54430f2",
427
+ "date": "2026-08-09",
428
+ "subject": "release v16"
429
+ },
430
+ {
431
+ "version": "v15",
432
+ "sha": "6b84d7264c81e",
433
+ "date": "2026-08-09",
434
+ "subject": "release v15"
435
+ },
436
+ {
437
+ "version": "v14",
438
+ "sha": "bee5eaf5b5cbd",
439
+ "date": "2026-08-07",
440
+ "subject": "release v14"
441
+ },
442
+ {
443
+ "version": "v13",
444
+ "sha": "314c0e368b1d2",
445
+ "date": "2026-08-06",
446
+ "subject": "release v13"
447
+ },
448
+ {
449
+ "version": "v12",
450
+ "sha": "4908f399785d2",
451
+ "date": "2026-08-06",
452
+ "subject": "release v12"
453
+ },
454
+ {
455
+ "version": "v11",
456
+ "sha": "312661712b980",
457
+ "date": "2026-08-05",
458
+ "subject": "release v11"
459
+ },
460
+ {
461
+ "version": "v10",
462
+ "sha": "43f8042874969",
463
+ "date": "2026-08-05",
464
+ "subject": "release v10"
465
+ },
466
+ {
467
+ "version": "v9",
468
+ "sha": "2e26f57396b19",
469
+ "date": "2026-08-05",
470
+ "subject": "release v9"
471
+ },
472
+ {
473
+ "version": "v8",
474
+ "sha": "8073b3ecbac3d",
475
+ "date": "2026-08-05",
476
+ "subject": "release v8"
477
+ },
478
+ {
479
+ "version": "v7",
480
+ "sha": "b1c5a7e9b8486",
481
+ "date": "2026-08-05",
482
+ "subject": "release v7"
483
+ },
484
+ {
485
+ "version": "v6",
486
+ "sha": "a344020f114ac",
487
+ "date": "2026-08-05",
488
+ "subject": "release v6"
489
+ },
490
+ {
491
+ "version": "v5",
492
+ "sha": "c0b0fbb40a7b5",
493
+ "date": "2026-08-05",
494
+ "subject": "release v5"
495
+ },
496
+ {
497
+ "version": "v4",
498
+ "sha": "de5037ff1c8fd",
499
+ "date": "2026-08-05",
500
+ "subject": "release v4"
501
+ },
502
+ {
503
+ "version": "v3",
504
+ "sha": "6b9fa62d93e87",
505
+ "date": "2026-08-05",
506
+ "subject": "release v3"
507
+ },
508
+ {
509
+ "version": "v2",
510
+ "sha": "fd05861998586",
511
+ "date": "2026-08-04",
512
+ "subject": "release v2"
513
+ },
514
+ {
515
+ "version": "v1",
516
+ "sha": "5b4e2c4bc3f3c",
517
+ "date": "2026-08-04",
518
+ "subject": "release v1"
519
  }
520
  ]
521
  }
VERSION CHANGED
@@ -1 +1 @@
1
- v88 (592ef76d1f)
 
1
+ caef761dd5af969155c5c31f930259757738efe4 (caef761dd5af9)
api/amazon_relational.py CHANGED
@@ -847,26 +847,38 @@ def _integer(value):
847
  return None if value in (None, "") else int(value)
848
 
849
 
850
- def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVED):
851
- """One binding from the current mirror schema, or an explicit unloaded-mirror reason.
852
-
853
- `status` is a `source_status` answer the caller already holds.
854
- """
855
- columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
856
- columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
857
- if not columns:
858
- return missing_source(key, cur=cur)
859
- names = {name for name, _dtype in columns}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
860
  table = key.removeprefix(PREFIX)
861
- prepared = {row[0] for row in cur.execute(
862
- "SELECT column_name FROM information_schema.columns WHERE table_catalog = current_database() "
863
- "AND table_schema = current_schema() AND table_name = ? AND column_name IN (?, ?, ?)",
864
- [SOURCE[key], amazon_schema.IDENTITY_OCCURRENCE, amazon_schema.IDENTITY_PID,
865
- amazon_schema.CONNECTION_ID]).fetchall()}
866
  identity_names = names | prepared
867
  identity = _identity_columns(table, identity_names)
868
- fallback = sorted(set((*identity, *_fallback_columns(table))))
869
  source = _quoted(SOURCE[key])
 
 
870
 
871
  def value(name, owner=source):
872
  return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
@@ -875,55 +887,62 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
875
  def digest(values):
876
  return f"sha256(to_json(list_value({', '.join(values)})))"
877
 
878
- complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity)
879
- peer = '"_amazon_identity_peer"'
880
- peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity)
881
- partition = ", ".join(value(name, peer) for name in fallback)
882
- # Unprepared mirrors rank once in a materialized relation, outside the row lookup.
883
- occurrence = (
884
- '(WITH "_amazon_missing_identity" AS MATERIALIZED '
885
- f'(SELECT {peer}.rowid AS "_identity_rowid", '
886
- f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 '
887
- f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) '
888
- 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_missing_identity" '
889
- f'WHERE "_identity_rowid" = {source}.rowid)')
890
- if amazon_schema.IDENTITY_OCCURRENCE in prepared:
891
- occurrence = f'CAST({source}.{_quoted(amazon_schema.IDENTITY_OCCURRENCE)} AS VARCHAR)'
892
- fallback_digest = digest(["'__amazon_missing_identity__'",
893
- *(value(name) for name in fallback), occurrence])
894
- digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
895
- f"ELSE {fallback_digest} END")
896
- if table in amazon_store.DERIVED_TABLES:
897
- # Derived views construct complete keys and have no physical rowid for fallback order.
898
- digest_sql = digest([value(name) for name in identity])
899
- # The shared grid, relation and selection contracts all use numeric pids. Thirteen
900
- # hex digits retain a stable 52-bit identity that JavaScript represents exactly.
901
- ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))"
 
 
902
  if amazon_schema.IDENTITY_PID in prepared:
903
  ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}'
904
- elif amazon_schema.CONNECTION_ID in prepared:
905
- legacy_owner = amazon_store.legacy_identity_owner(cur, table)
 
906
  if legacy_owner:
907
  literal = "'" + str(legacy_owner).replace("'", "''") + "'"
908
  old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID})
909
- old_fallback = sorted(set((*old_identity, *_fallback_columns(table))))
910
- old_complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in old_identity)
911
- old_peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in old_identity)
912
- old_partition = ", ".join(value(name, peer) for name in old_fallback)
913
- old_occurrence = (
914
- '(WITH "_amazon_legacy_identity" AS MATERIALIZED '
915
- f'(SELECT {peer}.rowid AS "_identity_rowid", '
916
- f'row_number() OVER (PARTITION BY {old_partition} ORDER BY {peer}.rowid) - 1 '
917
- f'AS "_identity_occurrence" FROM {source} AS {peer} '
918
- f'WHERE {peer}."_connection_id"={literal} AND NOT ({old_peer_complete})) '
919
- 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_legacy_identity" '
920
- f'WHERE "_identity_rowid" = {source}.rowid)')
921
- old_missing = digest(["'__amazon_missing_identity__'", *(value(name) for name in old_fallback),
922
- old_occurrence])
923
- old_digest = (f"CASE WHEN {old_complete} THEN {digest([value(name) for name in old_identity])} "
924
- f"ELSE {old_missing} END")
925
- old_ident = f"greatest(1, CAST(('0x' || substr(({old_digest}), 1, 13)) AS BIGINT))"
926
  ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
927
  cols = {}
928
  personal = set(amazon_store.personal_columns(table))
929
  for name, dtype in columns:
@@ -942,10 +961,8 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
942
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
943
  for lookup in lookups(cur, key, names, allowed=allow_lookups):
944
  cols[lookup["key"]] = (lookup["expr"], _text)
945
- spec = {"table": SOURCE[key], "where": "", "id": ident,
946
- "needs_excluded": False, "cols": cols}
947
- if table not in amazon_store.DERIVED_TABLES:
948
- spec["page_key"] = f"{source}.rowid"
949
  # D-2643: an empty table carries its code and reason to every door that reads this binding.
950
  status = source_status(cur, key) if status is _UNRESOLVED else status
951
  if status is not None:
@@ -953,6 +970,22 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
953
  return spec
954
 
955
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
956
  def page_source(cur, key, definition=None, *, allow_lookups=True):
957
  """Resolve one target's storage policy on the admitted cursor without provider discovery."""
958
  columns = mirror_columns(cur, SOURCE[key], strict=True)
 
847
  return None if value in (None, "") else int(value)
848
 
849
 
850
+ def _mirror_schemas(cur, tables):
851
+ rows = cur.execute(
852
+ 'SELECT c.table_name, c.column_name, c.data_type, t.table_type '
853
+ 'FROM information_schema.columns c JOIN information_schema.tables t '
854
+ 'USING (table_catalog, table_schema, table_name) '
855
+ 'WHERE c.table_catalog = current_database() AND c.table_schema = current_schema() '
856
+ 'AND c.table_name IN (SELECT unnest(?::VARCHAR[])) ORDER BY c.table_name, c.ordinal_position',
857
+ [list(tables)]).fetchall()
858
+ schemas = {}
859
+ for table, name, kind, table_type in rows:
860
+ columns, _view = schemas.setdefault(table, ([], table_type == 'VIEW'))
861
+ columns.append((name, kind))
862
+ return schemas
863
+
864
+
865
+ def _missing_occurrence(source, partition, complete, predicate=''):
866
+ return ('(WITH "_amazon_missing_identity" AS MATERIALIZED '
867
+ f'(SELECT "_amazon_identity_peer".rowid AS "_identity_rowid", '
868
+ f'row_number() OVER (PARTITION BY {partition} ORDER BY "_amazon_identity_peer".rowid) - 1 '
869
+ f'AS "_identity_occurrence" FROM {source} AS "_amazon_identity_peer" '
870
+ f'WHERE {predicate}NOT ({complete})) '
871
+ 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_missing_identity" '
872
+ f'WHERE "_identity_rowid" = {source}.rowid)')
873
+
874
+
875
+ def _grid_identity(cur, key, names, prepared, is_view):
876
  table = key.removeprefix(PREFIX)
 
 
 
 
 
877
  identity_names = names | prepared
878
  identity = _identity_columns(table, identity_names)
 
879
  source = _quoted(SOURCE[key])
880
+ peer = '"_amazon_identity_peer"'
881
+ ranks = []
882
 
883
  def value(name, owner=source):
884
  return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
 
887
  def digest(values):
888
  return f"sha256(to_json(list_value({', '.join(values)})))"
889
 
890
+ def identity_sql(keys, label, legacy=None):
891
+ if table in amazon_store.DERIVED_TABLES:
892
+ hashed = digest([value(name) for name in keys])
893
+ return f"greatest(1, CAST(('0x' || substr(({hashed}), 1, 13)) AS BIGINT))"
894
+ complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in keys)
895
+ fallback = sorted(set((*keys, *_fallback_columns(table))))
896
+ partition = ", ".join(value(name, peer) for name in fallback)
897
+ if is_view:
898
+ if legacy:
899
+ partition += f', ({peer}."_connection_id" = {legacy})'
900
+ ranks.append(f'row_number() OVER (PARTITION BY {partition} '
901
+ f'ORDER BY {peer}."_amazon_publication_order") - 1 AS {_quoted(label)}')
902
+ occurrence = f'CAST({source}.{_quoted(label)} AS VARCHAR)'
903
+ else:
904
+ peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in keys)
905
+ predicate = f'{peer}."_connection_id" = {legacy} AND ' if legacy else ''
906
+ occurrence = _missing_occurrence(source, partition, peer_complete, predicate)
907
+ if not legacy and amazon_schema.IDENTITY_OCCURRENCE in prepared:
908
+ occurrence = f'CAST({source}.{_quoted(amazon_schema.IDENTITY_OCCURRENCE)} AS VARCHAR)'
909
+ if is_view:
910
+ ranks.pop()
911
+ fallback_digest = digest(["'__amazon_missing_identity__'", *(value(name) for name in fallback), occurrence])
912
+ hashed = digest([value(name) for name in keys])
913
+ hashed = f'CASE WHEN {complete} THEN {hashed} ELSE {fallback_digest} END'
914
+ return f"greatest(1, CAST(('0x' || substr(({hashed}), 1, 13)) AS BIGINT))"
915
+
916
  if amazon_schema.IDENTITY_PID in prepared:
917
  ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}'
918
+ else:
919
+ ident = identity_sql(identity, '_amazon_publication_occurrence')
920
+ legacy_owner = amazon_store.legacy_identity_owner(cur, table) if amazon_schema.CONNECTION_ID in prepared else None
921
  if legacy_owner:
922
  literal = "'" + str(legacy_owner).replace("'", "''") + "'"
923
  old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID})
924
+ old_ident = identity_sql(old_identity, '_amazon_publication_legacy_occurrence', literal)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
925
  ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
926
+ spec = {'id': ident}
927
+ if ranks:
928
+ spec['from_sql'] = (f'(SELECT *, {", ".join(ranks)} FROM '
929
+ f'(SELECT *, row_number() OVER () - 1 AS "_amazon_publication_order" '
930
+ f'FROM {source}) AS {peer}) AS {source}')
931
+ if not is_view and table not in amazon_store.DERIVED_TABLES:
932
+ spec['page_key'] = f'{source}.rowid'
933
+ return spec
934
+
935
+
936
+ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVED, schema=None):
937
+ """Bind the canonical grid identity and fields to the admitted table or publication view."""
938
+ raw, is_view = schema if schema is not None else _mirror_schemas(cur, [SOURCE[key]]).get(SOURCE[key], ([], False))
939
+ columns = raw if columns is None else columns
940
+ columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
941
+ if not columns:
942
+ return missing_source(key, cur=cur)
943
+ names = {name for name, _dtype in columns}
944
+ prepared = {name for name, _kind in raw} & amazon_schema.MIRROR_INTERNAL_COLUMNS
945
+ table = key.removeprefix(PREFIX)
946
  cols = {}
947
  personal = set(amazon_store.personal_columns(table))
948
  for name, dtype in columns:
 
961
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
962
  for lookup in lookups(cur, key, names, allowed=allow_lookups):
963
  cols[lookup["key"]] = (lookup["expr"], _text)
964
+ spec = {"table": SOURCE[key], "where": "", "needs_excluded": False, "cols": cols,
965
+ **_grid_identity(cur, key, names, prepared, is_view)}
 
 
966
  # D-2643: an empty table carries its code and reason to every door that reads this binding.
967
  status = source_status(cur, key) if status is _UNRESOLVED else status
968
  if status is not None:
 
970
  return spec
971
 
972
 
973
+ def publication_sources(cur, keys):
974
+ """Resolve completed raw publications once without request readiness or store discovery."""
975
+ keys = [key for key in keys if key in SOURCE]
976
+ schemas = _mirror_schemas(cur, [SOURCE[key] for key in keys])
977
+ result = {}
978
+ for key in keys:
979
+ schema = schemas.get(SOURCE[key])
980
+ if schema is None:
981
+ continue
982
+ spec = grid_source(cur, key, allow_lookups=False, status=None, schema=schema)
983
+ if amazon_schema.CONNECTION_ID in dict(schema[0]):
984
+ spec['cols'][amazon_schema.CONNECTION_ID] = ('"_connection_id"', _text)
985
+ result[key] = spec
986
+ return result
987
+
988
+
989
  def page_source(cur, key, definition=None, *, allow_lookups=True):
990
  """Resolve one target's storage policy on the admitted cursor without provider discovery."""
991
  columns = mirror_columns(cur, SOURCE[key], strict=True)
api/registry_source.py CHANGED
@@ -526,6 +526,19 @@ class CanonicalStore(BatchStore):
526
  key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
527
  self.pool_cache[key] = (time.time(), rows)
528
 
 
 
 
 
 
 
 
 
 
 
 
 
 
529
  def get_projection(self, name, drop=()):
530
  if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
531
  return self.get(name)
@@ -846,6 +859,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
846
  # A caller's request snapshot carries the generation taken before its first read.
847
  metadata = {'_scan': ids is None} if metadata is None else metadata
848
  before = metadata.get('_generation') or generation(session, table_key)
 
849
  # A whole-field read needs no schema pass: its one row assembly serves every visible field.
850
  definition = metadata.get('_definition') or (None if needed is None else
851
  canonical_metadata(session, table_key, metadata=metadata, cur=cur))
@@ -890,7 +904,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
890
  cached = session.runtime.pool_cache.get(pool_key)
891
  fresh = bool(cached) and time.time() - cached[0] < 900
892
  # A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
893
- paused = not fresh and routes_keychain.odoo_paused(session.runtime)
894
  # One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
895
  size = None if table_key == 'product_data' else BATCH_ROWS
896
  if cached and (paused or fresh):
 
526
  key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
527
  self.pool_cache[key] = (time.time(), rows)
528
 
529
+ def revision(self, name):
530
+ before = self.metadata.get('_generation')
531
+ mapped = getattr(self._runtime, 'store_key', str)(name)
532
+ if before is not None and mapped in before[1]:
533
+ return copy.deepcopy(before[1][mapped])
534
+ return self._runtime.revision(name)
535
+
536
+ def odoo_paused(self):
537
+ slot = ('odoo_paused',)
538
+ if slot not in self.metadata:
539
+ self.metadata[slot] = self._runtime.odoo_paused()
540
+ return self.metadata[slot]
541
+
542
  def get_projection(self, name, drop=()):
543
  if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
544
  return self.get(name)
 
859
  # A caller's request snapshot carries the generation taken before its first read.
860
  metadata = {'_scan': ids is None} if metadata is None else metadata
861
  before = metadata.get('_generation') or generation(session, table_key)
862
+ metadata['_generation'] = before
863
  # A whole-field read needs no schema pass: its one row assembly serves every visible field.
864
  definition = metadata.get('_definition') or (None if needed is None else
865
  canonical_metadata(session, table_key, metadata=metadata, cur=cur))
 
904
  cached = session.runtime.pool_cache.get(pool_key)
905
  fresh = bool(cached) and time.time() - cached[0] < 900
906
  # A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
907
+ paused = not fresh and routes_keychain.odoo_paused(runtime)
908
  # One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
909
  size = None if table_key == 'product_data' else BATCH_ROWS
910
  if cached and (paused or fresh):
api/routes_automation.py CHANGED
@@ -2873,14 +2873,13 @@ def _received_projection(connector):
2873
  from fastapi import HTTPException
2874
  from core import user_tables
2875
  import routes_odoo_tables
2876
- result = {}
2877
  if connector == 'amazon':
2878
- # Amazon specs raise BinderException on a publication's raw views and cost about 7 s
2879
- # a table in the worker hand-off (sdlc85:88), so the projection waits for that fix.
2880
- logging.getLogger(__name__).debug('received-change projection skips amazon until its specs bind')
2881
- return result
2882
- prefix = 'ut_amzn_' if connector == 'amazon' else 'ut_odoo_'
2883
- for key in user_tables.all_defs(st=rt, schema_only=True):
2884
  if not str(key).startswith(prefix):
2885
  continue
2886
  try:
 
2873
  from fastapi import HTTPException
2874
  from core import user_tables
2875
  import routes_odoo_tables
2876
+ definitions = user_tables.all_defs(st=rt, schema_only=True)
2877
  if connector == 'amazon':
2878
+ import amazon_relational
2879
+ return amazon_relational.publication_sources(cursor, definitions)
2880
+ result = {}
2881
+ prefix = 'ut_odoo_'
2882
+ for key in definitions:
 
2883
  if not str(key).startswith(prefix):
2884
  continue
2885
  try:
api/routes_customers.py CHANGED
@@ -138,7 +138,10 @@ def _assembly_revisions(runtime, names, uses_roster=False, roster_tenant=None, r
138
  """Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
139
  from core import store, store_pg
140
  names = tuple(sorted(set(names)))
141
- handle = _assembly_pg_store(runtime)
 
 
 
142
  users_stamp = None
143
 
144
  def roster_stamp(revision):
 
138
  """Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
139
  from core import store, store_pg
140
  names = tuple(sorted(set(names)))
141
+ # The outer canonical generation check validates these borrowed pre-read revisions after hydration.
142
+ canonical = (getattr(runtime, '_canonical_read_only', False)
143
+ and getattr(runtime, 'metadata', {}).get('_generation') is not None)
144
+ handle = None if canonical else _assembly_pg_store(runtime)
145
  users_stamp = None
146
 
147
  def roster_stamp(revision):
api/routes_sheet.py CHANGED
@@ -7084,23 +7084,20 @@ def _drill_tokens(session, databases):
7084
 
7085
  @contextlib.contextmanager
7086
  def _drill_mirror(session, database, window, needed, databases, sources=None):
7087
- """A small reader-scoped LRU of query mirrors: one lease per key, short registry guard.
7088
-
7089
- `sources` is the caller's opening token pass. The lease key reuses it, so one request reads
7090
- the tokens once before the build. The build-miss and end-of-request rechecks still read anew.
7091
- """
7092
  from core import sheet_binding, perm_scope
7093
  from core.request_timing import raise_if_resource_error
7094
  import datetime
7095
 
7096
  for key in sorted(databases):
7097
  _require_caught_up(session, key)
 
7098
  if _native_table_definition(session, database) is not None:
7099
  # Native rows are a fresh scoped Data payload. Never cache their cursor/permissions
7100
  # across requests; the request's ExitStack closes it after the page is produced.
7101
  yield _mirror(session)(database, metric_window=window, needed=needed), None
 
7102
  return
7103
- sources = _drill_tokens(session, databases) if sources is None else dict(sources)
7104
  token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
7105
  # C1 also fingerprints derived wall sources, which can change without the primary table.
7106
  # An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
@@ -7118,23 +7115,11 @@ def _drill_mirror(session, database, window, needed, databases, sources=None):
7118
  getattr(session.runtime.tenant, "config", None)])
7119
  key = (session.tenant, session.uname, database, token, reader,
7120
  sheet_binding.fingerprint(window), datetime.date.today().isoformat())
7121
- attrs = ("_sheet_drill_cursor", "_sheet_drill_isolated", "_sheet_mirrors",
7122
- "_sheet_assemblies", "_sheet_stored_mirrors", "_sheet_amzn_cursor",
7123
- "_sheet_drill_entries")
7124
- absent = object()
7125
- previous = {name: getattr(session, name, absent) for name in attrs}
7126
  with _drill_lease(key if token else None) as lease:
7127
  with _drill_mirror_lock():
7128
  hit = _DRILL_MIRRORS.get(key) if token else None
7129
  lease["entry"] = hit
7130
- session._sheet_drill_isolated = True
7131
- session._sheet_drill_cursor = hit["owned"] if hit else None
7132
- session._sheet_drill_entries = hit["mirrors"] if hit else {}
7133
- session._sheet_mirrors = {}
7134
- session._sheet_assemblies = {}
7135
- session._sheet_stored_mirrors = {}
7136
- session._sheet_amzn_cursor = session._sheet_drill_cursor
7137
- try:
7138
  want = set(needed) | (hit["needed"] if hit else set())
7139
  if hit and want <= hit["needed"]:
7140
  made = hit["made"]
@@ -7150,42 +7135,70 @@ def _drill_mirror(session, database, window, needed, databases, sources=None):
7150
  owned = getattr(session, "_sheet_drill_cursor", None)
7151
  hit = {"made": made, "needed": want, "owned": owned,
7152
  "mirrors": session._sheet_drill_entries}
7153
- # A build racing a source change is usable for this request, never reusable.
7154
- if token and _drill_tokens(session, databases) == sources:
7155
- with _DRILL_MIRROR_LOCK:
7156
- _DRILL_MIRRORS[key] = hit
7157
- _DRILL_MIRRORS.move_to_end(key)
7158
- lease["entry"] = hit
7159
- while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
7160
- gone, evicted = _DRILL_MIRRORS.popitem(last=False)
7161
- # A leased cursor closes when its holder leaves, never under it.
7162
- held = _DRILL_LEASES.get(gone)
7163
- if evicted["owned"] and not (held and held["entry"] is evicted):
7164
- evicted["owned"][0].close()
7165
- else:
7166
- token = None
7167
  # Related leaves may revisit the primary registry. Keep its table intact and widen
7168
  # only within this cursor. Assemblies remain request-local and leave with the call.
7169
  session._sheet_mirrors[(database, json.dumps(window, sort_keys=True, default=str))] = (
7170
  want, made)
7171
- yield made, token
7172
- finally:
7173
- owned = getattr(session, "_sheet_drill_cursor", None)
7174
- entry = lease["entry"]
7175
- with _DRILL_MIRROR_LOCK:
7176
- lease["entry"] = None
7177
- kept = [item["owned"] for item in _DRILL_MIRRORS.values()]
7178
- # The last lease on an uncached or evicted cursor closes it.
7179
- spent = [owned, entry["owned"] if entry else None]
7180
- for index, cursor in enumerate(spent):
7181
- if (cursor and not any(seen is cursor for seen in spent[:index])
7182
- and not any(item is cursor for item in kept)):
7183
- cursor[0].close()
7184
- for name, value in previous.items():
7185
- if value is absent:
7186
- session.__dict__.pop(name, None)
7187
- else:
7188
- setattr(session, name, value)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7189
 
7190
 
7191
  def _drill_page(session, database, record_ids, offset, limit, total):
@@ -7533,21 +7546,19 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
7533
  binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
7534
  if all(sources.values()) and wall_stamp else None
7535
 
7536
- limits = list(limits) + list(spec.get("limits") or [])
7537
- out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
7538
- "record_ids_scope": "page", "id_limit": None, "limits": limits,
7539
- "population": population, "query": query}
7540
- if scope.get("view_id"):
7541
- out["view_id"] = scope["view_id"]
7542
- else:
7543
- out["filter"] = scope.get("filter")
7544
- if summary is not None:
7545
- out["summary"] = {"ok": True, **summary}
7546
- else:
7547
- out["page"] = _drill_page(session, database, record_ids, offset, limit, total)
7548
- out["page"]["totalUnfiltered"] = population_total
7549
- if _drill_tokens(session, databases) != sources:
7550
- raise err(409, "drill_population_changed", "These records changed while loading. Try again.")
7551
  out["token"] = token
7552
  return out
7553
 
 
7084
 
7085
  @contextlib.contextmanager
7086
  def _drill_mirror(session, database, window, needed, databases, sources=None):
7087
+ """Lease a reader-scoped mirror and validate its source tokens after page hydration."""
 
 
 
 
7088
  from core import sheet_binding, perm_scope
7089
  from core.request_timing import raise_if_resource_error
7090
  import datetime
7091
 
7092
  for key in sorted(databases):
7093
  _require_caught_up(session, key)
7094
+ sources = _drill_tokens(session, databases) if sources is None else dict(sources)
7095
  if _native_table_definition(session, database) is not None:
7096
  # Native rows are a fresh scoped Data payload. Never cache their cursor/permissions
7097
  # across requests; the request's ExitStack closes it after the page is produced.
7098
  yield _mirror(session)(database, metric_window=window, needed=needed), None
7099
+ _require_drill_tokens(session, databases, sources)
7100
  return
 
7101
  token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
7102
  # C1 also fingerprints derived wall sources, which can change without the primary table.
7103
  # An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
 
7115
  getattr(session.runtime.tenant, "config", None)])
7116
  key = (session.tenant, session.uname, database, token, reader,
7117
  sheet_binding.fingerprint(window), datetime.date.today().isoformat())
 
 
 
 
 
7118
  with _drill_lease(key if token else None) as lease:
7119
  with _drill_mirror_lock():
7120
  hit = _DRILL_MIRRORS.get(key) if token else None
7121
  lease["entry"] = hit
7122
+ with _drill_reader(session, hit, lease):
 
 
 
 
 
 
 
7123
  want = set(needed) | (hit["needed"] if hit else set())
7124
  if hit and want <= hit["needed"]:
7125
  made = hit["made"]
 
7135
  owned = getattr(session, "_sheet_drill_cursor", None)
7136
  hit = {"made": made, "needed": want, "owned": owned,
7137
  "mirrors": session._sheet_drill_entries}
7138
+ # Same-key readers wait for this lease until final validation succeeds.
7139
+ if token:
7140
+ _retain_drill_mirror(key, hit, lease)
 
 
 
 
 
 
 
 
 
 
 
7141
  # Related leaves may revisit the primary registry. Keep its table intact and widen
7142
  # only within this cursor. Assemblies remain request-local and leave with the call.
7143
  session._sheet_mirrors[(database, json.dumps(window, sort_keys=True, default=str))] = (
7144
  want, made)
7145
+ try:
7146
+ yield made, token
7147
+ _require_drill_tokens(session, databases, sources)
7148
+ except BaseException:
7149
+ with _DRILL_MIRROR_LOCK:
7150
+ if _DRILL_MIRRORS.get(key) is hit:
7151
+ _DRILL_MIRRORS.pop(key)
7152
+ raise
7153
+
7154
+
7155
+ @contextlib.contextmanager
7156
+ def _drill_reader(session, hit, lease):
7157
+ """Restore request attributes and close cursors whose leases no longer retain them."""
7158
+ attrs = ("_sheet_drill_cursor", "_sheet_drill_isolated", "_sheet_mirrors",
7159
+ "_sheet_assemblies", "_sheet_stored_mirrors", "_sheet_amzn_cursor", "_sheet_drill_entries")
7160
+ absent = object()
7161
+ previous = {name: getattr(session, name, absent) for name in attrs}
7162
+ session._sheet_drill_isolated = True
7163
+ session._sheet_drill_cursor = hit['owned'] if hit else None
7164
+ session._sheet_drill_entries = hit['mirrors'] if hit else {}
7165
+ session._sheet_mirrors, session._sheet_assemblies, session._sheet_stored_mirrors = {}, {}, {}
7166
+ session._sheet_amzn_cursor = session._sheet_drill_cursor
7167
+ try:
7168
+ yield
7169
+ finally:
7170
+ owned = getattr(session, '_sheet_drill_cursor', None)
7171
+ entry = lease['entry']
7172
+ with _DRILL_MIRROR_LOCK:
7173
+ lease['entry'] = None
7174
+ kept = [item['owned'] for item in _DRILL_MIRRORS.values()]
7175
+ spent = [owned, entry['owned'] if entry else None]
7176
+ for index, cursor in enumerate(spent):
7177
+ if (cursor and not any(seen is cursor for seen in spent[:index])
7178
+ and not any(item is cursor for item in kept)):
7179
+ cursor[0].close()
7180
+ for name, value in previous.items():
7181
+ if value is absent:
7182
+ session.__dict__.pop(name, None)
7183
+ else:
7184
+ setattr(session, name, value)
7185
+
7186
+
7187
+ def _require_drill_tokens(session, databases, sources):
7188
+ if _drill_tokens(session, databases) != sources:
7189
+ raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
7190
+
7191
+
7192
+ def _retain_drill_mirror(key, hit, lease):
7193
+ with _DRILL_MIRROR_LOCK:
7194
+ _DRILL_MIRRORS[key] = hit
7195
+ _DRILL_MIRRORS.move_to_end(key)
7196
+ lease['entry'] = hit
7197
+ while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
7198
+ gone, evicted = _DRILL_MIRRORS.popitem(last=False)
7199
+ held = _DRILL_LEASES.get(gone)
7200
+ if evicted['owned'] and not (held and held['entry'] is evicted):
7201
+ evicted['owned'][0].close()
7202
 
7203
 
7204
  def _drill_page(session, database, record_ids, offset, limit, total):
 
7546
  binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
7547
  if all(sources.values()) and wall_stamp else None
7548
 
7549
+ limits = list(limits) + list(spec.get("limits") or [])
7550
+ out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
7551
+ "record_ids_scope": "page", "id_limit": None, "limits": limits,
7552
+ "population": population, "query": query}
7553
+ if scope.get("view_id"):
7554
+ out["view_id"] = scope["view_id"]
7555
+ else:
7556
+ out["filter"] = scope.get("filter")
7557
+ if summary is not None:
7558
+ out["summary"] = {"ok": True, **summary}
7559
+ else:
7560
+ out["page"] = _drill_page(session, database, record_ids, offset, limit, total)
7561
+ out["page"]["totalUnfiltered"] = population_total
 
 
7562
  out["token"] = token
7563
  return out
7564
 
platform/core/amazon_rivals.py CHANGED
@@ -121,11 +121,11 @@ def _backend(rt):
121
  # --------------------------------------------------------------------------- mirror reads
122
 
123
  def weekly_period_sql():
124
- """Only finite dates or timestamps with an inclusive seven-day span define a report week."""
125
  start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
126
  end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
127
  return (f"isfinite({start}) AND isfinite({end}) AND "
128
- f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) = 6")
129
 
130
 
131
  def _week_sql():
 
121
  # --------------------------------------------------------------------------- mirror reads
122
 
123
  def weekly_period_sql():
124
+ """Admit connector weeks with exclusive ends and legacy weeks with inclusive ends."""
125
  start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
126
  end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
127
  return (f"isfinite({start}) AND isfinite({end}) AND "
128
+ f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) IN (6, 7)")
129
 
130
 
131
  def _week_sql():
platform/core/connector_spec.py CHANGED
@@ -3785,15 +3785,15 @@ AMAZON_ADS_REPORTS = ("campaigns", "targeting", "advertised_products", "search_t
3785
  _ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
3786
  _ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
3787
  "advertised_products": "advertised products", "search_terms": "search terms"}
3788
- #: `native` is the Ads report type id. Unknown: the exact v1 spelling; the first credentialed
3789
- #: report settles it (PRD P17), and each id is one literal here.
3790
  _ADS_NATIVE = {
3791
- "sp": {"campaigns": "spCampaigns", "targeting": "spTargeting",
3792
- "advertised_products": "spAdvertisedProduct", "search_terms": "spSearchTerm"},
3793
- "sb": {"campaigns": "sbCampaigns", "targeting": "sbTargeting",
3794
- "advertised_products": "sbAds", "search_terms": "sbSearchTerm"},
3795
- "sd": {"campaigns": "sdCampaigns", "targeting": "sdTargeting",
3796
- "advertised_products": "sdAdvertisedProduct", "search_terms": "sdSearchTerm"},
3797
  }
3798
  _ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
3799
  "advertised_products": "asin", "search_terms": "search_term"}
@@ -3822,21 +3822,22 @@ AMAZON_ADS_TABLES = tuple(
3822
  #: which verify_connectors holds equal to the `"amazon"` registration.
3823
  AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
3824
 
3825
- #: Days the first sync reaches back, per table: the longest window Amazon documents per ad type
3826
- #: (SP 95, SB 60, SD 65, search terms 60). Unknown: v1 retention (primer F12). The first
3827
- #: credentialed sync measures the oldest day Amazon returns.
3828
- AMAZON_ADS_BACKFILL_DAYS = {
3829
- spec.key: (60 if spec.key.endswith("_search_terms")
3830
- else {"sp": 95, "sb": 60, "sd": 65}[spec.key[len(_ADS):len(_ADS) + 2]])
3831
- for spec in AMAZON_ADS_TABLES}
3832
 
3833
  #: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
3834
  AMAZON_ADS_THROUGHPUT = (
3835
- "Amazon Reporting v1 runs one report at a time per advertiser and about 100 reports a day. "
3836
- "Sponsored Products, Brands and Display need 11 report types, and one report can take up to "
3837
- "3 hours, so one day of refresh may not finish in a day. Loopable runs the reports one at a "
3838
- "time and keeps every day it already has. If the refresh falls behind, ask Amazon Ads support "
3839
- "to raise the report limit for this advertiser.")
 
 
 
 
 
3840
 
3841
  #: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
3842
  AMAZON_ADS_NOT_OFFERED = {
@@ -3859,9 +3860,10 @@ def amazon_ads_description(key):
3859
  if unoffered:
3860
  return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
3861
  days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
3862
- return (f"{spec.label} by day, from Amazon Ads Reporting v1. History is kept from the day you "
3863
- f"connect, and the first sync reaches back {days} days, the longest Amazon keeps. "
3864
- f"{AMAZON_ADS_THROUGHPUT}")
 
3865
 
3866
 
3867
  def table_group(key):
 
3785
  _ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
3786
  _ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
3787
  "advertised_products": "advertised products", "search_terms": "search terms"}
3788
+ #: `native` is the Reporting v1 report group that fills the table (`amazon_ads_schema.REPORT_GROUPS`).
3789
+ #: v1 has no report types. SD search terms has no group, because v1 does not offer it.
3790
  _ADS_NATIVE = {
3791
+ "sp": {"campaigns": "campaigns", "targeting": "targeting",
3792
+ "advertised_products": "advertised_products", "search_terms": "search_terms"},
3793
+ "sb": {"campaigns": "campaigns", "targeting": "targeting",
3794
+ "advertised_products": "advertised_products_sb", "search_terms": "search_terms"},
3795
+ "sd": {"campaigns": "campaigns", "targeting": "targeting",
3796
+ "advertised_products": "advertised_products", "search_terms": "not_offered"},
3797
  }
3798
  _ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
3799
  "advertised_products": "asin", "search_terms": "search_term"}
 
3822
  #: which verify_connectors holds equal to the `"amazon"` registration.
3823
  AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
3824
 
3825
+ #: Days the first sync reaches back: Reporting v1 keeps 24 months of daily data, and standing
3826
+ #: rule 1 takes all of it.
3827
+ AMAZON_ADS_BACKFILL_DAYS = {spec.key: 730 for spec in AMAZON_ADS_TABLES}
 
 
 
 
3828
 
3829
  #: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
3830
  AMAZON_ADS_THROUGHPUT = (
3831
+ "Amazon Ads runs one report at a time for each Loopable app login (its client ID) and about "
3832
+ "100 reports a day, shared by every advertiser on that login. Sponsored Products, Brands and "
3833
+ "Display need 5 reports for each refresh, and one report can take up to 3 hours, so one day of "
3834
+ "refresh may not finish in a day. Loopable runs the reports one at a time and keeps every day "
3835
+ "it already has. If the refresh falls behind, ask Amazon Ads support to raise the report limit "
3836
+ "for this client ID.")
3837
+
3838
+ #: On screen: the attribution window Amazon applies to Sponsored Products seller metrics.
3839
+ AMAZON_ADS_SP_ATTRIBUTION = ("Sponsored Products sales, orders and units count purchases within 7 "
3840
+ "days of a click, Amazon's default for sellers.")
3841
 
3842
  #: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
3843
  AMAZON_ADS_NOT_OFFERED = {
 
3860
  if unoffered:
3861
  return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
3862
  days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
3863
+ attribution = AMAZON_ADS_SP_ATTRIBUTION + " " if spec.key.startswith(_ADS + "sp_") else ""
3864
+ return (f"{spec.label} by day, from Amazon Ads Reporting v1. {attribution}History is kept from "
3865
+ f"the day you connect, and the first sync reaches back {days} days, the longest Amazon "
3866
+ f"keeps. {AMAZON_ADS_THROUGHPUT}")
3867
 
3868
 
3869
  def table_group(key):
platform/core/received_changes.py CHANGED
@@ -406,6 +406,21 @@ def _move_baseline(rt, identity, expected, updated):
406
  raise BaselineUnavailable("The observation reference was not saved.")
407
 
408
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
409
  def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
410
  diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
411
  "USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
@@ -417,28 +432,29 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
417
  return
418
  cursor, done = con.cursor(), 0
419
  try:
420
- cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
421
- + " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
422
- while batch := cursor.fetchmany(256):
423
- for row_id, before_json, after_json in batch:
424
- # Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
425
- if deadline is not None and time.monotonic() >= deadline[0]:
426
- print(f"[received-changes] {database}: journal deadline {deadline[1]:g} s passed "
427
- f"after {done} of {count} rows; the rest are not journaled", flush=True)
428
- return
429
- before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
430
- changes = {}
431
- for key, (_sql, formatter) in spec["cols"].items():
432
- left, right = before.get(key), after.get(key)
433
- if formatter:
434
- left = formatter(left) if left is not None else None
435
- right = formatter(right) if right is not None else None
436
- if left != right:
437
- changes[key] = {"before": left, "after": right}
438
- operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
439
- publish(rt, source=source, database=database, row_id=row_id, changes=changes,
440
- receipt_id=snapshot, operation=operation)
441
- done += 1
 
442
  finally:
443
  cursor.close()
444
 
@@ -517,18 +533,31 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
517
  deadline = (time.monotonic() + seconds, seconds)
518
  con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
519
  try:
520
- for name, files in paths.items():
521
- con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
522
- for database, spec in resolver(rt, con).items():
523
- # A skipped database keeps its stored baseline, so the next publication compares again.
524
- if time.monotonic() >= deadline[0]:
525
- print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
526
- continue
527
- try:
528
- _snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
529
- except Exception as exc:
530
- _LOG.warning("Received-change projection deferred for %s: %s",
531
- database, type(exc).__name__)
 
 
 
 
 
 
 
 
 
 
 
 
 
532
  finally:
533
  con.close()
534
 
 
406
  raise BaselineUnavailable("The observation reference was not saved.")
407
 
408
 
409
+ @contextmanager
410
+ def _query_watchdog(con, deadline):
411
+ """Use the runtime's registered statement deadlines for publication queries."""
412
+ if deadline is None:
413
+ yield
414
+ return
415
+ from core import request_timing
416
+ parent = request_timing.CURRENT.get()
417
+ until = min(deadline[0], parent.deadline) if parent is not None and parent.deadline else deadline[0]
418
+ with request_timing.request_scope() as acc:
419
+ acc.deadline = until
420
+ with request_timing.duckdb_operation(con):
421
+ yield
422
+
423
+
424
  def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
425
  diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
426
  "USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
 
432
  return
433
  cursor, done = con.cursor(), 0
434
  try:
435
+ with _query_watchdog(cursor, deadline):
436
+ cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
437
+ + " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
438
+ while batch := cursor.fetchmany(256):
439
+ for row_id, before_json, after_json in batch:
440
+ # Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
441
+ if deadline is not None and time.monotonic() >= deadline[0]:
442
+ print(f"[received-changes] {database}: journal deadline {deadline[1]:g} s passed "
443
+ f"after {done} of {count} rows; the rest are not journaled", flush=True)
444
+ return
445
+ before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
446
+ changes = {}
447
+ for key, (_sql, formatter) in spec["cols"].items():
448
+ left, right = before.get(key), after.get(key)
449
+ if formatter:
450
+ left = formatter(left) if left is not None else None
451
+ right = formatter(right) if right is not None else None
452
+ if left != right:
453
+ changes[key] = {"before": left, "after": right}
454
+ operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
455
+ publish(rt, source=source, database=database, row_id=row_id, changes=changes,
456
+ receipt_id=snapshot, operation=operation)
457
+ done += 1
458
  finally:
459
  cursor.close()
460
 
 
533
  deadline = (time.monotonic() + seconds, seconds)
534
  con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
535
  try:
536
+ with _query_watchdog(con, deadline):
537
+ for name, files in paths.items():
538
+ con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
539
+ started = time.monotonic()
540
+ specs = resolver(rt, con)
541
+ resolution_seconds = time.monotonic() - started
542
+ snapshots_started = time.monotonic()
543
+ completed = deferred = skipped = 0
544
+ for database, spec in specs.items():
545
+ # A skipped database keeps its stored baseline for the next publication.
546
+ if time.monotonic() >= deadline[0]:
547
+ skipped += 1
548
+ print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
549
+ continue
550
+ try:
551
+ _snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
552
+ completed += 1
553
+ except Exception as exc:
554
+ deferred += 1
555
+ _LOG.warning("Received-change projection deferred for %s: %s",
556
+ database, type(exc).__name__)
557
+ print(f'[received-changes] publication connector={connector} tables={len(specs)} '
558
+ f'completed={completed} deferred={deferred} skipped={skipped} '
559
+ f'resolution={resolution_seconds:.3f}s snapshots={time.monotonic() - snapshots_started:.3f}s',
560
+ flush=True)
561
  finally:
562
  con.close()
563
 
platform/core/sheet_binding.py CHANGED
@@ -58,7 +58,7 @@ def database_change_token(st, database):
58
  key(connector_sync.JOBS_KEY), database,
59
  database, key(connector_sync.STATE_KEY))).fetchone()
60
  token, definition = row[0], row[1] or {}
61
- if store_pg._ut_rows(key(user_tables.STORE_KEY), handle.tenant_slug):
62
  from core import user_table_rows
63
  from psycopg import sql
64
  definition = user_table_rows.definition_view(user_table_rows.read_definition(
 
58
  key(connector_sync.JOBS_KEY), database,
59
  database, key(connector_sync.STATE_KEY))).fetchone()
60
  token, definition = row[0], row[1] or {}
61
+ if database.startswith('ut_') and store_pg._ut_rows(key(user_tables.STORE_KEY), handle.tenant_slug):
62
  from core import user_table_rows
63
  from psycopg import sql
64
  definition = user_table_rows.definition_view(user_table_rows.read_definition(
platform/harness/amazon_ads_client.py CHANGED
@@ -4,7 +4,7 @@
4
  read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
5
  refusal lives here, in the request builder, before any credential read, token exchange or socket:
6
 
7
- reports create, status and row pages on Reporting v1 (R8)
8
  streams create and list subscriptions for the dedicated V1 destination
9
  named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
10
  `bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
@@ -17,8 +17,11 @@ harness module never imports `modules` at load). A missing approval source refus
17
  made until Royal's Direct Advertiser credential arrives (R7, R17).
18
 
19
  NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
20
- stored with the credential, so the client needs no profile listing call.
 
 
21
  """
 
22
  import importlib
23
  import json
24
  import logging
@@ -31,6 +34,7 @@ import urllib.parse
31
  import urllib.request
32
  from contextlib import contextmanager
33
  from datetime import date, datetime, timedelta, timezone
 
34
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
35
 
36
  KEY = "amazon_ads"
@@ -46,22 +50,28 @@ DEFAULT_REGION = "na"
46
  LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
47
  CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
48
 
49
- #: Reporting v1 limits (primer F11, F12): one report in flight per advertiser, about 100 a day,
50
- #: up to 3 hours per report, at most 31 days per request.
51
  REPORT_CONCURRENCY = 1
52
  REPORTS_PER_DAY = 100
53
  REPORT_MAX_WAIT_SECONDS = 3 * 3600
54
- MAX_DAYS_PER_REQUEST = 31
55
- REPORT_POLL_SECONDS = 30.0
 
 
56
  _TIMEOUT = 60
57
 
58
- #: The only non-write calls: Reporting v1 create, status and row pages. Unknown: the exact v1
59
- #: paths; the first credentialed call settles them, and each is one literal here.
60
  REPORT_OPERATIONS = {
61
- "createReport": ("POST", "reporting/v1/reports"),
62
- "getReport": ("GET", "reporting/v1/reports/{reportId}"),
63
- "getReportRows": ("GET", "reporting/v1/reports/{reportId}/rows"),
64
  }
 
 
 
 
65
 
66
  # Subscription management never admits an advertising write.
67
  STREAM_OPERATIONS = {
@@ -208,7 +218,15 @@ class AdsWriteRefused(AdsError):
208
  f"the named writes: bid, budget, negative keyword and scheduled change.")
209
 
210
 
211
- class AdsReportUnavailable(AdsError):
 
 
 
 
 
 
 
 
212
  """Amazon does not offer this report to this advertiser."""
213
 
214
 
@@ -225,25 +243,10 @@ def fixture_mode():
225
  return os.environ.get(FIXTURE_ENV) == "1"
226
 
227
 
228
- def _template(path):
229
- return tuple(path.strip("/").split("/"))
230
-
231
-
232
- def _matches(template, path):
233
- parts = str(path or "").strip("/").split("/")
234
- if len(parts) != len(template):
235
- return False
236
- return all((bool(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part)))
237
- if seg.startswith("{") and seg.endswith("}") else part == seg
238
- for part, seg in zip(parts, template))
239
-
240
-
241
  def report_operation(method, path):
242
- """The Reporting v1 operation name for one call, or None."""
243
- for name, (verb, tmpl) in REPORT_OPERATIONS.items():
244
- if str(method).upper() == verb and _matches(_template(tmpl), path):
245
- return name
246
- return None
247
 
248
 
249
  class _Prepared:
@@ -503,12 +506,17 @@ def _send(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
503
  if not _admitted(prepared):
504
  raise AdsWriteRefused("the request did not come from the request builder")
505
  if fixture_mode():
506
- return FIXTURE.answer(prepared, tenant_key, profile_id)
 
507
  return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
508
 
509
 
510
  # --------------------------------------------------------------------------- live transport
511
 
 
 
 
 
512
  def credential(tenant_key, rt=None, connection_id=None):
513
  """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
514
  try:
@@ -538,11 +546,50 @@ def _token(fields):
538
  raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
539
 
540
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
541
  def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
542
  fields = credential(tenant_key, rt, connection_id=connection_id)
543
  if fields is None:
544
- raise AdsError("No Amazon Ads credential is stored for this tenant. "
545
- "Connect Amazon Ads under Amazon Ads Setup.")
546
  if prepared.operation in STREAM_OPERATIONS and (
547
  str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
548
  or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
@@ -551,13 +598,7 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
551
  url = f"{host}/{prepared.path}"
552
  if prepared.query:
553
  url += "?" + urllib.parse.urlencode(prepared.query)
554
- headers = {"Authorization": f"Bearer {_token(fields)}",
555
- "Amazon-Advertising-API-ClientId": fields["client_id"],
556
- "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
557
- "Content-Type": "application/json", "Accept": "application/json"}
558
- if prepared.operation in STREAM_OPERATIONS:
559
- headers["Content-Type"] = STREAM_CONTENT_TYPE
560
- headers["Accept"] = STREAM_CONTENT_TYPE
561
  data = json.dumps(prepared.body).encode() if prepared.body is not None else None
562
  req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
563
  try:
@@ -566,12 +607,10 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
566
  except urllib.error.HTTPError as exc:
567
  try:
568
  detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
569
- message = str(detail.get("details") or detail.get("message") or "")[:200]
570
- except Exception: # noqa: BLE001
571
- message = ""
572
- if exc.code == 400 and re.search(r"not (?:supported|offered|available)", message, re.I):
573
- raise AdsReportUnavailable(f"Amazon does not offer this report: {message}") from None
574
- raise AdsError(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}: {message}") from None
575
  except Exception as exc: # noqa: BLE001
576
  raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
577
 
@@ -580,89 +619,276 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
580
 
581
  _REPORT_LOCKS = {}
582
  _REPORT_LOCKS_GUARD = threading.Lock()
 
 
 
 
 
 
 
 
 
 
 
 
583
 
584
 
585
- def _report_lock(tenant_key, profile_id):
 
586
  with _REPORT_LOCKS_GUARD:
587
- return _REPORT_LOCKS.setdefault((tenant_key, str(profile_id or "")), threading.Lock())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
588
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
589
 
590
- def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep,
591
- now=time.monotonic, max_wait=None, report_id=None):
592
- """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).
593
 
594
- `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without createReport.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
595
  """
596
  from harness import amazon_ads_schema as schema
597
- spec = schema.TABLE_SPECS[schema.short_name(table)]
598
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
599
- raise ValueError("a report window is at most 31 days")
600
- body = {"reportTypeId": spec["report_type"], "adProduct": spec["ad_type"].upper(),
601
- "startDate": start.isoformat(), "endDate": end.isoformat(),
602
- "timeUnit": "DAILY", "columns": list(schema.vendor_columns(table)), "format": "JSON"}
603
  limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
604
- # One report in flight per advertiser (F11): the lock serialises this process's reports.
605
- with _report_lock(tenant_key, profile_id):
606
  if report_id is None:
607
- created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body,
608
- rt=rt, profile_id=profile_id, connection_id=connection_id)
609
- report_id = str(created.get("reportId") or "")
610
- else:
611
- report_id = str(report_id)
612
- if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id):
613
- raise AdsError("Amazon Ads returned no report id")
614
- started = now()
615
- status_path = REPORT_OPERATIONS["getReport"][1].replace("{reportId}", report_id)
616
- while True:
617
- status = request(tenant_key, "GET", status_path, rt=rt, profile_id=profile_id, connection_id=connection_id)
618
- state = str(status.get("status") or "").upper()
619
- if state == "COMPLETED":
620
- break
621
- if state in ("FAILED", "FAILURE", "CANCELLED"):
622
- reason = str(status.get("failureReason") or state)[:200]
623
- if re.search(r"not (?:supported|offered|available)", reason, re.I):
624
- raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
625
- raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}")
626
- elapsed = now() - started
627
- if elapsed > REPORT_MAX_WAIT_SECONDS:
628
- raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
629
- if limit is not None and elapsed >= limit:
630
- raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
631
- pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
632
- sleep(pause if limit is None else min(pause, limit - elapsed))
633
- rows, token = [], None
634
- rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id)
635
- while True:
636
- page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None,
637
- rt=rt, profile_id=profile_id, connection_id=connection_id)
638
- rows.extend(page.get("rows") or [])
639
- token = page.get("nextToken")
640
- if not token:
641
- return report_id, rows
642
 
643
 
644
  # --------------------------------------------------------------------------- recording fixture
645
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
646
  class _Fixture:
647
- """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to. Records each call; answers reports
648
- with deterministic rows, two report polls, and paged rows."""
649
 
650
- PAGE_SIZE = 25
 
 
 
 
 
651
 
652
  def __init__(self):
653
  self.reset()
654
 
655
  def reset(self):
656
- self.calls, self.reports = [], {}
657
- self.subscriptions = {}
658
  self.in_flight = self.max_in_flight = 0
659
- self.drop_days, self.unsupported = set(), set()
 
660
  self.responder = None
661
 
662
- def answer(self, prepared, tenant_key, profile_id):
 
 
 
 
663
  self.calls.append({"method": prepared.method, "path": prepared.path,
664
  "operation": prepared.operation, "body": prepared.body,
665
- "query": dict(prepared.query), "tenant": tenant_key,
666
  "named": dict(prepared.named) if prepared.named else None})
667
  if self.responder is not None:
668
  return self.responder(prepared)
@@ -678,88 +904,174 @@ class _Fixture:
678
  if prepared.named is not None:
679
  items = next(iter(prepared.body.values()))
680
  return {"success": [{"index": i} for i in range(len(items))], "error": []}
 
 
681
  if prepared.operation == "createReport":
682
- report_id = f"fx{len(self.reports) + 1:05d}"
683
- self.in_flight += 1
684
- self.max_in_flight = max(self.max_in_flight, self.in_flight)
685
- self.reports[report_id] = {"body": prepared.body, "polls": 0, "rows": None}
686
- return {"reportId": report_id, "status": "PENDING"}
687
- report_id = prepared.path.split("/")[3]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
688
  report = self.reports[report_id]
689
- if prepared.operation == "getReport":
690
- report["polls"] += 1
691
- if report["body"]["reportTypeId"] in self.unsupported:
692
- self.in_flight -= 1
693
- return {"status": "FAILED", "failureReason": "Report type is not supported for this advertiser"}
694
- if report["polls"] < 2:
695
- return {"status": "PROCESSING"}
696
- return {"status": "COMPLETED"}
697
- if report["rows"] is None:
698
- report["rows"] = fixture_rows(report["body"], drop_days=self.drop_days)
699
- start = int(prepared.query.get("nextToken") or 0)
700
- page = report["rows"][start:start + self.PAGE_SIZE]
701
- end = start + len(page)
702
- if end >= len(report["rows"]):
 
 
 
 
 
 
703
  self.in_flight -= 1
704
- return {"rows": page}
705
- return {"rows": page, "nextToken": str(end)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
706
 
707
 
708
  def fixture_rows(body, *, drop_days=()):
709
- """Deterministic Ads report rows for one request body, every day in its window."""
710
- report_type, columns = body["reportTypeId"], body["columns"]
711
- start, end = date.fromisoformat(body["startDate"]), date.fromisoformat(body["endDate"])
712
- ad = report_type[:2]
713
- out, day = [], start
714
- while day <= end:
715
- if day.isoformat() not in drop_days:
716
- for c in (1, 2):
717
- for g in ((0,) if report_type.endswith("Campaigns") else (1, 2)):
718
- n = day.toordinal() % 7 + c + g
719
- row = {
720
- "date": day.isoformat(), "campaignId": f"{ad.encode().hex()}10{c}",
721
- "campaignName": f"{ad.upper()} campaign {c}", "campaignStatus": "ENABLED",
722
- "campaignBudgetAmount": 25.0 * c, "campaignBudgetCurrencyCode": "USD",
723
- "adGroupId": f"20{c}{g}", "adGroupName": f"Ad group {c}.{g}",
724
- "keywordId": f"30{c}{g}", "targetingId": f"30{c}{g}",
725
- "targeting": f"term {c}{g}", "targetingExpression": f"asin is B00FX{c}{g}",
726
- "matchType": "exact", "keywordBid": 0.5 + g / 10,
727
- "adId": f"40{c}{g}", "advertisedSku": f"SKU-{c}{g}",
728
- "advertisedAsin": f"B00FX0{c}{g}0", "promotedSku": f"SKU-{c}{g}",
729
- "promotedAsin": f"B00FX0{c}{g}0", "searchTerm": f"search {c}{g}",
730
- "impressions": 100 * n, "clicks": 3 * n, "cost": round(1.25 * n, 2),
731
- "sales": round(9.5 * n, 2), "sales14d": round(9.5 * n, 2),
732
- "purchases": n, "purchases14d": n, "unitsSold": n, "unitsSoldClicks14d": n,
733
- }
734
- out.append({k: row.get(k) for k in columns})
735
- day += timedelta(days=1)
736
  return out
737
 
738
 
739
  FIXTURE = _Fixture()
740
 
741
 
 
 
 
 
 
 
742
  @contextmanager
743
  def recording_fixture():
744
- """Route every call to a fresh recording fixture for the block. -> the fixture."""
745
  previous = os.environ.get(FIXTURE_ENV)
746
  os.environ[FIXTURE_ENV] = "1"
747
  FIXTURE.reset()
 
748
  try:
749
  yield FIXTURE
750
  finally:
751
  FIXTURE.reset()
 
752
  if previous is None:
753
  os.environ.pop(FIXTURE_ENV, None)
754
  else:
755
  os.environ[FIXTURE_ENV] = previous
756
 
757
 
758
- __all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "NAMED_WRITES",
759
- "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
760
  "FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
761
  "SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
762
- "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsReportUnavailable",
763
- "AdsReportPending", "fixture_mode", "build_request", "report_operation",
764
- "send_named_write", "request", "credential", "run_report", "FIXTURE",
765
- "recording_fixture", "fixture_rows"]
 
4
  read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
5
  refusal lives here, in the request builder, before any credential read, token exchange or socket:
6
 
7
+ reports Reporting v1: create, retrieve, the advertiser account query, and the report files
8
  streams create and list subscriptions for the dedicated V1 destination
9
  named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
10
  `bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
 
17
  made until Royal's Direct Advertiser credential arrives (R7, R17).
18
 
19
  NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
20
+ stored with the credential. v1 names the account by `advertiserAccountId`, which the client looks
21
+ up once per connection from `query/advertiserAccounts`. Every path, header and field comes from
22
+ Amazon's saved contracts in `AutoWiki/research/amazon-ads-v1-schema/`.
23
  """
24
+ import gzip
25
  import importlib
26
  import json
27
  import logging
 
34
  import urllib.request
35
  from contextlib import contextmanager
36
  from datetime import date, datetime, timedelta, timezone
37
+ from typing import NamedTuple
38
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
39
 
40
  KEY = "amazon_ads"
 
50
  LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
51
  CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
52
 
53
+ #: Reporting v1 limits (limits.md, dates.md): one report in flight per client ID, about 100 a day,
54
+ #: up to 3 hours per report, at most 120 days per request.
55
  REPORT_CONCURRENCY = 1
56
  REPORTS_PER_DAY = 100
57
  REPORT_MAX_WAIT_SECONDS = 3 * 3600
58
+ MAX_DAYS_PER_REQUEST = 120
59
+ REPORT_POLL_SECONDS = 60.0
60
+ #: Amazon's error text reaches the sync report verbatim, up to this many characters.
61
+ ERROR_TEXT_LIMIT = 600
62
  _TIMEOUT = 60
63
 
64
+ #: The only non-write calls: the Reporting v1 operations, each a POST to one literal path from the
65
+ #: Reports and AdvertiserAccounts contracts. Every other `adsApi` path refuses, a GET included.
66
  REPORT_OPERATIONS = {
67
+ "createReport": ("POST", "adsApi/v1/create/reports"),
68
+ "retrieveReport": ("POST", "adsApi/v1/retrieve/reports"),
69
+ "queryAdvertiserAccounts": ("POST", "adsApi/v1/query/advertiserAccounts"),
70
  }
71
+ #: The contract names `Amazon-Ads-ClientId`. The limits page names the older header. v1 sends both.
72
+ V1_CLIENT_HEADERS = ("Amazon-Ads-ClientId", "Amazon-Advertising-API-ClientId")
73
+ #: A report file is a presigned S3 URL: https, on this host suffix, GET with no Ads header.
74
+ REPORT_FILE_HOST_SUFFIX = ".amazonaws.com"
75
 
76
  # Subscription management never admits an advertising write.
77
  STREAM_OPERATIONS = {
 
218
  f"the named writes: bid, budget, negative keyword and scheduled change.")
219
 
220
 
221
+ class AdsRejected(AdsError):
222
+ """Amazon answered an error. `errors` keeps each entry's code, message and fieldLocation."""
223
+
224
+ def __init__(self, message, errors=()):
225
+ super().__init__(message)
226
+ self.errors = list(errors)
227
+
228
+
229
+ class AdsReportUnavailable(AdsRejected):
230
  """Amazon does not offer this report to this advertiser."""
231
 
232
 
 
243
  return os.environ.get(FIXTURE_ENV) == "1"
244
 
245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  def report_operation(method, path):
247
+ """The Reporting v1 operation name for one call, or None. A path matches only its literal."""
248
+ return next((name for name, (verb, endpoint) in REPORT_OPERATIONS.items()
249
+ if str(method).upper() == verb and str(path or "").strip("/") == endpoint), None)
 
 
250
 
251
 
252
  class _Prepared:
 
506
  if not _admitted(prepared):
507
  raise AdsWriteRefused("the request did not come from the request builder")
508
  if fixture_mode():
509
+ headers = _headers(prepared, FIXTURE.credential(tenant_key), FIXTURE_TOKEN, profile_id)
510
+ return FIXTURE.answer(prepared, tenant_key, profile_id, headers)
511
  return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
512
 
513
 
514
  # --------------------------------------------------------------------------- live transport
515
 
516
+ _NO_CREDENTIAL = ("No Amazon Ads credential is stored for this tenant. "
517
+ "Connect Amazon Ads under Amazon Ads Setup.")
518
+
519
+
520
  def credential(tenant_key, rt=None, connection_id=None):
521
  """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
522
  try:
 
546
  raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
547
 
548
 
549
+ def _headers(prepared, fields, token, profile_id=None):
550
+ """The headers one admitted call sends. A v1 call names its account in the body, not a scope."""
551
+ if prepared.operation in REPORT_OPERATIONS:
552
+ return {"Authorization": f"Bearer {token}", **dict.fromkeys(V1_CLIENT_HEADERS, fields["client_id"]),
553
+ "Content-Type": "application/json", "Accept": "application/json"}
554
+ headers = {"Authorization": f"Bearer {token}",
555
+ "Amazon-Advertising-API-ClientId": fields["client_id"],
556
+ "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
557
+ "Content-Type": "application/json", "Accept": "application/json"}
558
+ if prepared.operation in STREAM_OPERATIONS:
559
+ headers["Content-Type"] = STREAM_CONTENT_TYPE
560
+ headers["Accept"] = STREAM_CONTENT_TYPE
561
+ return headers
562
+
563
+
564
+ def _errors(detail):
565
+ """Amazon's error entries from an error body, each with its code, message and fieldLocation."""
566
+ if not isinstance(detail, dict):
567
+ return []
568
+ entries = detail["errors"] if isinstance(detail.get("errors"), list) else [detail]
569
+ return [{"code": str(e.get("code") or ""), "message": str(e.get("details") or e.get("message") or ""),
570
+ "fieldLocation": str(e.get("fieldLocation") or "")}
571
+ for e in entries if isinstance(e, dict) and (e.get("code") or e.get("details") or e.get("message"))]
572
+
573
+
574
+ def _error_text(errors):
575
+ """Amazon's errors as one line a person reads, verbatim, cut to `ERROR_TEXT_LIMIT`."""
576
+ return " ".join(" ".join(filter(None, (e["code"] + ":" if e["code"] else "", e["message"],
577
+ f"(field {e['fieldLocation']})" if e["fieldLocation"] else "")))
578
+ for e in errors)[:ERROR_TEXT_LIMIT]
579
+
580
+
581
+ def _rejection(prefix, errors, *, may_be_unoffered=True):
582
+ """The exception for Amazon's errors: not offered when Amazon says so, else rejected."""
583
+ text = _error_text(errors)
584
+ if may_be_unoffered and re.search(r"not (?:supported|offered|available)", text, re.I):
585
+ return AdsReportUnavailable(f"Amazon does not offer this report: {text}", errors)
586
+ return AdsRejected(f"{prefix}: {text}" if text else prefix, errors)
587
+
588
+
589
  def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
590
  fields = credential(tenant_key, rt, connection_id=connection_id)
591
  if fields is None:
592
+ raise AdsError(_NO_CREDENTIAL)
 
593
  if prepared.operation in STREAM_OPERATIONS and (
594
  str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
595
  or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
 
598
  url = f"{host}/{prepared.path}"
599
  if prepared.query:
600
  url += "?" + urllib.parse.urlencode(prepared.query)
601
+ headers = _headers(prepared, fields, _token(fields), profile_id)
 
 
 
 
 
 
602
  data = json.dumps(prepared.body).encode() if prepared.body is not None else None
603
  req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
604
  try:
 
607
  except urllib.error.HTTPError as exc:
608
  try:
609
  detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
610
+ except Exception: # noqa: BLE001 an unreadable body still raises its status
611
+ detail = {}
612
+ raise _rejection(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}", _errors(detail),
613
+ may_be_unoffered=exc.code == 400) from None
 
 
614
  except Exception as exc: # noqa: BLE001
615
  raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
616
 
 
619
 
620
  _REPORT_LOCKS = {}
621
  _REPORT_LOCKS_GUARD = threading.Lock()
622
+ #: Process caches: the advertiser account per connection, the optional fields Amazon refused per
623
+ #: (client id, group) with Amazon's text, and the report file URLs a retrieve returned.
624
+ _ACCOUNTS = {}
625
+ _REFUSED = {}
626
+ _ISSUED_FILES = set()
627
+
628
+
629
+ class GroupReport(NamedTuple):
630
+ """One finished group report: its id, flat rows per member table, and notes a person reads."""
631
+ report_id: str
632
+ rows: dict
633
+ notes: tuple
634
 
635
 
636
+ def _report_lock(client_id):
637
+ """One lock per client ID: Amazon runs one report at a time for each app login."""
638
  with _REPORT_LOCKS_GUARD:
639
+ return _REPORT_LOCKS.setdefault(str(client_id), threading.Lock())
640
+
641
+
642
+ def _report_credential(tenant_key, rt, connection_id):
643
+ """The credential a report runs under: its client id keys the lock, its profile names the account."""
644
+ if fixture_mode():
645
+ return FIXTURE.credential(tenant_key)
646
+ fields = credential(tenant_key, rt, connection_id=connection_id)
647
+ if fields is None:
648
+ raise AdsError(_NO_CREDENTIAL)
649
+ return fields
650
+
651
+
652
+ def _lists_profile(account, profile_id):
653
+ return (isinstance(account, dict) and bool(account.get("advertiserAccountId"))
654
+ and any(isinstance(alt, dict) and str(alt.get("profileId")) == str(profile_id)
655
+ for alt in account.get("alternateIds") or ()))
656
+
657
 
658
+ def _advertiser_account(tenant_key, *, rt, profile_id, connection_id):
659
+ """The v1 `advertiserAccountId` whose alternate ids carry the stored profile id, cached per connection."""
660
+ key, token = (tenant_key, connection_id or "", str(profile_id)), None
661
+ while key not in _ACCOUNTS:
662
+ page = request(tenant_key, "POST", REPORT_OPERATIONS["queryAdvertiserAccounts"][1],
663
+ body={"maxResults": 100, **({"nextToken": token} if token else {})},
664
+ rt=rt, profile_id=profile_id, connection_id=connection_id)
665
+ match = next((a for a in page.get("advertiserAccounts") or () if _lists_profile(a, profile_id)), None)
666
+ token = page.get("nextToken")
667
+ if match is not None:
668
+ _ACCOUNTS[key] = str(match["advertiserAccountId"])
669
+ elif not token:
670
+ raise AdsError(f"No Amazon Ads advertiser account lists profile {profile_id}. "
671
+ f"Check the profile ID on this Amazon Ads connection.")
672
+ return _ACCOUNTS[key]
673
 
 
 
 
674
 
675
+ def _create_body(group, start, end, account, fields):
676
+ """A CreateReportRequest for one group and window, filtered to the group's ad products."""
677
+ from harness import amazon_ads_schema as schema
678
+ return {"accessRequestedAccounts": [{"advertiserAccountId": account}],
679
+ "reports": [{"format": "GZIP_JSON",
680
+ "periods": [{"datePeriod": {"startDate": start.isoformat(), "endDate": end.isoformat()}}],
681
+ "query": {"fields": list(fields),
682
+ "filter": {"on": {"field": schema.AD_PRODUCT_FIELD, "comparisonOperator": "IN",
683
+ "not": False, "values": list(group.tables)}}}}]}
684
+
685
+
686
+ def _first_report(reply, operation):
687
+ """The one report a 207 multi-status answer carries, or its errors raised verbatim."""
688
+ errors = [e for entry in reply.get("error") or () if isinstance(entry, dict)
689
+ for e in _errors({"errors": entry.get("errors") or []})]
690
+ if errors or reply.get("error"):
691
+ raise _rejection(f"Amazon Ads refused {operation}", errors)
692
+ for entry in reply.get("success") or ():
693
+ report = entry.get("report") if isinstance(entry, dict) else None
694
+ if isinstance(report, dict) and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", str(report.get("reportId") or "")):
695
+ return report
696
+ raise AdsError(f"Amazon Ads returned no report on {operation}")
697
+
698
+
699
+ def _named_fields(exc, fields):
700
+ """The requested fields an error names, by id in its text or by index in its field location."""
701
+ text = " ".join([str(exc)] + [e["fieldLocation"] for e in exc.errors])
702
+ named = {f for f in fields if re.search(rf"(?<![\w.]){re.escape(f)}(?!\w)", text)}
703
+ named.update(fields[int(i)] for i in re.findall(r"fields\[(\d+)\]", text) if int(i) < len(fields))
704
+ return named
705
+
706
+
707
+ def _create(tenant_key, group, start, end, account, refused, opts):
708
+ """Create one group report. An error naming only optional fields drops them and creates again."""
709
+ while True:
710
+ fields = group.fields(drop=refused)
711
+ try:
712
+ reply = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1],
713
+ body=_create_body(group, start, end, account, fields), **opts)
714
+ return _first_report(reply, "createReport")
715
+ except AdsRejected as exc:
716
+ named = _named_fields(exc, fields)
717
+ if not named or named & set(group.core):
718
+ raise
719
+ refused.update(dict.fromkeys(sorted(named), _error_text(exc.errors) or str(exc)))
720
+
721
+
722
+ def _await(tenant_key, report_id, opts, *, sleep, now, limit):
723
+ """Poll retrieve about once a minute until the report completes. -> its report file URLs."""
724
+ started = now()
725
+ while True:
726
+ report = _first_report(request(tenant_key, "POST", REPORT_OPERATIONS["retrieveReport"][1],
727
+ body={"reportIds": [report_id]}, **opts), "retrieveReport")
728
+ state = str(report.get("status") or "").upper()
729
+ if state == "COMPLETED":
730
+ urls = [str(part.get("url") or "") for part in report.get("completedReportParts") or ()
731
+ if isinstance(part, dict)]
732
+ _ISSUED_FILES.update(urls)
733
+ return urls
734
+ if state == "FAILED":
735
+ reason = " ".join(filter(None, (str(report.get("failureCode") or ""),
736
+ str(report.get("failureReason") or ""))))[:ERROR_TEXT_LIMIT] or state
737
+ if re.search(r"not (?:supported|offered|available)", reason, re.I):
738
+ raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
739
+ raise AdsError(f"Amazon Ads report {report_id} ended FAILED: {reason}")
740
+ elapsed = now() - started
741
+ if elapsed > REPORT_MAX_WAIT_SECONDS:
742
+ raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
743
+ if limit is not None and elapsed >= limit:
744
+ raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
745
+ pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
746
+ sleep(pause if limit is None else min(pause, limit - elapsed))
747
+
748
+
749
+ def _flat(row, prefix=""):
750
+ """A nested v1 row -> flat dotted field ids. A flat row stays as it is."""
751
+ out = {}
752
+ for key, value in row.items():
753
+ if isinstance(value, dict):
754
+ out.update(_flat(value, f"{prefix}{key}."))
755
+ else:
756
+ out[f"{prefix}{key}"] = value
757
+ return out
758
+
759
+
760
+ def _part_rows(raw):
761
+ """One report file -> flat rows. GZIP or plain, a JSON array or JSON lines."""
762
+ try:
763
+ text = (gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw).decode("utf-8", "replace").strip()
764
+ if not text:
765
+ return []
766
+ try:
767
+ data = json.loads(text)
768
+ except ValueError:
769
+ data = [json.loads(line) for line in text.splitlines() if line.strip()]
770
+ except (OSError, EOFError, ValueError) as exc:
771
+ raise AdsError(f"Amazon Ads sent a report file Loopable cannot read ({type(exc).__name__})") from None
772
+ return [_flat(row) for row in (data if isinstance(data, list) else [data]) if isinstance(row, dict)]
773
+
774
+
775
+ def _read_file(url):
776
+ """GET one report file with no Ads header. Only an https S3 URL a retrieve returned is admitted."""
777
+ parts = urllib.parse.urlsplit(str(url or ""))
778
+ host = parts.hostname or ""
779
+ if parts.scheme != "https" or not host.endswith(REPORT_FILE_HOST_SUFFIX) or url not in _ISSUED_FILES:
780
+ raise AdsWriteRefused(f"{parts.scheme or 'no scheme'} {_safe_path(host)} is not a report file Amazon returned")
781
+ _ISSUED_FILES.discard(url)
782
+ headers = {}
783
+ if fixture_mode():
784
+ return _part_rows(FIXTURE.download(url, headers))
785
+ try:
786
+ with urllib.request.urlopen(urllib.request.Request(url, method="GET", headers=headers),
787
+ timeout=_TIMEOUT) as r:
788
+ raw = r.read()
789
+ except Exception as exc: # noqa: BLE001 the URL carries a signature: never echo it
790
+ raise AdsError(f"Amazon Ads report file could not be read ({type(exc).__name__})") from None
791
+ return _part_rows(raw)
792
+
793
+
794
+ def _split(group, rows):
795
+ """Flat rows -> {member table: rows} by `adProduct.value`, and a note for rows no member takes."""
796
+ from harness import amazon_ads_schema as schema
797
+ out, tables, stray = {table: [] for table in group.members}, group.tables, 0
798
+ for row in rows:
799
+ table = tables.get(str(row.get(schema.AD_PRODUCT_FIELD) or "").upper())
800
+ if table is None or not row.get(schema.vendor_name("date")):
801
+ stray += 1
802
+ else:
803
+ out[table].append(row)
804
+ note = (f"Amazon Ads sent {stray} rows with no date or with another ad product. Loopable left them out.",)
805
+ return out, note if stray else ()
806
+
807
+
808
+ def run_group_report(tenant_key, group, start, end, *, rt=None, profile_id=None, connection_id=None,
809
+ sleep=time.sleep, now=time.monotonic, max_wait=None, report_id=None):
810
+ """Create one Reporting v1 group report, wait for it, and read every file. -> GroupReport.
811
+
812
+ `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without a create.
813
  """
814
  from harness import amazon_ads_schema as schema
815
+ group = schema.GROUPS[group] if isinstance(group, str) else group
816
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
817
+ raise ValueError(f"a report window is at most {MAX_DAYS_PER_REQUEST} days")
818
+ fields = _report_credential(tenant_key, rt, connection_id)
819
+ opts = {"rt": rt, "profile_id": str(profile_id or fields["profile_id"]), "connection_id": connection_id}
820
+ refused = _REFUSED.setdefault((str(fields["client_id"]), group.name), {})
821
  limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
822
+ with _report_lock(fields["client_id"]):
 
823
  if report_id is None:
824
+ account = _advertiser_account(tenant_key, **opts)
825
+ report_id = _create(tenant_key, group, start, end, account, refused, opts)["reportId"]
826
+ urls = _await(tenant_key, str(report_id), opts, sleep=sleep, now=now, limit=limit)
827
+ rows = [row for url in urls for row in _read_file(url)]
828
+ split, stray = _split(group, rows)
829
+ notes = tuple(f"Amazon Ads refused the field {field} on this report, so Loopable reads it without that "
830
+ f"field. Amazon said: {text}" for field, text in refused.items())
831
+ return GroupReport(str(report_id), split, notes + stray)
832
+
833
+
834
+ def run_report(tenant_key, table, start, end, **options):
835
+ """One table's rows from its group report. -> (report_id, rows). The store reads `run_group_report`."""
836
+ from harness import amazon_ads_schema as schema
837
+ group = schema.group_of(table)
838
+ if group is None:
839
+ raise AdsReportUnavailable(f"Amazon Reporting v1 offers no report for {schema.short_name(table)}")
840
+ result = run_group_report(tenant_key, group, start, end, **options)
841
+ return result.report_id, result.rows[schema.short_name(table)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
842
 
843
 
844
  # --------------------------------------------------------------------------- recording fixture
845
 
846
+ FIXTURE_TOKEN = "fixture-access-token"
847
+ FIXTURE_CREDENTIAL = {"client_id": "amzn1.application-oa2-client.fixture", "client_secret": "fixture",
848
+ "refresh_token": "fixture", "profile_id": "1000000001", "region": "na"}
849
+ FIXTURE_FILE_HOST = "https://fixture-ads-reports.s3.amazonaws.com"
850
+ FIXTURE_CREATED = "2026-10-20T09:00:00Z"
851
+ _FIXTURE_PREFIX = {"SPONSORED_PRODUCTS": ("71", "SP"), "SPONSORED_BRANDS": ("72", "SB"),
852
+ "SPONSORED_DISPLAY": ("73", "SD")}
853
+ #: Per campaign and day: the metric totals every finer row splits evenly.
854
+ _FIXTURE_TOTALS = {"metric.impressions": 400, "metric.clicks": 12, "metric.totalCost": 5.0,
855
+ "metric.sales": 38.0, "metric.purchases": 4, "metric.unitsSold": 4}
856
+
857
+
858
+ def _multi_status(report=None, errors=None):
859
+ """A ReportMultiStatusResponse with one success or one error entry at index 0."""
860
+ return {"success": [{"index": 0, "report": report}] if report else [],
861
+ "error": [{"index": 0, "errors": errors}] if errors else []}
862
+
863
+
864
  class _Fixture:
865
+ """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to, in the saved contracts' v1 shapes.
 
866
 
867
+ It records method, path, operation, headers and body of every call, report file downloads
868
+ included. Toggles: `drop_days`, `unsupported` (group names that end FAILED), `refuse_fields`
869
+ (a create naming one answers 400), `nested` (files carry nested JSON lines), `responder`.
870
+ """
871
+
872
+ FILES = 2
873
 
874
  def __init__(self):
875
  self.reset()
876
 
877
  def reset(self):
878
+ self.calls, self.reports, self.subscriptions, self.credentials = [], {}, {}, {}
 
879
  self.in_flight = self.max_in_flight = 0
880
+ self.drop_days, self.unsupported, self.refuse_fields = set(), set(), set()
881
+ self.nested = False
882
  self.responder = None
883
 
884
+ def credential(self, tenant_key):
885
+ """The fixture credential for one tenant. `credentials[tenant]` overrides its fields."""
886
+ return {**FIXTURE_CREDENTIAL, **self.credentials.get(tenant_key, {})}
887
+
888
+ def answer(self, prepared, tenant_key, profile_id, headers=None):
889
  self.calls.append({"method": prepared.method, "path": prepared.path,
890
  "operation": prepared.operation, "body": prepared.body,
891
+ "query": dict(prepared.query), "tenant": tenant_key, "headers": dict(headers or {}),
892
  "named": dict(prepared.named) if prepared.named else None})
893
  if self.responder is not None:
894
  return self.responder(prepared)
 
904
  if prepared.named is not None:
905
  items = next(iter(prepared.body.values()))
906
  return {"success": [{"index": i} for i in range(len(items))], "error": []}
907
+ if prepared.operation == "queryAdvertiserAccounts":
908
+ return self._accounts(prepared.body, profile_id)
909
  if prepared.operation == "createReport":
910
+ return self._create(prepared.body)
911
+ return self._retrieve(prepared.body["reportIds"][0])
912
+
913
+ def _accounts(self, body, profile_id):
914
+ """Two pages: an unrelated account, then the account that lists the caller's profile."""
915
+ if not body.get("nextToken"):
916
+ return {"advertiserAccounts": [{"advertiserAccountId": "amzn1.ads-account.g.fxother",
917
+ "alternateIds": [{"profileId": "999", "countryCode": "CA"}]}],
918
+ "nextToken": "fixture-page-2"}
919
+ return {"advertiserAccounts": [{"advertiserAccountId": f"amzn1.ads-account.g.fx{profile_id}",
920
+ "alternateIds": [{"profileId": str(profile_id), "countryCode": "US",
921
+ "entityId": "ENTITYFX"}]}]}
922
+
923
+ def _create(self, body):
924
+ if not (body.get("accessRequestedAccounts") or [{}])[0].get("advertiserAccountId"):
925
+ return _multi_status(errors=[{"code": "400001", "message": "An advertiser account is required.",
926
+ "fieldLocation": "accessRequestedAccounts"}])
927
+ refused = [f for f in body["reports"][0]["query"]["fields"] if f in self.refuse_fields]
928
+ if refused:
929
+ raise _rejection("Amazon Ads answered HTTP 400 on createReport",
930
+ [{"code": "400004", "message": f"Field {refused[0]} is invalid for this request.",
931
+ "fieldLocation": ""}], may_be_unoffered=False)
932
+ report_id = f"fx{len(self.reports) + 1:05d}"
933
+ self.in_flight += 1
934
+ self.max_in_flight = max(self.max_in_flight, self.in_flight)
935
+ self.reports[report_id] = {"body": body, "polls": 0, "files": None, "done": False}
936
+ return _multi_status(self._record(report_id, "PENDING"))
937
+
938
+ def _record(self, report_id, status, **extra):
939
+ """A contract `Report`: every required key, the request's format, periods and query echoed."""
940
+ request = self.reports[report_id]["body"]["reports"][0]
941
+ return {"reportId": report_id, "status": status, "format": request["format"],
942
+ "periods": request["periods"], "query": request["query"],
943
+ "creationDateTime": FIXTURE_CREATED, "lastUpdatedDateTime": FIXTURE_CREATED, **extra}
944
+
945
+ def _retrieve(self, report_id):
946
+ from harness import amazon_ads_schema as schema
947
  report = self.reports[report_id]
948
+ report["polls"] += 1
949
+ query = report["body"]["reports"][0]["query"]
950
+ group = schema.group_for(query["fields"], query["filter"]["on"]["values"])
951
+ if group is not None and group.name in self.unsupported:
952
+ self._finish(report)
953
+ return _multi_status(self._record(report_id, "FAILED", failureCode="REPORT_NOT_SUPPORTED",
954
+ failureReason="This report is not supported for this advertiser."))
955
+ if report["polls"] < 2:
956
+ return _multi_status(self._record(report_id, "PROCESSING"))
957
+ if report["files"] is None:
958
+ rows = fixture_rows(report["body"], drop_days=self.drop_days)
959
+ report["files"] = {f"{FIXTURE_FILE_HOST}/{report_id}/part-{i}.json.gz?X-Amz-Signature=fx{report_id}{i}":
960
+ _fixture_file(rows[i::self.FILES], self.nested) for i in range(self.FILES)}
961
+ return _multi_status(self._record(report_id, "COMPLETED", completedDateTime=FIXTURE_CREATED, completedReportParts=[
962
+ {"url": url, "sizeInBytes": len(raw), "urlExpirationDateTime": "2026-12-31T00:00:00Z"}
963
+ for url, raw in report["files"].items()]))
964
+
965
+ def _finish(self, report):
966
+ if not report["done"]:
967
+ report["done"] = True
968
  self.in_flight -= 1
969
+
970
+ def download(self, url, headers):
971
+ """Serve one report file and record the GET with the headers the client sent."""
972
+ self.calls.append({"method": "GET", "path": url, "operation": "downloadReportPart", "body": None,
973
+ "query": {}, "tenant": None, "headers": dict(headers), "named": None})
974
+ report = self.reports[urllib.parse.urlsplit(url).path.split("/")[1]]
975
+ raw = report["files"][url]
976
+ report.setdefault("read", set()).add(url)
977
+ if report["read"] == set(report["files"]):
978
+ self._finish(report)
979
+ return raw
980
+
981
+
982
+ def _fixture_file(rows, nested):
983
+ """GZIP JSON: an array of flat rows, or nested objects one per line."""
984
+ def nest(row):
985
+ out = {}
986
+ for field, value in row.items():
987
+ head, _, tail = field.partition(".")
988
+ out.setdefault(head, {})[tail] = value
989
+ return out
990
+ text = "\n".join(json.dumps(nest(r)) for r in rows) if nested else json.dumps(rows)
991
+ return gzip.compress(text.encode("utf-8"))
992
+
993
+
994
+ def _fixture_query(body):
995
+ report = body["reports"][0]
996
+ period = report["periods"][0]["datePeriod"]
997
+ return (report["query"]["fields"], report["query"]["filter"]["on"]["values"],
998
+ date.fromisoformat(period["startDate"]), date.fromisoformat(period["endDate"]))
999
+
1000
+
1001
+ def _fixture_days(start, end, drop_days):
1002
+ return [start + timedelta(days=i) for i in range((end - start).days + 1)
1003
+ if (start + timedelta(days=i)).isoformat() not in drop_days]
1004
+
1005
+
1006
+ def fixture_totals(body, *, drop_days=()):
1007
+ """The fixture's own totals for one create body. -> {(ad product, date, campaign id): {field: total}}."""
1008
+ _fields, products, start, end = _fixture_query(body)
1009
+ out = {}
1010
+ for day in _fixture_days(start, end, drop_days):
1011
+ for index, product in enumerate(products):
1012
+ for c in (1, 2):
1013
+ n = day.toordinal() % 7 + c + index
1014
+ out[(product, day.isoformat(), f"{_FIXTURE_PREFIX[product][0]}10{c}")] = {
1015
+ field: total * n for field, total in _FIXTURE_TOTALS.items()}
1016
+ return out
1017
 
1018
 
1019
  def fixture_rows(body, *, drop_days=()):
1020
+ """Deterministic v1 rows for one create body: one row per distinct value of every requested
1021
+ dimension and day, only the requested fields and ad products. Two ad groups split a campaign."""
1022
+ fields, _products, start, end = _fixture_query(body)
1023
+ groups = (1, 2) if "adGroup.id" in fields else (0,)
1024
+ out = []
1025
+ for (product, day, campaign), totals in fixture_totals(body, drop_days=drop_days).items():
1026
+ prefix, short = _FIXTURE_PREFIX[product]
1027
+ c = campaign[-1]
1028
+ for g in groups:
1029
+ row = {"date.value": day, "adProduct.value": product, "campaign.id": campaign,
1030
+ "campaign.name": f"{short} campaign {c}", "campaign.deliveryStatus": "DELIVERING",
1031
+ "campaign.budgetAmount": 25 * int(c), "budgetCurrency.value": "USD",
1032
+ "adGroup.id": f"{prefix}20{c}{g}", "adGroup.name": f"Ad group {c}.{g}",
1033
+ "adGroupTarget.id": f"{prefix}30{c}{g}", "target.value": f"term {c}{g}",
1034
+ "target.matchType": "EXACT", "adGroupTarget.bid": str(0.5 + g / 10),
1035
+ "ad.id": f"{prefix}40{c}{g}", "advertisedProduct.id": f"B0FX{prefix}{c}{g}00",
1036
+ "advertisedProductMarketplace.value": "ATVPDKIKX0DER", "advertisedProductSku.value": f"SKU-{c}{g}",
1037
+ "searchTerm.value": f"search {c}{g}",
1038
+ **{f: (t // len(groups) if isinstance(t, int) else t / len(groups)) for f, t in totals.items()}}
1039
+ out.append({f: row[f] for f in fields})
 
 
 
 
 
 
 
1040
  return out
1041
 
1042
 
1043
  FIXTURE = _Fixture()
1044
 
1045
 
1046
+ def _reset_caches():
1047
+ _ACCOUNTS.clear()
1048
+ _REFUSED.clear()
1049
+ _ISSUED_FILES.clear()
1050
+
1051
+
1052
  @contextmanager
1053
  def recording_fixture():
1054
+ """Route every call to a fresh recording fixture and fresh report caches for the block. -> the fixture."""
1055
  previous = os.environ.get(FIXTURE_ENV)
1056
  os.environ[FIXTURE_ENV] = "1"
1057
  FIXTURE.reset()
1058
+ _reset_caches()
1059
  try:
1060
  yield FIXTURE
1061
  finally:
1062
  FIXTURE.reset()
1063
+ _reset_caches()
1064
  if previous is None:
1065
  os.environ.pop(FIXTURE_ENV, None)
1066
  else:
1067
  os.environ[FIXTURE_ENV] = previous
1068
 
1069
 
1070
+ __all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "V1_CLIENT_HEADERS",
1071
+ "NAMED_WRITES", "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
1072
  "FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
1073
  "SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
1074
+ "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsRejected", "AdsReportUnavailable",
1075
+ "AdsReportPending", "GroupReport", "fixture_mode", "build_request", "report_operation",
1076
+ "send_named_write", "request", "credential", "run_group_report", "run_report", "FIXTURE",
1077
+ "recording_fixture", "fixture_rows", "fixture_totals"]
platform/harness/amazon_ads_schema.py CHANGED
@@ -1,16 +1,17 @@
1
  """amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
2
 
3
- One entry per ad type and report: the report type, the columns we store, the Ads column each one
4
- is read from, the row identity and the backfill window. The table keys, labels and backfill days
5
- come from `core.connector_spec`, so the two files cannot disagree on a name.
6
 
7
- Unknown: the exact Reporting v1 column spellings. The vendor names below are the Ads reporting
8
- vocabulary the primer records (F11, F12). The first credentialed report settles them (PRD P17),
9
- and a rename is one literal in `_VENDOR`.
10
  """
11
  import hashlib
12
  import sys
13
  from pathlib import Path
 
14
 
15
  _HERE = Path(__file__).resolve().parent
16
  if str(_HERE.parent) not in sys.path:
@@ -22,8 +23,6 @@ AD_TYPES = _cs.AMAZON_ADS_AD_TYPES
22
  REPORTS = _cs.AMAZON_ADS_REPORTS
23
  DATABASE_PREFIX = "ut_amzn_"
24
  MIRROR_PREFIX = "amzn_ads_"
25
- #: Amazon accepts at most 31 days per report request.
26
- MAX_DAYS_PER_REQUEST = 31
27
 
28
  #: The C6 columns every Ads table carries, in this order.
29
  C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
@@ -54,22 +53,76 @@ _EXTRA = {
54
  }
55
  _TAIL = ("profile_id", "currency")
56
 
57
- #: Our column -> the Ads column, shared by all three ad types unless `_VENDOR_BY_TYPE` says not.
58
  _VENDOR = {
59
- "campaign_id": "campaignId", "campaign_name": "campaignName",
60
- "campaign_status": "campaignStatus", "campaign_budget_amount": "campaignBudgetAmount",
61
- "ad_group_id": "adGroupId", "ad_group_name": "adGroupName", "target_id": "keywordId",
62
- "targeting": "targeting", "match_type": "matchType", "bid": "keywordBid",
63
- "ad_id": "adId", "advertised_sku": "advertisedSku", "asin": "advertisedAsin",
64
- "search_term": "searchTerm", "date": "date", "impressions": "impressions",
65
- "clicks": "clicks", "cost": "cost", "sales": "sales", "orders": "purchases",
66
- "units": "unitsSold", "currency": "campaignBudgetCurrencyCode",
67
- }
68
- _VENDOR_BY_TYPE = {
69
- "sp": {"sales": "sales14d", "orders": "purchases14d", "units": "unitsSoldClicks14d"},
70
- "sd": {"target_id": "targetingId", "targeting": "targetingExpression",
71
- "advertised_sku": "promotedSku", "asin": "promotedAsin"},
72
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
 
74
 
75
  def _short(ad_type, report):
@@ -89,25 +142,24 @@ def coltype(name):
89
  return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
90
 
91
 
92
- def vendor_name(ad_type, column):
93
- """The Ads column one of our columns is read from, or None when the vendor has none."""
94
- if column == "profile_id":
95
- return None
96
- return _VENDOR_BY_TYPE.get(ad_type, {}).get(column) or _VENDOR.get(column)
97
 
98
 
99
  TABLE_SPECS = {}
100
  for _spec in _cs.AMAZON_ADS_TABLES:
101
  _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
 
102
  TABLE_SPECS[_short(_ad_type, _report)] = {
103
  "key": _spec.key, "source": _spec.source, "label": _spec.label,
104
- "ad_type": _ad_type, "report": _report, "report_type": _spec.native,
105
  "columns": _columns(_report), "row_identity": tuple(_spec.identity),
106
  "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
107
  "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
108
  "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
109
  }
110
- del _spec, _ad_type, _report
111
 
112
 
113
  def short_name(table):
@@ -127,11 +179,16 @@ def columns_for(table):
127
  return TABLE_SPECS[short_name(table)]["columns"]
128
 
129
 
130
- def vendor_columns(table):
131
- """The Ads columns one report asks for, in our column order, each once."""
132
- spec = TABLE_SPECS[short_name(table)]
133
- names = (vendor_name(spec["ad_type"], c) for c in spec["columns"])
134
- return tuple(dict.fromkeys(n for n in names if n))
 
 
 
 
 
135
 
136
 
137
  def _cell(name, value):
@@ -149,11 +206,11 @@ def _cell(name, value):
149
 
150
 
151
  def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
152
- """One Ads report row -> one stored row, with its stable `_row_key` and report origin."""
153
  spec = TABLE_SPECS[short_name(table)]
154
  out = {}
155
  for column in spec["columns"]:
156
- source = vendor_name(spec["ad_type"], column)
157
  value = vendor_row.get(source) if source else None
158
  if column == "profile_id":
159
  value = profile_id
@@ -213,7 +270,8 @@ def stream_coltype(name):
213
  return "DOUBLE"
214
  return "VARCHAR"
215
 
216
- __all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "MAX_DAYS_PER_REQUEST",
 
217
  "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
218
- "vendor_columns", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
219
  "STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
 
1
  """amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
2
 
3
+ One entry per ad type and report: the columns we store, the Reporting v1 field each one is read
4
+ from, the row identity and the backfill window. The table keys, labels and backfill days come from
5
+ `core.connector_spec`, so the two files cannot disagree on a name.
6
 
7
+ Field ids come from Amazon's v1 field library, saved at
8
+ `AutoWiki/research/amazon-ads-v1-schema/reporting_v1_fields.json`. v1 has no report types: one
9
+ report per `REPORT_GROUPS` row covers several ad types, and its rows split by `adProduct.value`.
10
  """
11
  import hashlib
12
  import sys
13
  from pathlib import Path
14
+ from typing import NamedTuple
15
 
16
  _HERE = Path(__file__).resolve().parent
17
  if str(_HERE.parent) not in sys.path:
 
23
  REPORTS = _cs.AMAZON_ADS_REPORTS
24
  DATABASE_PREFIX = "ut_amzn_"
25
  MIRROR_PREFIX = "amzn_ads_"
 
 
26
 
27
  #: The C6 columns every Ads table carries, in this order.
28
  C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
 
53
  }
54
  _TAIL = ("profile_id", "currency")
55
 
56
+ #: Our column -> its Reporting v1 field id, one map for every ad type. The store stamps profile_id.
57
  _VENDOR = {
58
+ "date": "date.value", "campaign_id": "campaign.id", "campaign_name": "campaign.name",
59
+ "campaign_status": "campaign.deliveryStatus", "campaign_budget_amount": "campaign.budgetAmount",
60
+ "ad_group_id": "adGroup.id", "ad_group_name": "adGroup.name", "target_id": "adGroupTarget.id",
61
+ "targeting": "target.value", "match_type": "target.matchType", "bid": "adGroupTarget.bid",
62
+ "ad_id": "ad.id", "advertised_sku": "advertisedProductSku.value", "asin": "advertisedProduct.id",
63
+ "search_term": "searchTerm.value", "impressions": "metric.impressions", "clicks": "metric.clicks",
64
+ "cost": "metric.totalCost", "sales": "metric.sales", "orders": "metric.purchases",
65
+ "units": "metric.unitsSold", "currency": "budgetCurrency.value",
 
 
 
 
 
66
  }
67
+ #: The v1 field that names each row's ad type, and its value per ad type.
68
+ AD_PRODUCT_FIELD = "adProduct.value"
69
+ AD_PRODUCTS = {"sp": "SPONSORED_PRODUCTS", "sb": "SPONSORED_BRANDS", "sd": "SPONSORED_DISPLAY"}
70
+ #: The name the v1 field library lists each ad type under.
71
+ LIBRARY_AD_PRODUCTS = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
72
+
73
+ METRICS = ("metric.impressions", "metric.clicks", "metric.totalCost", "metric.sales",
74
+ "metric.purchases", "metric.unitsSold")
75
+ _HEAD = ("date.value", AD_PRODUCT_FIELD, "campaign.id", "campaign.name")
76
+ _AD_GROUP = ("adGroup.id", "adGroup.name")
77
+ _TAIL_FIELDS = ("budgetCurrency.value",) + METRICS
78
+ #: ⛔ Not `targeting.value` or `targetingMatchType.value`: the library lists them for no ad product.
79
+ #: The live probe of 2026-11-18 (`probe-2026-11-18.json`) proved these names, absent from the library.
80
+ _TARGET_TEXT = ("target.value", "target.matchType")
81
+ #: Fields a live Reporting v1 create accepted that the saved field library does not list.
82
+ PROBE_PROVEN = frozenset(_TARGET_TEXT)
83
+
84
+
85
+ class ReportGroup(NamedTuple):
86
+ """One Reporting v1 report and the member tables its rows split into by `adProduct.value`.
87
+
88
+ `optional` fields list no ad product in Amazon's library, or only a live probe proved them:
89
+ a create that refuses one drops it.
90
+ """
91
+ name: str
92
+ members: tuple
93
+ core: tuple
94
+ optional: tuple = ()
95
+
96
+ @property
97
+ def tables(self):
98
+ """`adProduct.value` -> the member table its rows land in."""
99
+ return {AD_PRODUCTS[table.split("_", 1)[0]]: table for table in self.members}
100
+
101
+ def fields(self, drop=()):
102
+ """The fields one create asks for, without the optional fields Amazon refused."""
103
+ return tuple(f for f in self.core + self.optional if f not in drop)
104
+
105
+
106
+ #: ⛔ Every requested dimension is a group-by and sets the row grain. A group asks only for its
107
+ #: tables' identity dimensions, attributes of those, metrics, the ad product and required partners.
108
+ #: Every core field lists every ad product of its group in the library: SB lacks the SKU field and
109
+ #: the marketplace field `advertisedProduct.id` requires, so SB advertised products is its own report.
110
+ REPORT_GROUPS = (
111
+ ReportGroup("campaigns", ("sp_campaigns", "sb_campaigns", "sd_campaigns"),
112
+ _HEAD + ("campaign.deliveryStatus", "campaign.budgetAmount") + _TAIL_FIELDS),
113
+ ReportGroup("targeting", ("sp_targeting", "sb_targeting", "sd_targeting"),
114
+ _HEAD + _AD_GROUP + ("adGroupTarget.id",) + _TAIL_FIELDS,
115
+ _TARGET_TEXT + ("adGroupTarget.bid",)),
116
+ ReportGroup("advertised_products", ("sp_advertised_products", "sd_advertised_products"),
117
+ _HEAD + _AD_GROUP + ("ad.id", "advertisedProduct.id", "advertisedProductMarketplace.value",
118
+ "advertisedProductSku.value") + _TAIL_FIELDS),
119
+ ReportGroup("advertised_products_sb", ("sb_advertised_products",),
120
+ _HEAD + _AD_GROUP + ("ad.id",) + _TAIL_FIELDS),
121
+ ReportGroup("search_terms", ("sp_search_terms", "sb_search_terms"),
122
+ _HEAD + _AD_GROUP + ("adGroupTarget.id", "searchTerm.value") + _TAIL_FIELDS, _TARGET_TEXT),
123
+ )
124
+ GROUPS = {group.name: group for group in REPORT_GROUPS}
125
+ _GROUP_OF = {table: group for group in REPORT_GROUPS for table in group.members}
126
 
127
 
128
  def _short(ad_type, report):
 
142
  return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
143
 
144
 
145
+ def vendor_name(column):
146
+ """The v1 field one of our columns is read from, or None when the vendor has none."""
147
+ return _VENDOR.get(column)
 
 
148
 
149
 
150
  TABLE_SPECS = {}
151
  for _spec in _cs.AMAZON_ADS_TABLES:
152
  _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
153
+ _group = _GROUP_OF.get(_short(_ad_type, _report))
154
  TABLE_SPECS[_short(_ad_type, _report)] = {
155
  "key": _spec.key, "source": _spec.source, "label": _spec.label,
156
+ "ad_type": _ad_type, "report": _report, "group": _group.name if _group else None,
157
  "columns": _columns(_report), "row_identity": tuple(_spec.identity),
158
  "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
159
  "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
160
  "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
161
  }
162
+ del _spec, _ad_type, _report, _group
163
 
164
 
165
  def short_name(table):
 
179
  return TABLE_SPECS[short_name(table)]["columns"]
180
 
181
 
182
+ def group_of(table):
183
+ """The report group that fills one Ads table, or None for a table v1 does not offer."""
184
+ return _GROUP_OF.get(short_name(table))
185
+
186
+
187
+ def group_for(fields, ad_products):
188
+ """The report group a create request asks for, by its fields and ad product filter, or None."""
189
+ fields, ad_products = set(fields), set(ad_products)
190
+ return next((g for g in REPORT_GROUPS if set(g.tables) == ad_products
191
+ and set(g.core) <= fields <= set(g.core + g.optional)), None)
192
 
193
 
194
  def _cell(name, value):
 
206
 
207
 
208
  def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
209
+ """One flat v1 report row (dotted field ids) -> one stored row, with its `_row_key` and report origin."""
210
  spec = TABLE_SPECS[short_name(table)]
211
  out = {}
212
  for column in spec["columns"]:
213
+ source = vendor_name(column)
214
  value = vendor_row.get(source) if source else None
215
  if column == "profile_id":
216
  value = profile_id
 
270
  return "DOUBLE"
271
  return "VARCHAR"
272
 
273
+ __all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "AD_PRODUCT_FIELD", "AD_PRODUCTS",
274
+ "LIBRARY_AD_PRODUCTS", "METRICS", "ReportGroup", "REPORT_GROUPS", "GROUPS",
275
  "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
276
+ "group_of", "group_for", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
277
  "STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
platform/harness/amazon_ads_store.py CHANGED
@@ -10,8 +10,8 @@ back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_B
10
  A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
11
  No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
12
 
13
- ⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one
14
- report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
15
  `amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
16
 
17
  Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
@@ -178,7 +178,7 @@ def _day_marker(con, key):
178
 
179
 
180
  def windows(con, table, connect_day, today, connection_id=None):
181
- """The report windows one sync asks for, oldest first, each at most 31 days."""
182
  table = schema.short_name(table)
183
  spec = schema.TABLE_SPECS[table]
184
  cadence = amazon_client.CADENCE["ads_" + table]
@@ -237,13 +237,13 @@ def _upsert(con, table, rows):
237
  cols = _columns_for(table)
238
  by_key = {r["_row_key"]: r for r in rows}
239
  keys = list(by_key)
 
240
  for i in range(0, len(keys), INSERT_BATCH_ROWS):
241
  part = keys[i:i + INSERT_BATCH_ROWS]
242
  con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
243
- con.executemany(
244
- f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
245
- f'VALUES ({", ".join("?" for _ in cols)})',
246
- [[by_key[k].get(c) for c in cols] for k in part])
247
  return len(by_key)
248
 
249
 
@@ -389,16 +389,158 @@ def _land(con, table, rows, report_id, start, end, profile, connection_id, label
389
 
390
 
391
  def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
392
- """Require named transport support instead of sending a sibling's credential."""
393
  options = {"rt": rt, "profile_id": profile_id, **extra}
394
  if connection_id:
395
  options["connection_id"] = connection_id
396
- try:
397
- return client.run_report(key, table, start, end, **options)
398
- except TypeError as exc:
399
- if connection_id and "unexpected keyword argument 'connection_id'" in str(exc):
400
- raise client.AdsError("The Amazon Ads client needs named connection support before this connection can sync.") from None
401
- raise
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
402
 
403
 
404
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
@@ -409,6 +551,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
409
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
410
  `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
411
  pending ledger, and the next pass reads it before it creates any other report.
 
412
  """
413
  key = amazon_store._tenant_key(tenant_key)
414
  report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
@@ -425,10 +568,9 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
425
  own = con is None
426
  if own:
427
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
428
- bounded = max_seconds is not None
429
  started = _clock()
430
- deadline = started + float(max_seconds) if bounded else None
431
- waits = report["waits"]
432
  try:
433
  datastore._ensure_sync_state(con)
434
  now = _utc_now(now)
@@ -437,164 +579,24 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
437
  def stamp():
438
  return now + timedelta(seconds=max(0.0, _clock() - started))
439
 
440
- def wait(label, spec, reason):
441
- waits.append(f"{label}: {spec['label']}: {reason}")
442
-
443
- synced = []
444
- checkpointed = []
445
- for cid, label, profile, legacy in connections:
446
  connect_day = _connect_day(con, today, cid, legacy=legacy)
447
  report.setdefault("connectDay", connect_day.isoformat())
448
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
449
  report["connections"].append(entry)
450
- used = _budget(con, today, cid, legacy=legacy)
451
- # Resume every report this connection left preparing before it creates another one.
452
- resumed, blocked = {}, False
453
- for table in TABLES:
454
- pending = _read_pending(con, table, cid)
455
- if pending is None:
456
- continue
457
- spec = schema.TABLE_SPECS[table]
458
- _ensure_table(con, table)
459
- if table_name(table) not in checkpointed:
460
- checkpointed.append(table_name(table))
461
- age = (stamp() - pending["created"]).total_seconds()
462
- expired = (f"Amazon Ads report {pending['report_id']} did not finish in 3 hours")
463
- if age >= client.REPORT_MAX_WAIT_SECONDS:
464
- _clear_pending(con, table, cid)
465
- report["problems"].append(f"{label}: {spec['label']}: {expired}")
466
- resumed[table] = ("failed", expired, 0, None)
467
- continue
468
- limit = client.REPORT_MAX_WAIT_SECONDS - age
469
- if bounded:
470
- limit = min(limit, max(0.0, deadline - _clock()))
471
- try:
472
- report_id, rows = _run_report(key, table, pending["start"], pending["end"], rt, profile, cid,
473
- max_wait=limit, report_id=pending["report_id"], now=_clock)
474
- except client.AdsReportUnavailable as exc:
475
- _clear_pending(con, table, cid)
476
- resumed[table] = ("not_offered", _not_offered_reason(spec, exc), 0, None)
477
- continue
478
- except client.AdsReportPending:
479
- if (stamp() - pending["created"]).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS:
480
- _clear_pending(con, table, cid)
481
- report["problems"].append(f"{label}: {spec['label']}: {expired}")
482
- resumed[table] = ("failed", expired, 0, None)
483
- else:
484
- blocked = True
485
- wait(label, spec, PREPARING_REASON)
486
- resumed[table] = ("deferred", PREPARING_REASON, 0, None)
487
- continue
488
- except client.AdsError as exc:
489
- _clear_pending(con, table, cid)
490
- reason = str(exc)[:200]
491
- report["problems"].append(f"{label}: {spec['label']}: {reason}")
492
- resumed[table] = ("failed", reason, 0, None)
493
- continue
494
- _clear_pending(con, table, cid)
495
- pulled = _land(con, table, rows, report_id, pending["start"], pending["end"], profile, cid, label)
496
- resumed[table] = ("done", "", pulled, (pending["start"], pending["end"]))
497
- requested = [schema.short_name(t) for t in (tables or TABLES)]
498
  for table in list(dict.fromkeys(requested + list(resumed))):
499
- spec = schema.TABLE_SPECS[table]
500
- _ensure_table(con, table)
501
- origin_changed = _refresh_origin(con, table, cid, label, legacy)
502
- pulled, state, mark, reason = 0, "done", _marker_table(table), ""
503
- prior = resumed.get(table)
504
- landed = None
505
- if prior is not None:
506
- state, reason, pulled, landed = prior
507
- if state != "done" or table not in requested:
508
- pass
509
- elif not spec["offered"]:
510
- state, reason = "not_offered", spec["not_offered_reason"]
511
- elif not _cadence_due(con, table, now, cid):
512
- state = "skipped"
513
- elif blocked:
514
- state, reason = "deferred", PREPARING_REASON
515
- wait(label, spec, reason)
516
- else:
517
- for start, end in windows(con, table, connect_day, today, cid):
518
- if landed and landed[0] <= start and end <= landed[1]:
519
- continue # The resumed report just read these days.
520
- if used >= client.REPORTS_PER_DAY:
521
- state = "deferred"
522
- reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
523
- f"reports are used, so days from {start.isoformat()} load on the next sync.")
524
- report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit")
525
- waits.append(report["problems"][-1])
526
- break
527
- if bounded and _clock() >= deadline:
528
- state, reason = "deferred", TIME_LIMIT_REASON
529
- wait(label, spec, reason)
530
- break
531
- used += 1
532
- _spend_budget(con, today, used, cid)
533
- created = stamp()
534
- try:
535
- if bounded:
536
- report_id, rows = _run_report(key, table, start, end, rt, profile, cid,
537
- max_wait=max(0.0, deadline - _clock()), now=_clock)
538
- else:
539
- report_id, rows = _run_report(key, table, start, end, rt, profile, cid)
540
- except client.AdsReportUnavailable as exc:
541
- state = "not_offered"
542
- reason = _not_offered_reason(spec, exc)
543
- break
544
- except client.AdsError as exc:
545
- if (bounded and isinstance(exc, client.AdsReportPending) and exc.report_id
546
- and (stamp() - created).total_seconds() < client.REPORT_MAX_WAIT_SECONDS):
547
- _write_pending(con, table, cid, exc.report_id, start, end, created)
548
- if table_name(table) not in checkpointed:
549
- checkpointed.append(table_name(table))
550
- blocked = True
551
- state, reason = "deferred", PREPARING_REASON
552
- wait(label, spec, reason)
553
- break
554
- state = "failed"
555
- reason = str(exc)[:200]
556
- report["problems"].append(f"{label}: {spec['label']}: {reason}")
557
- break
558
- pulled += _land(con, table, rows, report_id, start, end, profile, cid, label)
559
- if table_name(table) not in checkpointed:
560
- checkpointed.append(table_name(table))
561
- if state == "done":
562
- _record_cadence(con, table, now, cid)
563
- if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
564
- report["partial"] = True
565
- if reason:
566
- _record_limit(con, table, reason, cid, state=state)
567
- shown = f"{label}: {reason}" if cid else reason
568
- report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
569
- elif state == "done":
570
- _clear_limit(con, table, cid)
571
- total = _count(con, table)
572
- entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
573
- if state != "skipped":
574
- amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid),
575
- "done" if state == "done" else "partial", _count(con, table, cid), state)
576
- previous = report["tables"].get(table, {})
577
- severity = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
578
- aggregate = max((state, previous.get("state", state)), key=severity.get)
579
- report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled,
580
- "in_mirror": total}
581
- if (pulled or origin_changed) and table_name(table) not in synced:
582
  synced.append(table_name(table))
583
- log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
584
- for table, state in report["tables"].items():
585
- if state["state"] == "skipped":
586
- continue
587
- amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
588
- "done" if state["state"] == "done" else "partial",
589
- state["in_mirror"], state["state"])
590
- if table in report["limits"]:
591
- _record_limit(con, table, report["limits"][table], state=state["state"])
592
- else:
593
- _clear_limit(con, table)
594
- report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
595
- for cid, *_ in connections for table in TABLES)
596
- if report["partial"]:
597
- report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
598
  if own and rt is not None and (synced or checkpointed):
599
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
600
  finally:
@@ -604,10 +606,75 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
604
  return report
605
 
606
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
607
  def _not_offered_reason(spec, exc):
608
  return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
609
- f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, "
610
- f"contact Amazon Ads support with the report type {spec['report_type']}.")
611
 
612
 
613
  def _persist(tenant_key, rt, con, synced, report, log):
 
10
  A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
11
  No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
12
 
13
+ ⛔ STANDING RULE 1. Every report is read whole, every file of it. The limits that cannot be removed
14
+ (one report at a time per client ID, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
15
  `amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
16
 
17
  Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
 
178
 
179
 
180
  def windows(con, table, connect_day, today, connection_id=None):
181
+ """The report windows one sync asks for, oldest first, each at most 120 days (`client.MAX_DAYS_PER_REQUEST`)."""
182
  table = schema.short_name(table)
183
  spec = schema.TABLE_SPECS[table]
184
  cadence = amazon_client.CADENCE["ads_" + table]
 
237
  cols = _columns_for(table)
238
  by_key = {r["_row_key"]: r for r in rows}
239
  keys = list(by_key)
240
+ one = f'({", ".join("?" for _ in cols)})'
241
  for i in range(0, len(keys), INSERT_BATCH_ROWS):
242
  part = keys[i:i + INSERT_BATCH_ROWS]
243
  con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
244
+ # One multi-row INSERT per batch: DuckDB's executemany is about 5 times slower at 24 months a pass.
245
+ con.execute(f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
246
+ f'VALUES {", ".join(one for _ in part)}', [by_key[k].get(c) for k in part for c in cols])
 
247
  return len(by_key)
248
 
249
 
 
389
 
390
 
391
  def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
392
+ """Run the group report that fills `table` under this connection's own credential. -> GroupReport."""
393
  options = {"rt": rt, "profile_id": profile_id, **extra}
394
  if connection_id:
395
  options["connection_id"] = connection_id
396
+ return client.run_group_report(key, schema.group_of(table), start, end, **options)
397
+
398
+
399
+ class _Pass:
400
+ """One connection's sync pass: its budget, clock, and what its group reports already covered.
401
+
402
+ `eligible` holds the tables a group report may land into this pass: requested, offered, due and
403
+ not yet finished. `covered` holds each table's spans landed this pass, `pulled` the rows a
404
+ sibling's report landed, `outcome` a group's failure, `notes` Amazon's text per table.
405
+ """
406
+
407
+ def __init__(self, con, key, rt, connection, report, *, today, used, deadline, stamp, checkpointed):
408
+ self.con, self.key, self.rt, self.report = con, key, rt, report
409
+ self.cid, self.label, self.profile = connection[:3]
410
+ self.today, self.used, self.deadline, self.stamp = today, used, deadline, stamp
411
+ self.checkpointed = checkpointed
412
+ self.eligible, self.covered, self.pulled, self.outcome, self.notes = set(), {}, {}, {}, {}
413
+ self.blocked = False
414
+
415
+ def wait(self, spec, reason):
416
+ self.report["waits"].append(f"{self.label}: {spec['label']}: {reason}")
417
+
418
+ def checkpoint(self, table):
419
+ if table_name(table) not in self.checkpointed:
420
+ self.checkpointed.append(table_name(table))
421
+
422
+ def covers(self, table, start, end):
423
+ """True when reports this pass already read every day from `start` to `end` for `table`."""
424
+ spans = self.covered.get(table, ())
425
+ return all(any(lo <= start + timedelta(days=i) <= hi for lo, hi in spans)
426
+ for i in range((end - start).days + 1))
427
+
428
+ def bounds(self):
429
+ """The poll limit for one new report: none, or the time left in this bounded pass."""
430
+ return {} if self.deadline is None else {"max_wait": max(0.0, self.deadline - _clock()), "now": _clock}
431
+
432
+ def fail(self, table, exc):
433
+ """Record a group report's failure for every member. -> (state, reason) for `table`."""
434
+ self.outcome[schema.group_of(table).name] = exc
435
+ return self.failure(table, exc)
436
+
437
+ def failure(self, table, exc):
438
+ spec = schema.TABLE_SPECS[table]
439
+ if isinstance(exc, client.AdsReportUnavailable):
440
+ return "not_offered", _not_offered_reason(spec, exc)
441
+ reason = str(exc)[:client.ERROR_TEXT_LIMIT]
442
+ self.report["problems"].append(f"{self.label}: {spec['label']}: {reason}")
443
+ return "failed", reason
444
+
445
+ def land(self, table, result, start, end):
446
+ """Land one group report into `table` and each eligible sibling, covering the span. -> rows into `table`."""
447
+ own = 0
448
+ for member in schema.group_of(table).members:
449
+ if member != table and member not in self.eligible:
450
+ continue
451
+ _ensure_table(self.con, member)
452
+ rows = _land(self.con, member, result.rows.get(member, []), result.report_id, start, end,
453
+ self.profile, self.cid, self.label)
454
+ self.covered.setdefault(member, []).append((start, end))
455
+ self.notes[member] = list(dict.fromkeys(self.notes.get(member, []) + list(result.notes)))
456
+ self.checkpoint(member)
457
+ if member == table:
458
+ own = rows
459
+ else:
460
+ self.pulled[member] = self.pulled.get(member, 0) + rows
461
+ for note in result.notes:
462
+ if f"{self.label}: {note}" not in self.report["problems"]:
463
+ self.report["problems"].append(f"{self.label}: {note}")
464
+ return own
465
+
466
+
467
+ def _resume(p):
468
+ """Read every report this connection left preparing before it creates another. -> {table: outcome}."""
469
+ resumed = {}
470
+ for table in TABLES:
471
+ pending = _read_pending(p.con, table, p.cid)
472
+ if pending is None:
473
+ continue
474
+ spec = schema.TABLE_SPECS[table]
475
+ _ensure_table(p.con, table)
476
+ p.checkpoint(table)
477
+ expired = f"Amazon Ads report {pending['report_id']} did not finish in 3 hours"
478
+ age = (p.stamp() - pending["created"]).total_seconds()
479
+ if age >= client.REPORT_MAX_WAIT_SECONDS:
480
+ _clear_pending(p.con, table, p.cid)
481
+ resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
482
+ continue
483
+ limit = client.REPORT_MAX_WAIT_SECONDS - age
484
+ if p.deadline is not None:
485
+ limit = min(limit, max(0.0, p.deadline - _clock()))
486
+ try:
487
+ result = _run_report(p.key, table, pending["start"], pending["end"], p.rt, p.profile, p.cid,
488
+ max_wait=limit, report_id=pending["report_id"], now=_clock)
489
+ except client.AdsReportPending:
490
+ if (p.stamp() - pending["created"]).total_seconds() < client.REPORT_MAX_WAIT_SECONDS:
491
+ p.blocked = True
492
+ p.wait(spec, PREPARING_REASON)
493
+ resumed[table] = ("deferred", PREPARING_REASON, 0)
494
+ continue
495
+ _clear_pending(p.con, table, p.cid)
496
+ resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
497
+ continue
498
+ except client.AdsError as exc:
499
+ _clear_pending(p.con, table, p.cid)
500
+ resumed[table] = (*p.fail(table, exc), 0)
501
+ continue
502
+ _clear_pending(p.con, table, p.cid)
503
+ resumed[table] = ("done", "", p.land(table, result, pending["start"], pending["end"]))
504
+ return resumed
505
+
506
+
507
+ def _run_windows(p, table, connect_day):
508
+ """Run the group report for each window of `table` no report covered yet. -> (state, reason, rows)."""
509
+ spec = schema.TABLE_SPECS[table]
510
+ failed = p.outcome.get(schema.group_of(table).name)
511
+ if failed is not None:
512
+ return (*p.failure(table, failed), 0)
513
+ pulled = 0
514
+ for start, end in windows(p.con, table, connect_day, p.today, p.cid):
515
+ if p.covers(table, start, end):
516
+ continue
517
+ if p.used >= client.REPORTS_PER_DAY:
518
+ reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
519
+ f"reports are used, so days from {start.isoformat()} load on the next sync.")
520
+ p.report["problems"].append(f"{p.label}: {spec['label']}: waiting for tomorrow's report limit")
521
+ p.report["waits"].append(p.report["problems"][-1])
522
+ return "deferred", reason, pulled
523
+ if p.deadline is not None and _clock() >= p.deadline:
524
+ p.wait(spec, TIME_LIMIT_REASON)
525
+ return "deferred", TIME_LIMIT_REASON, pulled
526
+ p.used += 1
527
+ _spend_budget(p.con, p.today, p.used, p.cid)
528
+ created = p.stamp()
529
+ try:
530
+ result = _run_report(p.key, table, start, end, p.rt, p.profile, p.cid, **p.bounds())
531
+ except client.AdsReportPending as exc:
532
+ if (p.deadline is None or not exc.report_id
533
+ or (p.stamp() - created).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS):
534
+ return (*p.failure(table, exc), pulled)
535
+ _write_pending(p.con, table, p.cid, exc.report_id, start, end, created)
536
+ p.checkpoint(table)
537
+ p.blocked = True
538
+ p.wait(spec, PREPARING_REASON)
539
+ return "deferred", PREPARING_REASON, pulled
540
+ except client.AdsError as exc:
541
+ return (*p.fail(table, exc), pulled)
542
+ pulled += p.land(table, result, start, end)
543
+ return "done", "", pulled
544
 
545
 
546
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
 
551
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
552
  `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
553
  pending ledger, and the next pass reads it before it creates any other report.
554
+ One group report fills every requested member table, so a sibling's covered days spend no report.
555
  """
556
  key = amazon_store._tenant_key(tenant_key)
557
  report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
 
568
  own = con is None
569
  if own:
570
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
 
571
  started = _clock()
572
+ deadline = None if max_seconds is None else started + float(max_seconds)
573
+ synced, checkpointed = [], []
574
  try:
575
  datastore._ensure_sync_state(con)
576
  now = _utc_now(now)
 
579
  def stamp():
580
  return now + timedelta(seconds=max(0.0, _clock() - started))
581
 
582
+ requested = [schema.short_name(t) for t in (tables or TABLES)]
583
+ for connection in connections:
584
+ cid, label, _profile, legacy = connection
 
 
 
585
  connect_day = _connect_day(con, today, cid, legacy=legacy)
586
  report.setdefault("connectDay", connect_day.isoformat())
587
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
588
  report["connections"].append(entry)
589
+ p = _Pass(con, key, rt, connection, report, today=today, used=_budget(con, today, cid, legacy=legacy),
590
+ deadline=deadline, stamp=stamp, checkpointed=checkpointed)
591
+ p.eligible = {t for t in requested if schema.TABLE_SPECS[t]["offered"] and _cadence_due(con, t, now, cid)}
592
+ resumed = _resume(p)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
593
  for table in list(dict.fromkeys(requested + list(resumed))):
594
+ landed = _sync_table(p, table, entry, connect_day, now, legacy, table in requested,
595
+ resumed.get(table), log)
596
+ p.eligible.discard(table)
597
+ if landed and table_name(table) not in synced:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
598
  synced.append(table_name(table))
599
+ _record_tables(con, report, connections)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
600
  if own and rt is not None and (synced or checkpointed):
601
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
602
  finally:
 
606
  return report
607
 
608
 
609
+ _SEVERITY = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
610
+
611
+
612
+ def _sync_table(p, table, entry, connect_day, now, legacy, requested, prior, log):
613
+ """One table's turn in one connection's pass: run or skip it, record its state. -> True if it changed."""
614
+ con, cid, label, report = p.con, p.cid, p.label, p.report
615
+ spec = schema.TABLE_SPECS[table]
616
+ _ensure_table(con, table)
617
+ origin_changed = _refresh_origin(con, table, cid, label, legacy)
618
+ state, reason, pulled = prior or ("done", "", 0)
619
+ pulled += p.pulled.pop(table, 0)
620
+ if state != "done" or not requested:
621
+ pass
622
+ elif not spec["offered"]:
623
+ state, reason = "not_offered", spec["not_offered_reason"]
624
+ elif not _cadence_due(con, table, now, cid):
625
+ state = "skipped"
626
+ elif p.blocked:
627
+ state, reason = "deferred", PREPARING_REASON
628
+ p.wait(spec, reason)
629
+ else:
630
+ state, reason, rows = _run_windows(p, table, connect_day)
631
+ pulled += rows
632
+ if state == "done":
633
+ _record_cadence(con, table, now, cid)
634
+ if state == "done" and not reason and p.notes.get(table):
635
+ reason = " ".join(p.notes[table])
636
+ if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
637
+ report["partial"] = True
638
+ if reason:
639
+ _record_limit(con, table, reason, cid, state=state)
640
+ shown = f"{label}: {reason}" if cid else reason
641
+ report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
642
+ elif state == "done":
643
+ _clear_limit(con, table, cid)
644
+ total = _count(con, table)
645
+ entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
646
+ if state != "skipped":
647
+ amazon_store._record_state(con, _state_key(amazon_store.entity_key(_marker_table(table)), cid),
648
+ "done" if state == "done" else "partial", _count(con, table, cid), state)
649
+ previous = report["tables"].get(table, {})
650
+ report["tables"][table] = {"state": max((state, previous.get("state", state)), key=_SEVERITY.get),
651
+ "pulled": previous.get("pulled", 0) + pulled, "in_mirror": total}
652
+ log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
653
+ return bool(pulled or origin_changed)
654
+
655
+
656
+ def _record_tables(con, report, connections):
657
+ """Write each table's aggregate state and limit across connections, and the pass's pending count."""
658
+ for table, state in report["tables"].items():
659
+ if state["state"] == "skipped":
660
+ continue
661
+ amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
662
+ "done" if state["state"] == "done" else "partial",
663
+ state["in_mirror"], state["state"])
664
+ if table in report["limits"]:
665
+ _record_limit(con, table, report["limits"][table], state=state["state"])
666
+ else:
667
+ _clear_limit(con, table)
668
+ report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
669
+ for cid, *_ in connections for table in TABLES)
670
+ if report["partial"]:
671
+ report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
672
+
673
+
674
  def _not_offered_reason(spec, exc):
675
  return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
676
+ f"({str(exc)[:client.ERROR_TEXT_LIMIT]}). The table keeps the days it has. To ask for it, "
677
+ f"contact Amazon Ads support and quote Amazon's message.")
678
 
679
 
680
  def _persist(tenant_key, rt, con, synced, report, log):
platform/harness/amazon_client.py CHANGED
@@ -1127,7 +1127,7 @@ for _tables, _passes, _daily, _intraday in (
1127
  from harness import amazon_ads_schema as _ads_schema
1128
 
1129
  CADENCE.update({
1130
- "ads_" + table: {"perDay": 4, "lookbackDays": 21, "intradayLookbackDays": 5, "period": "day"}
1131
  for table in _ads_schema.TABLE_SPECS
1132
  })
1133
  del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
 
1127
  from harness import amazon_ads_schema as _ads_schema
1128
 
1129
  CADENCE.update({
1130
+ "ads_" + table: {"perDay": 4, "lookbackDays": 60, "intradayLookbackDays": 5, "period": "day"}
1131
  for table in _ads_schema.TABLE_SPECS
1132
  })
1133
  del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
platform/harness/canonical_source.py CHANGED
@@ -6,6 +6,7 @@ import logging
6
  import re
7
  import uuid
8
  from contextlib import contextmanager
 
9
 
10
  from core import odoo as _odoo
11
 
@@ -53,6 +54,17 @@ class Rows:
53
  return bool(len(self))
54
 
55
 
 
 
 
 
 
 
 
 
 
 
 
56
  class Source:
57
  """Own bounded source operands and release every temporary snapshot on exit."""
58
 
@@ -119,6 +131,30 @@ class Source:
119
  ['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
120
  return self.cache[key]
121
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  def sku_revenue(self, groups):
123
  products = self.products()
124
  code = code_sql('p')
@@ -171,7 +207,7 @@ class Source:
171
 
172
  def categories(self, keep_blank_code=False):
173
  rows = self.products()
174
- cats = self.search_read('product.category', [], ['id', 'complete_name'])
175
  blank = '' if keep_blank_code else "AND json_extract_string(p.body, '/default_code') != ''"
176
  return {code: category for code, category in self.cur.execute(f"""SELECT
177
  {code_sql('p')} code,
@@ -206,7 +242,7 @@ class Source:
206
  ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
207
  products = self.search_read('product.product', [('default_code', '!=', False)],
208
  ['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
209
- cats = self.search_read('product.category', [], ['id', 'complete_name'])
210
  query = f"""WITH raw AS (SELECT p.ordinal,
211
  trim(json_extract_string(p.body, '/default_code')) sku,
212
  json_extract_string(p.body, '/name') AS product_name,
@@ -231,17 +267,18 @@ class Source:
231
  def tiers(self, prods):
232
  from core import periods
233
  from modules import products
234
- lists = self.search_read('product.pricelist', [], ['id', 'name'])
235
  today = periods.today().isoformat()
236
- rules = self.search_read('product.pricelist.item', [('compute_price', '=', 'fixed'),
237
- ('applied_on', 'in', ['0_product_variant', '1_product']),
238
- '|', ('date_start', '=', False), ('date_start', '<=', today),
239
- '|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)],
240
  ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
241
- 'fixed_price', 'packaging_id', 'packaging_price'])
 
 
 
242
  normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
243
  canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
244
- prods = products.active_products() if prods is None else prods
245
  declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
246
  # Two equality joins and an anti-join: an OR join with a correlated NOT EXISTS is a nested loop
247
  # (234 s for the whole catalogue on the staging mirror).
@@ -250,7 +287,7 @@ class Source:
250
  json_extract_string(p.body, '/product_tmpl_id/0') tmpl
251
  FROM "{prods.table}" p),
252
  rr AS (
253
- SELECT r.ordinal rule_order, json_extract_string(r.body, '/applied_on') applied_on,
254
  json_extract_string(r.body, '/product_id/0') rpid,
255
  json_extract_string(r.body, '/product_tmpl_id/0') rtmpl,
256
  json_extract_string(r.body, '/pricelist_id/0') plid,
@@ -289,10 +326,11 @@ class Source:
289
  entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
290
  if name in declared and n > products.PRICELIST_TIER_DEPTH:
291
  entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
292
- return out, {'bounded': True,
293
- 'rules_total': self.search_count('product.pricelist.item', []),
294
- 'rules_not_fixed': self.search_count('product.pricelist.item',
295
- [('compute_price', '!=', 'fixed')])}
 
296
 
297
  def search_read(self, model, domain=None, fields=None, **kw):
298
  domain = self.domain(model, domain)
@@ -615,6 +653,9 @@ class _Query:
615
  expr = expr[0]
616
  if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
617
  raise _Unmirrored(f'name search on {path}')
 
 
 
618
  if kind == 'bool':
619
  expr = f'coalesce({expr}, false)'
620
  value = [_plain(v) for v in value] if isinstance(value, (list, tuple, set)) else _plain(value)
@@ -721,6 +762,11 @@ class MirrorSource(Source):
721
  """The FROM operand for `model`: its mirror table, or an empty typed one while unsynced."""
722
  if model not in self._relations:
723
  target = _MIRROR_MODELS[model]
 
 
 
 
 
724
  if not target.startswith('(') and not self.cur.execute(
725
  'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1',
726
  [target]).fetchone():
@@ -750,6 +796,24 @@ class MirrorSource(Source):
750
  # Live `_code_category` keys every `default_code != False` product, a blank '' code included.
751
  return Source.categories(self, keep_blank_code)
752
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
753
  def servable(self, model, names):
754
  """The `names` the mirror declares for `model`; the rest are logged once and dropped."""
755
  kept, dropped = [], []
@@ -922,9 +986,10 @@ class MirrorSource(Source):
922
  coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
923
  FROM {prods.from_sql()} WHERE {pw}),
924
  quant AS (SELECT b.product_id id, sum(coalesce(b.quantity, 0)) qty
925
- FROM {quants.from_sql()} WHERE {qw} GROUP BY 1),
926
  sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
927
- sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw} GROUP BY 1)
 
928
  SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
929
  bool_or(p.storable) storable, sum(coalesce(q.qty, 0)) on_hand,
930
  sum(coalesce(q.qty, 0) * p.cost) inv_value, sum(coalesce(s.qty, 0) * p.cost) cogs_ltm,
@@ -962,6 +1027,7 @@ class MirrorSource(Source):
962
  "SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
963
  "JOIN stock_location l ON l.id = q.location_id "
964
  "WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
 
965
  "GROUP BY 1) f ON f.product_id = p.id "
966
  "WHERE p.id IN (SELECT unnest($2::BIGINT[]))",
967
  [view[1], ids])
 
6
  import re
7
  import uuid
8
  from contextlib import contextmanager
9
+ from dataclasses import dataclass
10
 
11
  from core import odoo as _odoo
12
 
 
54
  return bool(len(self))
55
 
56
 
57
+ @dataclass(frozen=True)
58
+ class RowIds:
59
+ """A SQL-backed selection of related identities from one source snapshot."""
60
+ rows: Rows
61
+ path: str
62
+
63
+ def sql(self):
64
+ return (f'SELECT DISTINCT try_cast(json_extract_string(body, \'{self.path}\') AS BIGINT) id '
65
+ f'FROM "{self.rows.table}" WHERE id IS NOT NULL')
66
+
67
+
68
  class Source:
69
  """Own bounded source operands and release every temporary snapshot on exit."""
70
 
 
131
  ['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
132
  return self.cache[key]
133
 
134
+ def related_read(self, model, selections, fields, domain=()):
135
+ """Page related source reads by selected identities without retaining the population in Python."""
136
+ def records():
137
+ for field, ids, extra in selections:
138
+ last = -1
139
+ while True:
140
+ page = [row[0] for row in self.cur.execute(
141
+ f'SELECT id FROM ({ids.sql()}) selected WHERE id > ? ORDER BY id LIMIT ?',
142
+ [last, BATCH]).fetchall()]
143
+ if not page:
144
+ break
145
+ selected = self.search_read(model, [*domain, *extra, (field, 'in', page)], fields)
146
+ try:
147
+ yield from selected
148
+ finally:
149
+ self.cur.execute(f'DROP TABLE "{selected.table}"')
150
+ self.tables.remove(selected.table)
151
+ last = page[-1]
152
+ return self.snapshot(records())
153
+
154
+ def category_rows(self, products):
155
+ return self.related_read('product.category', [('id', RowIds(products, '/categ_id/0'), [])],
156
+ ['id', 'complete_name'])
157
+
158
  def sku_revenue(self, groups):
159
  products = self.products()
160
  code = code_sql('p')
 
207
 
208
  def categories(self, keep_blank_code=False):
209
  rows = self.products()
210
+ cats = self.category_rows(rows)
211
  blank = '' if keep_blank_code else "AND json_extract_string(p.body, '/default_code') != ''"
212
  return {code: category for code, category in self.cur.execute(f"""SELECT
213
  {code_sql('p')} code,
 
242
  ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
243
  products = self.search_read('product.product', [('default_code', '!=', False)],
244
  ['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
245
+ cats = self.category_rows(products)
246
  query = f"""WITH raw AS (SELECT p.ordinal,
247
  trim(json_extract_string(p.body, '/default_code')) sku,
248
  json_extract_string(p.body, '/name') AS product_name,
 
267
  def tiers(self, prods):
268
  from core import periods
269
  from modules import products
270
+ prods = products.active_products() if prods is None else prods
271
  today = periods.today().isoformat()
272
+ selections = [('product_id', RowIds(prods, '/id'), [('applied_on', '=', '0_product_variant')]),
273
+ ('product_tmpl_id', RowIds(prods, '/product_tmpl_id/0'), [('applied_on', '=', '1_product')])]
274
+ rules = self.related_read('product.pricelist.item', selections,
 
275
  ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
276
+ 'fixed_price', 'packaging_id', 'packaging_price'], [('compute_price', '=', 'fixed'),
277
+ '|', ('date_start', '=', False), ('date_start', '<=', today),
278
+ '|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)])
279
+ lists = self.related_read('product.pricelist', [('id', RowIds(rules, '/pricelist_id/0'), [])], ['id', 'name'])
280
  normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
281
  canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
 
282
  declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
283
  # Two equality joins and an anti-join: an OR join with a correlated NOT EXISTS is a nested loop
284
  # (234 s for the whole catalogue on the staging mirror).
 
287
  json_extract_string(p.body, '/product_tmpl_id/0') tmpl
288
  FROM "{prods.table}" p),
289
  rr AS (
290
+ SELECT try_cast(json_extract_string(r.body, '/id') AS BIGINT) rule_order, json_extract_string(r.body, '/applied_on') applied_on,
291
  json_extract_string(r.body, '/product_id/0') rpid,
292
  json_extract_string(r.body, '/product_tmpl_id/0') rtmpl,
293
  json_extract_string(r.body, '/pricelist_id/0') plid,
 
326
  entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
327
  if name in declared and n > products.PRICELIST_TIER_DEPTH:
328
  entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
329
+ return out, self.tier_report()
330
+
331
+ def tier_report(self):
332
+ return {'bounded': True, 'rules_total': self.search_count('product.pricelist.item', []),
333
+ 'rules_not_fixed': self.search_count('product.pricelist.item', [('compute_price', '!=', 'fixed')])}
334
 
335
  def search_read(self, model, domain=None, fields=None, **kw):
336
  domain = self.domain(model, domain)
 
653
  expr = expr[0]
654
  if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
655
  raise _Unmirrored(f'name search on {path}')
656
+ if op == 'in' and isinstance(value, RowIds):
657
+ cond = f'{expr} IN ({value.sql()})'
658
+ return f'coalesce(({guard} AND {cond}), false)' if guard else f'coalesce(({cond}), false)'
659
  if kind == 'bool':
660
  expr = f'coalesce({expr}, false)'
661
  value = [_plain(v) for v in value] if isinstance(value, (list, tuple, set)) else _plain(value)
 
762
  """The FROM operand for `model`: its mirror table, or an empty typed one while unsynced."""
763
  if model not in self._relations:
764
  target = _MIRROR_MODELS[model]
765
+ if model == 'product.category' and self.kind == 'product':
766
+ products = self.products()
767
+ target = ("(SELECT DISTINCT try_cast(json_extract_string(body, '/categ_id/0') AS BIGINT) AS id, "
768
+ "json_extract_string(body, '/categ_id/1') AS complete_name "
769
+ f'FROM "{products.table}" WHERE id IS NOT NULL)')
770
  if not target.startswith('(') and not self.cur.execute(
771
  'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1',
772
  [target]).fetchone():
 
796
  # Live `_code_category` keys every `default_code != False` product, a blank '' code included.
797
  return Source.categories(self, keep_blank_code)
798
 
799
+ def related_read(self, model, selections, fields, domain=()):
800
+ branches = [['&'] * len(extra) + [*extra, (field, 'in', ids)]
801
+ for field, ids, extra in selections]
802
+ scoped = ['|'] * (len(branches) - 1) + [term for branch in branches for term in branch]
803
+ return self.search_read(model, [*domain, *scoped], fields)
804
+
805
+ def tier_report(self):
806
+ owner = self.parent or self
807
+ if 'tier_report' not in owner.cache:
808
+ query = _Query(self, 'product.pricelist.item')
809
+ price = query.field('compute_price')[0]
810
+ where = query.where([])
811
+ total, other = self.cur.execute(
812
+ f"SELECT count(*), count(*) FILTER (WHERE {price} IS DISTINCT FROM 'fixed') "
813
+ f'FROM {query.from_sql()} WHERE {where}', query.params).fetchone()
814
+ owner.cache['tier_report'] = {'bounded': True, 'rules_total': total, 'rules_not_fixed': other}
815
+ return dict(owner.cache['tier_report'])
816
+
817
  def servable(self, model, names):
818
  """The `names` the mirror declares for `model`; the rest are logged once and dropped."""
819
  kept, dropped = [], []
 
986
  coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
987
  FROM {prods.from_sql()} WHERE {pw}),
988
  quant AS (SELECT b.product_id id, sum(coalesce(b.quantity, 0)) qty
989
+ FROM {quants.from_sql()} WHERE {qw} AND b.product_id IN (SELECT id FROM prods) GROUP BY 1),
990
  sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
991
+ sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw}
992
+ AND b.product_id IN (SELECT id FROM prods) GROUP BY 1)
993
  SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
994
  bool_or(p.storable) storable, sum(coalesce(q.qty, 0)) on_hand,
995
  sum(coalesce(q.qty, 0) * p.cost) inv_value, sum(coalesce(s.qty, 0) * p.cost) cogs_ltm,
 
1027
  "SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
1028
  "JOIN stock_location l ON l.id = q.location_id "
1029
  "WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
1030
+ "AND q.product_id IN (SELECT unnest($2::BIGINT[])) "
1031
  "GROUP BY 1) f ON f.product_id = p.id "
1032
  "WHERE p.id IN (SELECT unnest($2::BIGINT[]))",
1033
  [view[1], ids])
platform/modules/competitor_watch.py CHANGED
@@ -26,6 +26,8 @@ import re
26
  from pathlib import Path
27
  from urllib.parse import unquote_plus
28
 
 
 
29
  TRIGGERS = {
30
  "new_asin": "New ASIN in top 3",
31
  "lost_spot": "Lost spot",
@@ -438,14 +440,6 @@ def _columns(cur, table):
438
  raise _ReadFailed(type(exc).__name__) from exc
439
 
440
 
441
- def _weekly_period_sql():
442
- """Only finite dates or timestamps with an inclusive seven-day span define a report week."""
443
- start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
444
- end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
445
- return (f"isfinite({start}) AND isfinite({end}) AND "
446
- f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) = 6")
447
-
448
-
449
  def _ba_columns(cur):
450
  """Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
451
  cols = _columns(cur, BA_TABLE) if cur is not None else set()
 
26
  from pathlib import Path
27
  from urllib.parse import unquote_plus
28
 
29
+ from core.amazon_rivals import weekly_period_sql as _weekly_period_sql
30
+
31
  TRIGGERS = {
32
  "new_asin": "New ASIN in top 3",
33
  "lost_spot": "Lost spot",
 
440
  raise _ReadFailed(type(exc).__name__) from exc
441
 
442
 
 
 
 
 
 
 
 
 
443
  def _ba_columns(cur):
444
  """Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
445
  cols = _columns(cur, BA_TABLE) if cur is not None else set()
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 46d43dc743596d3d8e1ac82e1144c0e3a4aaf1788b17dac4cae5e886e014a769
 
1
+ 8c86af445a651c7ecbd3085c9d8b5afa5671a13f052ac92abb0ba0d5a7811991