fsanyoto commited on
Commit
07a68d0
·
verified ·
1 Parent(s): 0645c1f

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. Dockerfile +9 -1
  2. RELEASES.json +20 -458
  3. VERSION +1 -1
  4. api/agent_runtime.py +14 -2
  5. api/amazon_relational.py +34 -9
  6. api/app_routes.py +70 -0
  7. api/app_rules.py +1 -2
  8. api/app_templates.py +205 -0
  9. api/application_registry.py +331 -109
  10. api/assistant_stream.py +19 -24
  11. api/catalog_store.py +33 -5
  12. api/connectors_amazon_ads.py +130 -0
  13. api/deps.py +57 -0
  14. api/error_sink.py +151 -0
  15. api/external_activity.py +33 -7
  16. api/external_freshness.py +17 -4
  17. api/external_openapi.py +2 -2
  18. api/loopy_catalog_tools.py +24 -4
  19. api/main.py +214 -34
  20. api/odoo_relational.py +225 -77
  21. api/process_role.py +357 -11
  22. api/providers.py +98 -2
  23. api/registry_source.py +96 -21
  24. api/request_class.py +456 -53
  25. api/request_timing.py +26 -4
  26. api/routes_account_health.py +142 -0
  27. api/routes_admin.py +174 -1
  28. api/routes_aged_stock.py +167 -0
  29. api/routes_amazon_operator.py +133 -0
  30. api/routes_assets.py +24 -9
  31. api/routes_automation.py +5 -3
  32. api/routes_catalogs.py +17 -10
  33. api/routes_competitor_watch.py +9 -6
  34. api/routes_connectors.py +49 -10
  35. api/routes_customers.py +16 -0
  36. api/routes_external.py +142 -8
  37. api/routes_keychain.py +1 -1
  38. api/routes_nav.py +92 -13
  39. api/routes_odoo_attachments.py +38 -4
  40. api/routes_opportunity_finder.py +169 -0
  41. api/routes_payouts.py +228 -0
  42. api/routes_products.py +18 -20
  43. api/routes_query.py +2 -0
  44. api/routes_ranking_diagnostic.py +158 -0
  45. api/routes_reimbursements.py +118 -0
  46. api/routes_restock_planner.py +156 -0
  47. api/routes_returns_repeat.py +159 -0
  48. api/routes_shares.py +106 -0
  49. api/routes_sheet.py +471 -93
  50. api/routes_starred.py +12 -3
Dockerfile CHANGED
@@ -120,4 +120,12 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
120
  # nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other
121
  # proxies in this class use, so it is the safest available floor without over-fitting to a number
122
  # we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise.
123
- CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
 
 
 
 
 
 
 
 
 
120
  # nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other
121
  # proxies in this class use, so it is the safest available floor without over-fitting to a number
122
  # we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise.
123
+ #
124
+ # S79-T37 (R25): the launcher sizes the web tier to the container. `request_class.py --env` reads
125
+ # the cgroup CPU and memory limits (never os.cpu_count(), which reports the host) and exports the
126
+ # uvicorn worker count, heavy concurrency and DuckDB caps. An env value set on the Space wins.
127
+ # Several workers start only with AIOS_MIRROR_VERSIONS=1, which opens the mirror read-only in each
128
+ # of them, and the shell holds W at 1 without it. 2 vCPU keeps one worker and today's caps.
129
+ # Several workers load the app through a factory: uvicorn imports a plain app string in its
130
+ # supervisor too, and that process would run the boot threads without serving.
131
+ CMD ["sh", "-c", "eval \"$(python api/request_class.py --env)\"; W=1; if [ \"${AIOS_MIRROR_VERSIONS:-0}\" = 1 ]; then W=\"${AIOS_UVICORN_WORKERS:-1}\"; fi; export AIOS_UVICORN_WORKERS=\"$W\"; APP=api.main:app; if [ \"$W\" -gt 1 ]; then APP=\"--factory api.request_class:web_app\"; fi; exec uvicorn $APP --host 0.0.0.0 --port ${PORT} --workers \"$W\" --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
RELEASES.json CHANGED
@@ -1,497 +1,59 @@
1
  {
2
- "current": "f75ce5527c (f75ce5527c67)",
3
  "releases": [
 
 
 
 
 
 
 
 
 
 
 
 
4
  {
5
  "version": "v83",
6
- "sha": "9b7471968e79",
7
  "date": "2026-10-21",
8
  "subject": "release v83"
9
  },
10
  {
11
  "version": "v82",
12
- "sha": "9c61b1ee94d0",
13
  "date": "2026-10-17",
14
  "subject": "release v82"
15
  },
16
  {
17
  "version": "v81",
18
- "sha": "e141ccc85ef0",
19
  "date": "2026-10-15",
20
  "subject": "release v81"
21
  },
22
  {
23
  "version": "v80",
24
- "sha": "869b8fcc62ac",
25
  "date": "2026-10-13",
26
  "subject": "release v80"
27
  },
28
  {
29
  "version": "v79",
30
- "sha": "406417b8ded0",
31
  "date": "2026-10-13",
32
  "subject": "release v79"
33
  },
34
  {
35
  "version": "v78",
36
- "sha": "aad6561d63e6",
37
  "date": "2026-10-09",
38
  "subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs"
39
  },
40
  {
41
  "version": "v77",
42
- "sha": "e741e5ca604d",
43
  "date": "2026-10-05",
44
  "subject": "release v77"
45
- },
46
- {
47
- "version": "v76",
48
- "sha": "7990f03e0254",
49
- "date": "2026-10-01",
50
- "subject": "release v76"
51
- },
52
- {
53
- "version": "v75",
54
- "sha": "2c5146981ccf",
55
- "date": "2026-09-19",
56
- "subject": "release v75"
57
- },
58
- {
59
- "version": "v74",
60
- "sha": "40dceba9262e",
61
- "date": "2026-09-14",
62
- "subject": "release v74: waves 58+59 incl. the Amazon credential door (W59-T21/22/23) and sync producer (W59-T42); hotfix promote so"
63
- },
64
- {
65
- "version": "v73",
66
- "sha": "bb94de5665e6",
67
- "date": "2026-09-13",
68
- "subject": "release v73"
69
- },
70
- {
71
- "version": "v71",
72
- "sha": "96143dc6fb65",
73
- "date": "2026-09-04",
74
- "subject": "release v71 - wave 51, QA'd 24 PASS / 6 FAIL / 5 UNPROVEN"
75
- },
76
- {
77
- "version": "v70",
78
- "sha": "eedfb6a00163",
79
- "date": "2026-09-03",
80
- "subject": "release v70 - wave 50 (grid analytics, the Sheet as a spreadsheet, view trash and audit)"
81
- },
82
- {
83
- "version": "v69",
84
- "sha": "8598f56293a0",
85
- "date": "2026-09-02",
86
- "subject": "release v69: wave48-continuation validated + hotfixes v67/v68/d4e3e31 + go-live QA repairs (Supplier option limit, optio"
87
- },
88
- {
89
- "version": "v68",
90
- "sha": "62b065a2610a",
91
- "date": "2026-09-01",
92
- "subject": "release v68: unified holiday date offsets"
93
- },
94
- {
95
- "version": "v67",
96
- "sha": "3b2241982cf7",
97
- "date": "2026-09-01",
98
- "subject": "release v67"
99
- },
100
- {
101
- "version": "v66",
102
- "sha": "73c94f076740",
103
- "date": "2026-08-31",
104
- "subject": "release v66"
105
- },
106
- {
107
- "version": "v65",
108
- "sha": "9f6cfedb8a43",
109
- "date": "2026-08-31",
110
- "subject": "release v65"
111
- },
112
- {
113
- "version": "v64",
114
- "sha": "bb8985c6e73e",
115
- "date": "2026-08-31",
116
- "subject": "release v64"
117
- },
118
- {
119
- "version": "v63",
120
- "sha": "ec75cb8f3412",
121
- "date": "2026-08-31",
122
- "subject": "release v63"
123
- },
124
- {
125
- "version": "v62",
126
- "sha": "2c8f702c964f",
127
- "date": "2026-08-31",
128
- "subject": "release v62"
129
- },
130
- {
131
- "version": "v61",
132
- "sha": "bbc4c0760ab5",
133
- "date": "2026-08-30",
134
- "subject": "v61 - wave 46 hotfix: D-590 schema overlay reaches the grid read (all three assemblies), D-599 route to zone association"
135
- },
136
- {
137
- "version": "v60",
138
- "sha": "dd1b0620210e",
139
- "date": "2026-08-30",
140
- "subject": "v60 - wave 46"
141
- },
142
- {
143
- "version": "v59",
144
- "sha": "3cb73ce8021e",
145
- "date": "2026-08-29",
146
- "subject": "release v59"
147
- },
148
- {
149
- "version": "v58",
150
- "sha": "c526cede82f9",
151
- "date": "2026-08-29",
152
- "subject": "release v58"
153
- },
154
- {
155
- "version": "v57",
156
- "sha": "4859c035e6a7",
157
- "date": "2026-08-29",
158
- "subject": "release v57"
159
- },
160
- {
161
- "version": "v56",
162
- "sha": "2970b729e8f0",
163
- "date": "2026-08-28",
164
- "subject": "release v56"
165
- },
166
- {
167
- "version": "v55",
168
- "sha": "b766b904b8fa",
169
- "date": "2026-08-28",
170
- "subject": "release v55"
171
- },
172
- {
173
- "version": "v54",
174
- "sha": "6a8e99a064d9",
175
- "date": "2026-08-27",
176
- "subject": "release v54"
177
- },
178
- {
179
- "version": "v53",
180
- "sha": "89ed2cb867da",
181
- "date": "2026-08-24",
182
- "subject": "release v53"
183
- },
184
- {
185
- "version": "v52",
186
- "sha": "826349376241",
187
- "date": "2026-08-23",
188
- "subject": "release v52"
189
- },
190
- {
191
- "version": "v51",
192
- "sha": "5740d20fa585",
193
- "date": "2026-08-23",
194
- "subject": "release v51"
195
- },
196
- {
197
- "version": "v50",
198
- "sha": "8027c83d1da6",
199
- "date": "2026-08-23",
200
- "subject": "release v50"
201
- },
202
- {
203
- "version": "v49",
204
- "sha": "9a14c01a48a4",
205
- "date": "2026-08-23",
206
- "subject": "release v49"
207
- },
208
- {
209
- "version": "v48",
210
- "sha": "3c69fd93da7c",
211
- "date": "2026-08-23",
212
- "subject": "release v48"
213
- },
214
- {
215
- "version": "v47",
216
- "sha": "917f9a0eb509",
217
- "date": "2026-08-23",
218
- "subject": "release v47"
219
- },
220
- {
221
- "version": "v46",
222
- "sha": "a8dc531b5078",
223
- "date": "2026-08-23",
224
- "subject": "release v46"
225
- },
226
- {
227
- "version": "v45",
228
- "sha": "0b2b5010d86f",
229
- "date": "2026-08-23",
230
- "subject": "release v45"
231
- },
232
- {
233
- "version": "v44",
234
- "sha": "a7877aaf20ff",
235
- "date": "2026-08-23",
236
- "subject": "Hotfix: allow renaming the active Map view"
237
- },
238
- {
239
- "version": "v43",
240
- "sha": "e2ba4616fa0f",
241
- "date": "2026-08-23",
242
- "subject": "release v43"
243
- },
244
- {
245
- "version": "v42",
246
- "sha": "25b885e6373a",
247
- "date": "2026-08-23",
248
- "subject": "release v42"
249
- },
250
- {
251
- "version": "v41",
252
- "sha": "1d6adcc3c3ac",
253
- "date": "2026-08-22",
254
- "subject": "v41 vertical route navigation"
255
- },
256
- {
257
- "version": "v40",
258
- "sha": "1451837168f3",
259
- "date": "2026-08-22",
260
- "subject": "release v40 route UI correction"
261
- },
262
- {
263
- "version": "v39",
264
- "sha": "a78bdf00e38e",
265
- "date": "2026-08-21",
266
- "subject": "W39 clean stale migration targets"
267
- },
268
- {
269
- "version": "v38",
270
- "sha": "a556631cf979",
271
- "date": "2026-08-21",
272
- "subject": "W39 reversible Live store cutover"
273
- },
274
- {
275
- "version": "v37",
276
- "sha": "1ddd8cf82379",
277
- "date": "2026-08-21",
278
- "subject": "W39: coalesce derived workspace reads"
279
- },
280
- {
281
- "version": "v36",
282
- "sha": "51db3b7fee2c",
283
- "date": "2026-08-21",
284
- "subject": "W39 share-field oid repair"
285
- },
286
- {
287
- "version": "v35",
288
- "sha": "0f642129c8f7",
289
- "date": "2026-08-21",
290
- "subject": "W39 visual QA fixture"
291
- },
292
- {
293
- "version": "v34",
294
- "sha": "cb4caa188e05",
295
- "date": "2026-08-21",
296
- "subject": "W39 Staging PostgreSQL egress controls"
297
- },
298
- {
299
- "version": "v33",
300
- "sha": "2b115e573d6a",
301
- "date": "2026-08-21",
302
- "subject": "W39: stop nonproduction Postgres background egress"
303
- },
304
- {
305
- "version": "v32",
306
- "sha": "b47b27ca7b2a",
307
- "date": "2026-08-21",
308
- "subject": "W39: repair Staging Postgres projection"
309
- },
310
- {
311
- "version": "v31",
312
- "sha": "8880a49a244e",
313
- "date": "2026-08-21",
314
- "subject": "v31: Wave 39 staging candidate"
315
- },
316
- {
317
- "version": "v30",
318
- "sha": "71776ca6c8df",
319
- "date": "2026-08-20",
320
- "subject": "v30: W39-T29 Member-owned View first-share hotfix"
321
- },
322
- {
323
- "version": "v29",
324
- "sha": "55ced5018ff2",
325
- "date": "2026-08-18",
326
- "subject": "v29: the build staging has been running since 2026-08-18 (55ced50)."
327
- },
328
- {
329
- "version": "v28",
330
- "sha": "2793e4ba53dc",
331
- "date": "2026-08-18",
332
- "subject": "release v28"
333
- },
334
- {
335
- "version": "v27",
336
- "sha": "c05a794837fc",
337
- "date": "2026-08-18",
338
- "subject": "release v27"
339
- },
340
- {
341
- "version": "v25",
342
- "sha": "bc248295ed66",
343
- "date": "2026-08-14",
344
- "subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)"
345
- },
346
- {
347
- "version": "v26",
348
- "sha": "6a642e139cfa",
349
- "date": "2026-08-14",
350
- "subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)"
351
- },
352
- {
353
- "version": "v24",
354
- "sha": "514d8292ce73",
355
- "date": "2026-08-09",
356
- "subject": "release v24"
357
- },
358
- {
359
- "version": "v23",
360
- "sha": "44cc28380fca",
361
- "date": "2026-08-09",
362
- "subject": "release v23"
363
- },
364
- {
365
- "version": "v22",
366
- "sha": "f14e199ff32f",
367
- "date": "2026-08-09",
368
- "subject": "release v22"
369
- },
370
- {
371
- "version": "v21",
372
- "sha": "5fdffce8b558",
373
- "date": "2026-08-09",
374
- "subject": "release v21"
375
- },
376
- {
377
- "version": "v20",
378
- "sha": "442e4a1430a7",
379
- "date": "2026-08-09",
380
- "subject": "release v20"
381
- },
382
- {
383
- "version": "v19",
384
- "sha": "94cb4a4dfdfe",
385
- "date": "2026-08-09",
386
- "subject": "release v19"
387
- },
388
- {
389
- "version": "v18",
390
- "sha": "0106ef949805",
391
- "date": "2026-08-09",
392
- "subject": "release v18"
393
- },
394
- {
395
- "version": "v17",
396
- "sha": "308240cc75e8",
397
- "date": "2026-08-09",
398
- "subject": "release v17"
399
- },
400
- {
401
- "version": "v16",
402
- "sha": "a804df54430f",
403
- "date": "2026-08-09",
404
- "subject": "release v16"
405
- },
406
- {
407
- "version": "v15",
408
- "sha": "6b84d7264c81",
409
- "date": "2026-08-09",
410
- "subject": "release v15"
411
- },
412
- {
413
- "version": "v14",
414
- "sha": "bee5eaf5b5cb",
415
- "date": "2026-08-07",
416
- "subject": "release v14"
417
- },
418
- {
419
- "version": "v13",
420
- "sha": "314c0e368b1d",
421
- "date": "2026-08-06",
422
- "subject": "release v13"
423
- },
424
- {
425
- "version": "v12",
426
- "sha": "4908f399785d",
427
- "date": "2026-08-06",
428
- "subject": "release v12"
429
- },
430
- {
431
- "version": "v11",
432
- "sha": "312661712b98",
433
- "date": "2026-08-05",
434
- "subject": "release v11"
435
- },
436
- {
437
- "version": "v10",
438
- "sha": "43f804287496",
439
- "date": "2026-08-05",
440
- "subject": "release v10"
441
- },
442
- {
443
- "version": "v9",
444
- "sha": "2e26f57396b1",
445
- "date": "2026-08-05",
446
- "subject": "release v9"
447
- },
448
- {
449
- "version": "v8",
450
- "sha": "8073b3ecbac3",
451
- "date": "2026-08-05",
452
- "subject": "release v8"
453
- },
454
- {
455
- "version": "v7",
456
- "sha": "b1c5a7e9b848",
457
- "date": "2026-08-05",
458
- "subject": "release v7"
459
- },
460
- {
461
- "version": "v6",
462
- "sha": "a344020f114a",
463
- "date": "2026-08-05",
464
- "subject": "release v6"
465
- },
466
- {
467
- "version": "v5",
468
- "sha": "c0b0fbb40a7b",
469
- "date": "2026-08-05",
470
- "subject": "release v5"
471
- },
472
- {
473
- "version": "v4",
474
- "sha": "de5037ff1c8f",
475
- "date": "2026-08-05",
476
- "subject": "release v4"
477
- },
478
- {
479
- "version": "v3",
480
- "sha": "6b9fa62d93e8",
481
- "date": "2026-08-05",
482
- "subject": "release v3"
483
- },
484
- {
485
- "version": "v2",
486
- "sha": "fd0586199858",
487
- "date": "2026-08-04",
488
- "subject": "release v2"
489
- },
490
- {
491
- "version": "v1",
492
- "sha": "5b4e2c4bc3f3",
493
- "date": "2026-08-04",
494
- "subject": "release v1"
495
  }
496
  ]
497
  }
 
1
  {
2
+ "current": "v85 (b36dd49921)",
3
  "releases": [
4
+ {
5
+ "version": "v85",
6
+ "sha": "b36dd49921",
7
+ "date": "2026-10-25",
8
+ "subject": "release v85"
9
+ },
10
+ {
11
+ "version": "v84",
12
+ "sha": "7bb3a2e09e",
13
+ "date": "2026-10-24",
14
+ "subject": "release(sdlc78): v84 QA records the staging build change and what v84 excludes"
15
+ },
16
  {
17
  "version": "v83",
18
+ "sha": "9b7471968e",
19
  "date": "2026-10-21",
20
  "subject": "release v83"
21
  },
22
  {
23
  "version": "v82",
24
+ "sha": "9c61b1ee94",
25
  "date": "2026-10-17",
26
  "subject": "release v82"
27
  },
28
  {
29
  "version": "v81",
30
+ "sha": "e141ccc85e",
31
  "date": "2026-10-15",
32
  "subject": "release v81"
33
  },
34
  {
35
  "version": "v80",
36
+ "sha": "869b8fcc62",
37
  "date": "2026-10-13",
38
  "subject": "release v80"
39
  },
40
  {
41
  "version": "v79",
42
+ "sha": "406417b8de",
43
  "date": "2026-10-13",
44
  "subject": "release v79"
45
  },
46
  {
47
  "version": "v78",
48
+ "sha": "aad6561d63",
49
  "date": "2026-10-09",
50
  "subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs"
51
  },
52
  {
53
  "version": "v77",
54
+ "sha": "e741e5ca60",
55
  "date": "2026-10-05",
56
  "subject": "release v77"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  }
58
  ]
59
  }
VERSION CHANGED
@@ -1 +1 @@
1
- f75ce5527c (f75ce5527c67)
 
1
+ v85 (b36dd49921)
api/agent_runtime.py CHANGED
@@ -742,7 +742,18 @@ def _hold(session, name, args, effect, held, turn):
742
  resolved = _fill(args, refs, len(held) + 1)
743
  if _PLACEHOLDER.search(json.dumps(resolved)):
744
  raise ValueError("A change can only use an earlier step's created object.")
745
- preview = assistant_write_preview.prepare(session, name, resolved, prior=held)
 
 
 
 
 
 
 
 
 
 
 
746
  _observe_sources(session, turn, name, resolved, preview["result"])
747
  held.append({"tool": name, "args": copy.deepcopy(args), "effect": effect, "ref": ref,
748
  "resolvedArgs": resolved, "preview": preview,
@@ -826,7 +837,8 @@ def _apply_change(session, entry):
826
  import assistant_write_preview
827
  if args != step.get("resolvedArgs"):
828
  raise ValueError("An earlier step no longer matches this preview. Ask again.")
829
- result = assistant_write_preview.apply(session, step)
 
830
  _observe_sources(session, turn, step["tool"], args, result)
831
  finally:
832
  _PRODUCING_TURN.reset(token)
 
742
  resolved = _fill(args, refs, len(held) + 1)
743
  if _PLACEHOLDER.search(json.dumps(resolved)):
744
  raise ValueError("A change can only use an earlier step's created object.")
745
+ if name == "apps_confirm":
746
+ import routes_external as external
747
+ payload = external._read_confirmation_token(resolved.get("confirmation_token"), session=session)
748
+ if payload.get("tool") != "apps":
749
+ raise ValueError("Choose an App preview to confirm.")
750
+ external._load_operation(session, payload["operation_id"], confirming=True)
751
+ result = external._preview_external_write(
752
+ session, "apps", payload["arguments"], issue_token=False, execution=payload["execution"],
753
+ purpose=payload.get("purpose"), about=payload.get("about"))
754
+ preview = {"documents": {}, "result": result, "diff": result.get("diff") or []}
755
+ else:
756
+ preview = assistant_write_preview.prepare(session, name, resolved, prior=held)
757
  _observe_sources(session, turn, name, resolved, preview["result"])
758
  held.append({"tool": name, "args": copy.deepcopy(args), "effect": effect, "ref": ref,
759
  "resolvedArgs": resolved, "preview": preview,
 
837
  import assistant_write_preview
838
  if args != step.get("resolvedArgs"):
839
  raise ValueError("An earlier step no longer matches this preview. Ask again.")
840
+ result = (_dispatch(session, "apps_confirm", args) if step["tool"] == "apps_confirm"
841
+ else assistant_write_preview.apply(session, step))
842
  _observe_sources(session, turn, step["tool"], args, result)
843
  finally:
844
  _PRODUCING_TURN.reset(token)
api/amazon_relational.py CHANGED
@@ -47,6 +47,7 @@ if str(_API.parent.parent / "platform") not in sys.path:
47
  import odoo_relational as _odoo # noqa: E402
48
  from core import connector_spec # noqa: E402
49
  from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402
 
50
 
51
  Refused = _odoo.Refused
52
 
@@ -57,7 +58,10 @@ PREFIX = "ut_amzn_"
57
 
58
  #: `ut_*` key -> the mirror table `amazon_store` writes. Derived from ONE list so a table cannot
59
  #: exist on one side and not the other.
60
- SOURCE = {PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_TABLES}
 
 
 
61
 
62
  #: ⭐⭐ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE**
63
  #: (W53-T37, contract C2). `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their module-level
@@ -66,7 +70,20 @@ SOURCE = {PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_
66
  #: longer be stated twice and say two things. Every comment that used to justify a literal here
67
  #: (the two spines, R12, the report-vs-API spellings, the cardinalities) travelled with the values
68
  #: into `connector_spec.AMAZON_TABLES`; read it there.
69
- _SPEC = {s.key: s for s in connector_spec.tables_for("amazon")}
 
 
 
 
 
 
 
 
 
 
 
 
 
70
 
71
  #: ⛔ A `ut_*` KEY THAT `amazon_store` SPAWNS BUT THE SPEC DOES NOT DECLARE IS ABSENT FROM ALL FOUR
72
  #: DICTS BELOW, ON PURPOSE. `verify_amazon.section_relational` asserts the four key sets equal
@@ -106,8 +123,10 @@ def source_attribution(key):
106
  if key not in SOURCE or key not in _SPEC:
107
  raise KeyError(f"unknown Amazon database {key!r}")
108
  return {
109
- "key": "amazon_sp",
110
- "origin": "Apify Amazon search dataset" if key == PREFIX + "keyword_ranks" else "Amazon Services API",
 
 
111
  "rawTable": SOURCE[key],
112
  "native": _SPEC[key].native,
113
  "readOnly": True,
@@ -621,8 +640,8 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True):
621
  return missing_source(key)
622
  names = {name for name, _dtype in columns}
623
  table = key.removeprefix(PREFIX)
624
- identity = amazon_schema.row_key_columns_for(table, names)
625
- fallback = sorted(set((*identity, *amazon_schema.row_fallback_columns_for(table))))
626
  source = _quoted(SOURCE[key])
627
 
628
  def value(name, owner=source):
@@ -713,14 +732,14 @@ def grid_sources(cur=None):
713
  def _row_id(row, key, occurrence=0):
714
  """Return a legacy row key from declared identity, lineage and equal-key occurrence."""
715
  table = key.removeprefix(PREFIX)
716
- columns = amazon_schema.row_key_columns_for(table, row)
717
  identity = tuple(row.get(name) for name in columns)
718
  if all(value not in (None, "") for value in identity):
719
  if table in _LEGACY_IDENTITY_TABLES and len(identity) == 1:
720
  return str(identity[0])
721
  payload = json.dumps(identity, ensure_ascii=True, default=str)
722
  else:
723
- fallback = sorted(set((*columns, *amazon_schema.row_fallback_columns_for(table))))
724
  values = tuple(None if row.get(name) in (None, "") else str(row[name])
725
  for name in fallback)
726
  payload = json.dumps(("__amazon_missing_identity__", *values, str(occurrence)),
@@ -1100,6 +1119,9 @@ def apply_plan(rt, built, username="amazon", today=None):
1100
  for key, _label, _fields, _rows in plans}
1101
  listing_ids = (_listing_identity_rows(cur_mirror)
1102
  if PREFIX + "listing_items" in built else ())
 
 
 
1103
  for key, _label, _fields, rows in plans:
1104
  if isinstance(rows, dict):
1105
  continue
@@ -1116,7 +1138,8 @@ def apply_plan(rt, built, username="amazon", today=None):
1116
  candidates = (
1117
  _row_id(raw, key, occurrence),
1118
  hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32])
1119
- declared = amazon_schema.TABLE_SPECS[key.removeprefix(PREFIX)]["row_identity"]
 
1120
  if all(raw.get(name) not in (None, "") for name in declared):
1121
  old_composite = json.dumps(tuple(raw[name] for name in declared),
1122
  ensure_ascii=True, default=str)
@@ -1235,6 +1258,8 @@ def apply_plan(rt, built, username="amazon", today=None):
1235
  # DPP 1.8 attribution is explicit in the stored definition, not inferred from a
1236
  # mutable display label or from the process having imported this module.
1237
  doc[key]["connector"] = source_attribution(key)
 
 
1238
  if key == PREFIX + "listing_items":
1239
  doc[key]["amazonListingIdentity"] = "sku"
1240
  return doc
 
47
  import odoo_relational as _odoo # noqa: E402
48
  from core import connector_spec # noqa: E402
49
  from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402
50
+ from harness import amazon_ads_store # noqa: E402
51
 
52
  Refused = _odoo.Refused
53
 
 
58
 
59
  #: `ut_*` key -> the mirror table `amazon_store` writes. Derived from ONE list so a table cannot
60
  #: exist on one side and not the other.
61
+ #: S79-T13: the Ads tables (`ut_amzn_ads_*`, connector `amazon_ads`) live in the same mirror.
62
+ _ADS_SPEC = {s.key: s for s in connector_spec.tables_for("amazon_ads")}
63
+ SOURCE = {**{PREFIX + t: amazon_store.table_name(t) for t in amazon_store.DATABASE_TABLES},
64
+ **{k: amazon_ads_store.table_name(s.source) for k, s in _ADS_SPEC.items()}}
65
 
66
  #: ⭐⭐ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE**
67
  #: (W53-T37, contract C2). `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their module-level
 
70
  #: longer be stated twice and say two things. Every comment that used to justify a literal here
71
  #: (the two spines, R12, the report-vs-API spellings, the cardinalities) travelled with the values
72
  #: into `connector_spec.AMAZON_TABLES`; read it there.
73
+ _SPEC = {**{s.key: s for s in connector_spec.tables_for("amazon")}, **_ADS_SPEC}
74
+
75
+
76
+ def _identity_columns(table, columns=()):
77
+ """An Ads row is identified by its stored `_row_key`; every other table by amazon_schema."""
78
+ if PREFIX + table in _ADS_SPEC:
79
+ return ("_row_key",)
80
+ return amazon_schema.row_key_columns_for(table, columns)
81
+
82
+
83
+ def _fallback_columns(table):
84
+ if PREFIX + table in _ADS_SPEC:
85
+ return ("_report_id",)
86
+ return amazon_schema.row_fallback_columns_for(table)
87
 
88
  #: ⛔ A `ut_*` KEY THAT `amazon_store` SPAWNS BUT THE SPEC DOES NOT DECLARE IS ABSENT FROM ALL FOUR
89
  #: DICTS BELOW, ON PURPOSE. `verify_amazon.section_relational` asserts the four key sets equal
 
123
  if key not in SOURCE or key not in _SPEC:
124
  raise KeyError(f"unknown Amazon database {key!r}")
125
  return {
126
+ "key": "amazon_ads" if key in _ADS_SPEC else "amazon_sp",
127
+ "origin": ("Amazon Ads API" if key in _ADS_SPEC
128
+ else "Apify Amazon search dataset" if key == PREFIX + "keyword_ranks"
129
+ else "Amazon Services API"),
130
  "rawTable": SOURCE[key],
131
  "native": _SPEC[key].native,
132
  "readOnly": True,
 
640
  return missing_source(key)
641
  names = {name for name, _dtype in columns}
642
  table = key.removeprefix(PREFIX)
643
+ identity = _identity_columns(table, names)
644
+ fallback = sorted(set((*identity, *_fallback_columns(table))))
645
  source = _quoted(SOURCE[key])
646
 
647
  def value(name, owner=source):
 
732
  def _row_id(row, key, occurrence=0):
733
  """Return a legacy row key from declared identity, lineage and equal-key occurrence."""
734
  table = key.removeprefix(PREFIX)
735
+ columns = _identity_columns(table, row)
736
  identity = tuple(row.get(name) for name in columns)
737
  if all(value not in (None, "") for value in identity):
738
  if table in _LEGACY_IDENTITY_TABLES and len(identity) == 1:
739
  return str(identity[0])
740
  payload = json.dumps(identity, ensure_ascii=True, default=str)
741
  else:
742
+ fallback = sorted(set((*columns, *_fallback_columns(table))))
743
  values = tuple(None if row.get(name) in (None, "") else str(row[name])
744
  for name in fallback)
745
  payload = json.dumps(("__amazon_missing_identity__", *values, str(occurrence)),
 
1119
  for key, _label, _fields, _rows in plans}
1120
  listing_ids = (_listing_identity_rows(cur_mirror)
1121
  if PREFIX + "listing_items" in built else ())
1122
+ # S79-T13: each Ads table's description states its history window and any recorded limit.
1123
+ ads_descriptions = {key: amazon_ads_store.table_description(SOURCE[key], cur_mirror)
1124
+ for key, *_rest in plans if key in _ADS_SPEC}
1125
  for key, _label, _fields, rows in plans:
1126
  if isinstance(rows, dict):
1127
  continue
 
1138
  candidates = (
1139
  _row_id(raw, key, occurrence),
1140
  hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32])
1141
+ declared = (("_row_key",) if key in _ADS_SPEC
1142
+ else amazon_schema.TABLE_SPECS[key.removeprefix(PREFIX)]["row_identity"])
1143
  if all(raw.get(name) not in (None, "") for name in declared):
1144
  old_composite = json.dumps(tuple(raw[name] for name in declared),
1145
  ensure_ascii=True, default=str)
 
1258
  # DPP 1.8 attribution is explicit in the stored definition, not inferred from a
1259
  # mutable display label or from the process having imported this module.
1260
  doc[key]["connector"] = source_attribution(key)
1261
+ if ads_descriptions.get(key):
1262
+ doc[key]["description"] = ads_descriptions[key]
1263
  if key == PREFIX + "listing_items":
1264
  doc[key]["amazonListingIdentity"] = "sku"
1265
  return doc
api/app_routes.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """C3: mount every C1 App router by name, recording a failed import instead of raising it."""
2
+ from __future__ import annotations
3
+
4
+ import sys
5
+ from importlib import import_module
6
+
7
+ import app_templates
8
+
9
+
10
+ def _failures_record():
11
+ main = sys.modules.get("main")
12
+ record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None
13
+ return record if isinstance(record, dict) else app_templates.ROUTER_MOUNT_FAILURES
14
+
15
+
16
+ def _flat(routes):
17
+ """Every route, descending into a router FastAPI keeps as one included entry."""
18
+ for route in routes:
19
+ yield route
20
+ inner = getattr(route, "original_router", None)
21
+ if inner is not None:
22
+ yield from _flat(inner.routes)
23
+
24
+
25
+ def _included(app, router):
26
+ endpoints = {getattr(route, "endpoint", None) for route in _flat(router.routes)}
27
+ endpoints.discard(None)
28
+ return any(getattr(route, "original_router", None) is router
29
+ or getattr(route, "endpoint", None) in endpoints for route in _flat(app.routes))
30
+
31
+
32
+ def held_dependencies(kind):
33
+ """Router dependencies that answer 404 on every route of a held kind (R11), else none."""
34
+ if not app_templates.held(kind):
35
+ return []
36
+ from fastapi import Depends
37
+ from deps import err
38
+
39
+ def refuse_held():
40
+ raise err(404, "app_held", "This App is not available yet.")
41
+ return [Depends(refuse_held)]
42
+
43
+
44
+ def mount_app_routers(app, failures=None):
45
+ """Include each C1 `router` once, with main.py `_mount_router` semantics; returns those mounted."""
46
+ record = failures if isinstance(failures, dict) else _failures_record()
47
+ mounted = []
48
+ for row in app_templates.TEMPLATES:
49
+ name = row["router"]
50
+ if not name:
51
+ continue
52
+ try:
53
+ router = import_module(name).router
54
+ # An already included router (Competitor watch, mounted by main.py) is not added twice.
55
+ if not _included(app, router):
56
+ app.include_router(router, dependencies=held_dependencies(row["kind"]))
57
+ except Exception as exc: # Boot must answer even when a router cannot load.
58
+ failure = {"error": type(exc).__name__,
59
+ "missing_module": exc.name if isinstance(exc, ModuleNotFoundError) else ""}
60
+ record.setdefault(name, []).append(failure)
61
+ if record is not _failures_record():
62
+ # The offer predicate reads main.py's record and this one, never a caller's own dict.
63
+ app_templates.ROUTER_MOUNT_FAILURES.setdefault(name, []).append(failure)
64
+ # Exception messages can contain configuration values; report the class/module only.
65
+ print(f"[aios-api] ROUTER MOUNT FAILED: {name}: {failure}; "
66
+ "its App template is withheld until the router mounts")
67
+ continue
68
+ app_templates.MOUNTED_ROUTERS.add(name)
69
+ mounted.append(name)
70
+ return mounted
api/app_rules.py CHANGED
@@ -441,8 +441,7 @@ def _catalog_selftest():
441
  source = rt.get("product_table_workspace__shared")
442
  contracts = registry.chart_field_contracts(session, ["product_data"])
443
  app = registry.create(rt, tenant, "member", {
444
- "name": "Template fixture", "templateKind": "catalog", "sourceDatabase": "product_data",
445
- "fieldMap": {"key": "code", "title": "product", "image": "catalog_photo"},
446
  "navPlacement": {"section": "app", "order": 0}},
447
  allowed_databases={"product_data"}, field_contracts=contracts)
448
  api = FastAPI()
 
441
  source = rt.get("product_table_workspace__shared")
442
  contracts = registry.chart_field_contracts(session, ["product_data"])
443
  app = registry.create(rt, tenant, "member", {
444
+ "name": "Template fixture", "templateKind": "catalog",
 
445
  "navPlacement": {"section": "app", "order": 0}},
446
  allowed_databases={"product_data"}, field_contracts=contracts)
447
  api = FastAPI()
api/app_templates.py ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """C1: the shipped App template catalogue, one row per App template, each bundled with its connector.
2
+
3
+ The registry's template contracts and `core.external_tools`' templateKind enum derive from or
4
+ mirror `TEMPLATES`; `verify_app_templates.py` holds all three name for name.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import sys
9
+ from importlib import import_module
10
+ from types import MappingProxyType
11
+
12
+ PROCESSES = ("Create value", "Acquire", "Convert", "Deliver", "Retain", "Finance and cash",
13
+ "Risk and compliance", "None")
14
+
15
+ # A C1 connector key is not a keychain entry type: Amazon SP-API is stored as `amazon_sp` (F25).
16
+ CONNECTOR_ENTRY_TYPES = MappingProxyType({"odoo": "odoo", "amazon": "amazon_sp", "amazon_ads": "amazon_ads"})
17
+
18
+
19
+ def _amazon(kind, label, purpose, process, sources, *, connector="amazon", proposes=True, tools=()):
20
+ return {"kind": kind, "label": label, "purpose": purpose, "connector": connector,
21
+ "sources": tuple(sources), "process": process, "mark": kind,
22
+ "module": "modules." + kind, "router": "routes_" + kind,
23
+ "view": kind.replace("_", "-"), "tools": tuple(tools), "proposes": proposes}
24
+
25
+
26
+ TEMPLATES = (
27
+ # Routing and Catalog render through Map mode; their routes are mounted by main.py.
28
+ # They read FIELD_BINDINGS over their source, so an App of either kind stores no field map.
29
+ {"kind": "route", "label": "Routing", "purpose": "Plan delivery routes for your customers.",
30
+ "connector": "odoo", "sources": ("customer_data",), "process": "None", "mark": "route",
31
+ "module": None, "router": None, "view": None, "tools": (), "proposes": False},
32
+ {"kind": "catalog", "label": "Catalog", "purpose": "Build and print product catalogs.",
33
+ "connector": "odoo", "sources": ("product_data",), "process": "None", "mark": "catalog",
34
+ "module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
35
+ "proposes": False},
36
+ _amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
37
+ "Acquire", ("ut_amzn_brand_analytics",), proposes=False),
38
+ _amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
39
+ "Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
40
+ "ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
41
+ "ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False),
42
+ _amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
43
+ "Acquire", (), connector="amazon_ads", proposes=False),
44
+ _amazon("restock_planner", "Restock planner", "Send stock to Amazon before it runs out.",
45
+ "Deliver", ("ut_amzn_restock_recommendations", "ut_amzn_inventory_planning",
46
+ "ut_amzn_fba_inventory")),
47
+ _amazon("reimbursements", "Reimbursements", "Find FBA money Amazon owes you.",
48
+ "Create value", ("ut_amzn_reimbursements", "ut_amzn_inventory_ledger_detail",
49
+ "ut_amzn_fba_customer_returns")),
50
+ _amazon("aged_stock", "Aged stock", "Move old stock before storage fees grow.",
51
+ "Create value", ("ut_amzn_inventory_planning", "ut_amzn_long_term_storage_fees",
52
+ "ut_amzn_storage_fees", "ut_amzn_recommended_removals",
53
+ "ut_amzn_stranded_inventory")),
54
+ _amazon("opportunity_finder", "Opportunity finder", "Find rising searches you do not serve yet.",
55
+ "Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings")),
56
+ _amazon("returns_repeat", "Returns and repeat buyers", "See why buyers return and who buys again.",
57
+ "Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_repeat_purchase",
58
+ "ut_amzn_fba_shipped_sales", "ut_amzn_market_basket")),
59
+ _amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
60
+ "Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions")),
61
+ _amazon("account_health", "Account health", "Catch account risks before Amazon acts.",
62
+ "Risk and compliance", ("ut_amzn_account_health",)),
63
+ )
64
+ BY_KIND = MappingProxyType({row["kind"]: row for row in TEMPLATES})
65
+ KINDS = tuple(BY_KIND)
66
+
67
+ #: R11 (owner 2026-10-25): every Amazon App is held out of the product until SDLC 80 rebuilds it.
68
+ #: A held kind is never offered, listed or opened. Its stored App records and data stay untouched.
69
+ HELD_KINDS = frozenset(row["kind"] for row in TEMPLATES if row["connector"] in ("amazon", "amazon_ads"))
70
+
71
+
72
+ def held(kind_or_app):
73
+ """True when a template kind, or an App record's `templateKind`, is held out (R11)."""
74
+ kind = kind_or_app.get("templateKind") if isinstance(kind_or_app, dict) else kind_or_app
75
+ return isinstance(kind, str) and kind in HELD_KINDS
76
+
77
+ #: R33: the fixed role to field binding each kind reads from its first C1 source.
78
+ #: Catalog price is Royal 1 tier 1: the pooled `price_1` is not in the typed product_data schema.
79
+ FIELD_BINDINGS = MappingProxyType({
80
+ "route": MappingProxyType({"title": "customer", "address": "street",
81
+ "latitude": "lat", "longitude": "lon"}),
82
+ "catalog": MappingProxyType({"key": "code", "title": "product", "image": "image",
83
+ "sku": "code", "price": "price_royal1_t1", "category": "category"}),
84
+ "competitor_watch": MappingProxyType({"term": "search_term"}),
85
+ })
86
+ #: Kinds whose App record stores no source or field map: the registry projects C1 on read.
87
+ BOUND_KINDS = ("route", "catalog")
88
+ _BINDING_KEYS = ("sourceDatabase", "fieldMap")
89
+
90
+
91
+ def source(kind):
92
+ """The C1 source table a kind reads, or None."""
93
+ row = BY_KIND.get(kind)
94
+ return row["sources"][0] if row and row["sources"] else None
95
+
96
+
97
+ def field_binding(kind):
98
+ """A copy of the kind's fixed role to field map; empty when it names none."""
99
+ return dict(FIELD_BINDINGS.get(kind) or {})
100
+
101
+
102
+ def bound(app):
103
+ """Read projection: a copy of a bound kind's App with the C1 source and field map filled in.
104
+
105
+ A stored App that still carries its own map keeps it until the S79-T04 migration.
106
+ """
107
+ if not isinstance(app, dict) or app.get("templateKind") not in BOUND_KINDS:
108
+ return app
109
+ row = dict(app)
110
+ projection = {"sourceDatabase": source(row["templateKind"]),
111
+ "fieldMap": field_binding(row["templateKind"])}
112
+ if row.get("fieldMap"):
113
+ projection = {"sourceDatabase": row.get("sourceDatabase") or projection["sourceDatabase"]}
114
+ row.update(projection)
115
+ return row
116
+
117
+
118
+ def own_binding(app):
119
+ """The binding keys a stored App carries that differ from what `bound` projects."""
120
+ if not isinstance(app, dict) or app.get("templateKind") not in BOUND_KINDS:
121
+ return {key: app[key] for key in _BINDING_KEYS if isinstance(app, dict) and key in app}
122
+ kind = app["templateKind"]
123
+ fixed = field_binding(kind)
124
+ held = app.get("fieldMap") or {}
125
+ if (app.get("sourceDatabase") in (None, source(kind))
126
+ and all(fixed.get(role) == field for role, field in held.items())):
127
+ return {}
128
+ return {key: app[key] for key in _BINDING_KEYS if key in app}
129
+
130
+ #: Routers `app_routes.mount_app_routers` failed to mount when main.py's record is absent.
131
+ ROUTER_MOUNT_FAILURES = {}
132
+ #: Routers `app_routes.mount_app_routers` mounted or found already included.
133
+ MOUNTED_ROUTERS = set()
134
+ # A failed seam import is printed once per process and withholds its kind.
135
+ _SEAM_FAILURES = {}
136
+
137
+
138
+ def entry_type(connector):
139
+ """The keychain entry type that proves a C1 connector key is connected."""
140
+ return CONNECTOR_ENTRY_TYPES.get(connector, connector)
141
+
142
+
143
+ def connectors_present(session):
144
+ """C1 connector keys this reader has. Any read failure counts as nothing connected."""
145
+ try:
146
+ from routes_connectors import _entry_types
147
+ types, _scopes, failure = _entry_types(session.runtime, session.uname)
148
+ have = set() if failure else set(types or ())
149
+ except Exception as exc: # noqa: BLE001
150
+ from core.request_timing import raise_if_resource_error
151
+ raise_if_resource_error(exc)
152
+ have = set()
153
+ if "odoo" not in have:
154
+ try:
155
+ from routes_keychain import env_odoo_available
156
+ if env_odoo_available(session.runtime):
157
+ have.add("odoo")
158
+ except Exception: # noqa: BLE001
159
+ pass
160
+ return {key for key in CONNECTOR_ENTRY_TYPES if entry_type(key) in have}
161
+
162
+
163
+ def connector_present(connector, session):
164
+ return connector in connectors_present(session)
165
+
166
+
167
+ def router_failures():
168
+ """Every recorded router mount failure: main.py's record joined with this module's own."""
169
+ merged = {name: list(rows) for name, rows in ROUTER_MOUNT_FAILURES.items()}
170
+ main = sys.modules.get("main")
171
+ record = getattr(main, "ROUTER_MOUNT_FAILURES", None) if main is not None else None
172
+ if isinstance(record, dict) and record is not ROUTER_MOUNT_FAILURES:
173
+ for name, rows in record.items():
174
+ merged.setdefault(name, []).extend(rows)
175
+ return merged
176
+
177
+
178
+ def _importable(name, attribute=None):
179
+ if name in _SEAM_FAILURES:
180
+ return False
181
+ try:
182
+ module = import_module(name)
183
+ if attribute and not hasattr(module, attribute):
184
+ raise ImportError(f"{name} has no {attribute}")
185
+ return True
186
+ except Exception as exc: # noqa: BLE001
187
+ _SEAM_FAILURES[name] = type(exc).__name__
188
+ print(f"[aios-api] APP TEMPLATE WITHHELD: {name}: {type(exc).__name__}")
189
+ return False
190
+
191
+
192
+ def seams_ready(kind):
193
+ """True when the kind's module imports and its router imports and never failed to mount."""
194
+ row = BY_KIND.get(kind)
195
+ if row is None:
196
+ return False
197
+ if row["module"] and not _importable(row["module"]):
198
+ return False
199
+ router = row["router"]
200
+ return not router or (_importable(router, "router") and router not in router_failures())
201
+
202
+
203
+ def reset_seam_cache():
204
+ """Forget recorded seam import failures (test fixtures only)."""
205
+ _SEAM_FAILURES.clear()
api/application_registry.py CHANGED
@@ -8,6 +8,7 @@ import json
8
  from core import shares, write_execution
9
  from core.request_timing import raise_if_resource_error
10
  import app_rules
 
11
 
12
  from modules.finance_apps import (finance_application as _finance_application,
13
  finance_applications as _finance_applications,
@@ -19,6 +20,7 @@ APPLICATION_TYPES = ("chart", "route", "map", "code")
19
  # Finance descriptors remain available to the governed fact routes.
20
  FINANCE_TEMPLATE_KIND = "finance"
21
  _KEY = "applications_v1"
 
22
  _MAX = 500
23
  _MAX_DISPLAY_BYTES = 512 * 1024
24
 
@@ -66,28 +68,58 @@ _TEMPLATE_CONTRACTS = {
66
  },
67
  }
68
 
 
 
 
 
 
 
 
 
 
69
  # Keep stored copy identities readable after a template stops accepting new Apps.
70
  APPLICATION_WIZARD_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
71
  APPLICATION_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
72
 
73
  #: S78-T17 (C4, R20, R21): templates offered only to a tenant that passes the kind's gate.
74
  #: They stay out of the ungated tuples above, which other gates pin to Routing and Catalog.
 
75
  BRAND_ANALYTICS_DATABASE = "ut_amzn_brand_analytics"
76
- _GATED_TEMPLATE_CONTRACTS = {
 
77
  "competitor_watch": {
78
- "kind": "competitor_watch",
79
- "label": "Competitor watch",
80
- # The screen reads the competitor watch routes; the source names the Amazon database.
81
- "applicationType": "map",
82
- "displayMode": "map",
83
  "fieldRoles": [
84
  {"key": "term", "label": "Search term", "required": True, "types": []},
85
  ],
86
- "defaultPlacement": {"section": "app", "order": 0},
87
- "sourceDatabase": BRAND_ANALYTICS_DATABASE,
88
  "sourceLabel": "Amazon brand analytics",
89
  },
90
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  GATED_TEMPLATE_KINDS = tuple(_GATED_TEMPLATE_CONTRACTS)
92
 
93
 
@@ -140,13 +172,56 @@ def brand_analytics_offered(session):
140
  return False
141
 
142
 
143
- _TEMPLATE_GATES = {"competitor_watch": brand_analytics_offered}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
144
 
145
 
146
  class ApplicationError(ValueError):
147
  pass
148
 
149
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
  class ApplicationRuleError(ApplicationError):
151
  """Expose an App rule refusal through every writer's existing error handler."""
152
  code = "app_rule"
@@ -176,7 +251,8 @@ def template_identity(app):
176
  if kind == FINANCE_TEMPLATE_KIND:
177
  key = str(app.get("financeKey") or "")
178
  return f"{FINANCE_TEMPLATE_KIND}:{key}" if key else None
179
- return kind if kind in APPLICATION_WIZARD_TEMPLATE_KINDS else None
 
180
 
181
 
182
  def _own_template_app(applications, user, identity):
@@ -208,12 +284,15 @@ def own_template_apps(runtime, user):
208
  def template_contracts(session=None):
209
  """The trusted renderer contracts offered by the Application creation flow.
210
 
211
- A gated template joins only for a session whose tenant passes its gate (R21).
212
  """
213
- rows = _ungated_template_contracts()
214
- if session is not None:
215
- rows.extend(copy.deepcopy(contract) for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()
216
- if _TEMPLATE_GATES[kind](session))
 
 
 
217
  return rows
218
 
219
 
@@ -366,8 +445,12 @@ def _clean_doc(raw):
366
 
367
 
368
  def load(runtime):
369
- try: return _clean_doc(runtime.get(_KEY))
 
370
  except Exception: return blank_doc()
 
 
 
371
 
372
 
373
  def _server_origin(origin_door=None, via=None):
@@ -436,6 +519,14 @@ def _database_free_code(app):
436
  and not app.get("compatibleDatabases"))
437
 
438
 
 
 
 
 
 
 
 
 
439
  def _finance_permission(app):
440
  try:
441
  return _finance_application(str(app.get("financeKey") or ""))["permissionKey"]
@@ -711,66 +802,45 @@ def _refuse_legacy_catalog_changes(app, config):
711
 
712
  def _template_definition(kind, payload, *, semantic_records=None, field_contracts=None,
713
  allowed_databases=None):
 
714
  contract = _TEMPLATE_CONTRACTS.get(kind) or _GATED_TEMPLATE_CONTRACTS.get(kind)
715
  if contract is None:
716
  raise ApplicationError("application template kind is not recognized")
717
  template_kind = str(payload.get("templateKind") or "").strip().lower()
718
  if template_kind != kind:
719
  raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
720
- source = str(payload.get("sourceDatabase") or "").strip()[:120]
721
- if not source:
 
 
 
 
 
 
722
  raise ApplicationError("sourceDatabase is required")
723
- if contract.get("sourceDatabase") and source != contract["sourceDatabase"]:
724
  raise ApplicationError(f"{contract['label']} reads "
725
- f"{contract.get('sourceLabel') or contract['sourceDatabase']} only. "
726
  "Choose it as the source database.")
727
- if source not in set(allowed_databases or ()):
728
  raise PermissionError("an Application cannot bind a database you cannot open")
729
- if "compatibleDatabases" in payload:
730
  compatible = _compat(payload.get("compatibleDatabases"))
731
  if compatible != [source]:
732
  raise ApplicationError("a template Application has exactly one sourceDatabase")
733
-
734
- raw_map = payload.get("fieldMap")
735
- if not isinstance(raw_map, dict):
736
- raise ApplicationError("fieldMap must be an object")
737
- roles = {row["key"]: row for row in contract["fieldRoles"]}
738
- extras = sorted(set(raw_map) - set(roles))
739
- if extras:
740
- raise ApplicationError("fieldMap contains roles outside the template contract: "
741
- + ", ".join(extras) + ". Accepted roles: " + ", ".join(roles))
742
- missing = sorted(key for key, role in roles.items()
743
- if role["required"] and not str(raw_map.get(key) or "").strip())
744
- if missing:
745
- raise ApplicationError("fieldMap is missing required roles: " + ", ".join(missing))
746
- available = _contract_fields(source, semantic_records, field_contracts)
747
- if not available:
748
- raise ApplicationError("the source database has no readable field contract")
749
- field_map = {}
750
- for role, raw_field in raw_map.items():
751
- if raw_field is not None and not isinstance(raw_field, str):
752
- raise ApplicationError(f"fieldMap role {role!r} must name a source field with a text key")
753
- field = str(raw_field or "").strip()
754
- if not field:
755
- continue
756
- if len(field) > 160 or field not in available:
757
- raise ApplicationError(f"fieldMap role {role!r} does not name a source field")
758
- allowed_types = set(roles[role].get("types") or ())
759
- if allowed_types and available[field] not in allowed_types:
760
- raise ApplicationError(
761
- f"fieldMap role {role!r} cannot use field type {available[field]!r}")
762
- field_map[role] = field
763
- latitude, longitude = field_map.get("latitude"), field_map.get("longitude")
764
- if bool(latitude) != bool(longitude):
765
- raise ApplicationError("Routing needs both coordinate fields or neither")
766
- if latitude and latitude == longitude:
767
- raise ApplicationError("Routing latitude and longitude must use different fields")
768
- # A new Route must name at least one placement source. Stored definitions are not revalidated
769
- # on read, so this create-time refusal does not retire older Applications with an empty map.
770
- if kind == "route" and not (latitude and longitude) and not field_map.get("address"):
771
- raise ApplicationError("Routing needs both coordinate fields or an address field")
772
-
773
- raw_placement = payload.get("navPlacement")
774
  if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
775
  raise ApplicationError("navPlacement must contain section and order")
776
  section = str(raw_placement.get("section") or "").strip().lower()
@@ -779,8 +849,33 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
779
  raise ApplicationError("template Applications must be placed in App")
780
  if isinstance(order, bool) or not isinstance(order, int) or not 0 <= order <= 999:
781
  raise ApplicationError("navPlacement order must be an integer from 0 to 999")
782
- return {"templateKind": kind, "sourceDatabase": source, "fieldMap": field_map,
783
- "navPlacement": {"section": "app", "order": order}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
784
 
785
 
786
  def validate_application(payload, tenant, *, semantic_records=None, field_contracts=None,
@@ -830,7 +925,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra
830
 
831
  def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
832
  is_default=False):
833
- row=copy.deepcopy(app)
834
  row.setdefault("origin", None)
835
  if row.get("type") == "code":
836
  # Only the authenticated code door returns the body.
@@ -844,6 +939,11 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
844
  "default":bool(is_admin)}
845
  row["starred"] = user in (app.get("starredBy") or ())
846
  row["isDefault"] = bool(is_default)
 
 
 
 
 
847
  if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
848
  # A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
849
  row["capabilities"]["default"] = False
@@ -864,6 +964,8 @@ def dependent_view_counts(runtime, database):
864
  """S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
865
  user's Views in that database's workspace. The Delete App dialog names this count before
866
  confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
 
 
867
  """
868
  from core.view_templates import workspace_key
869
  key = workspace_key(database)
@@ -874,16 +976,11 @@ def dependent_view_counts(runtime, database):
874
  except Exception:
875
  return None
876
  counts = {}
877
- for member, bucket in (doc.items() if isinstance(doc, dict) else ()):
878
- if member in _RESERVED_WORKSPACE_MEMBERS or not isinstance(bucket, dict):
879
- continue
880
- views = bucket.get("views")
881
- for view in (views.values() if isinstance(views, dict) else ()):
882
- config = view.get("config") if isinstance(view, dict) else None
883
- ref = config.get("application") if isinstance(config, dict) else None
884
- app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
885
- if app_id:
886
- counts[app_id] = counts.get(app_id, 0) + 1
887
  return counts
888
 
889
 
@@ -893,6 +990,8 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
893
  doc = load(runtime)
894
  rows=[]
895
  for app in doc["applications"].values():
 
 
896
  if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
897
  continue
898
  if database:
@@ -900,7 +999,7 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
900
  if not may_read(app,user,is_admin): continue
901
  if database not in (app.get("compatibleDatabases") or ()): continue
902
  elif not _listed(app, user): continue
903
- if (not _database_free_code(app)
904
  and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue
905
  if kind and app.get("type") != kind: continue
906
  if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
@@ -947,7 +1046,7 @@ def _build_application(tenant, user, payload, *, semantic_records=None, field_co
947
  name=" ".join(str(payload.get("name") or "").split())[:120]
948
  if not name: raise ApplicationError("application name is required")
949
  app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat()
950
- compatible = ([definition["sourceDatabase"]] if definition
951
  else _compat(payload.get("compatibleDatabases"), allow_empty=kind == "code"))
952
  if _allowed(compatible, allowed_databases) != set(compatible):
953
  raise PermissionError("an Application cannot bind a database you cannot open")
@@ -1035,7 +1134,8 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr
1035
  identity = template_identity(app)
1036
  def apply(raw):
1037
  doc=_clean_doc(raw)
1038
- if len(doc["applications"])>=_MAX: raise ApplicationError("application limit reached")
 
1039
  # R216 and R247 allow every template to have several independent copies.
1040
  doc["applications"][app_id]=app; return doc
1041
  _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of)
@@ -1066,10 +1166,14 @@ def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
1066
  """
1067
  result = _ensure_route_seed(runtime, tenant)
1068
  copies = ensure_user_app_copies(runtime, tenant, accounts=accounts, wall=wall)
 
1069
  folded = dict(result)
1070
  folded["userCopies"] = copies
 
1071
  note = "user copies " + copies["outcome"] + (
1072
  ": " + copies["reason"] if copies.get("reason") else "")
 
 
1073
  folded["reason"] = result["reason"] + "; " + note if result.get("reason") else note
1074
  return folded
1075
 
@@ -1106,13 +1210,12 @@ def _ensure_route_seed(runtime, tenant):
1106
  app_id = existing.get("id") if existing is not None else None
1107
  outcome = "existing" if existing is not None else "no_routes"
1108
  if existing is None and route_count:
1109
- if len(doc["applications"]) >= _MAX:
1110
- raise ApplicationError("application limit reached")
 
1111
  app = _build_application(
1112
  tenant, "loopable",
1113
  {"name": "Routing", "templateKind": "route", "type": "route",
1114
- "sourceDatabase": "customer_data",
1115
- "fieldMap": {"title": "customer", "latitude": "lat", "longitude": "lon"},
1116
  "navPlacement": {"section": "app", "order": 0}},
1117
  field_contracts={"customer_data": [
1118
  {"key": "customer", "type": "text"}, {"key": "lat", "type": "text"},
@@ -1130,6 +1233,96 @@ def _ensure_route_seed(runtime, tenant):
1130
  return result
1131
 
1132
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1133
  _COPIES = "userAppCopies"
1134
  _APP_INDEX_KEY = "module_index_app"
1135
 
@@ -1331,7 +1524,7 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie
1331
  doc=_clean_doc(raw); app=doc["applications"].get(app_id)
1332
  if not app: raise ApplicationError("application does not exist")
1333
  if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
1334
- if not _database_free_code(app):
1335
  raise ApplicationError("application does not exist")
1336
  if (app.get("templateKind") in APPLICATION_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS
1337
  and "compatibleDatabases" in patch):
@@ -1366,14 +1559,9 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, fie
1366
  _refuse_legacy_catalog_changes(app, config)
1367
  app["config"] = config
1368
  if "fieldMap" in patch:
1369
- kind = app.get("templateKind")
1370
- if kind not in APPLICATION_WIZARD_TEMPLATE_KINDS:
1371
- raise ApplicationError("only a field-mapped template Application may change fieldMap")
1372
- definition = _template_definition(
1373
- kind, {**app, "fieldMap": patch["fieldMap"]},
1374
- semantic_records=semantic_records, field_contracts=field_contracts,
1375
- allowed_databases=allowed_databases)
1376
- app["fieldMap"] = definition["fieldMap"]
1377
  compatible = (_compat(patch["compatibleDatabases"], allow_empty=app["type"] == "code")
1378
  if "compatibleDatabases" in patch else app.get("compatibleDatabases") or [])
1379
  if _allowed(compatible, allowed_databases) != set(compatible):
@@ -1419,13 +1607,16 @@ def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records
1419
  app=load(runtime)["applications"].get(app_id)
1420
  if not may_read(app,user,is_admin): raise ApplicationError("application does not exist")
1421
  if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
1422
- if not _database_free_code(app):
1423
  raise ApplicationError("application does not exist")
1424
  payload={"type":app["type"],"name":f"{app['name']} copy",
1425
  "compatibleDatabases":app.get("compatibleDatabases"),
1426
  "semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}
1427
- for key in ("templateKind", "sourceDatabase", "fieldMap", "navPlacement", "financeKey"):
 
1428
  if key in app: payload[key]=copy.deepcopy(app[key])
 
 
1429
  return create(runtime,tenant,user,payload,
1430
  semantic_records=semantic_records, field_contracts=field_contracts,
1431
  allowed_databases=allowed_databases, via=via, on_behalf_of=on_behalf_of,
@@ -1435,11 +1626,10 @@ def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records
1435
  _ANY_NAME = object()
1436
 
1437
 
1438
- def _strip_view_refs(doc, app_id):
1439
- """Remove `config.application` naming `app_id` from every View in one workspace document.
1440
 
1441
- Walks every top-level member, reserved ones included, and each owner's trash entries.
1442
- Returns the number of Views changed. Mutates `doc` in place.
1443
  """
1444
  def nested_views(rows):
1445
  for row in (rows.values() if isinstance(rows, dict) else rows if isinstance(rows, list) else ()):
@@ -1448,20 +1638,38 @@ def _strip_view_refs(doc, app_id):
1448
  elif isinstance(row, (dict, list)):
1449
  yield from nested_views(row)
1450
 
1451
- def views_of(bucket):
1452
- views = bucket.get("views")
1453
- yield from nested_views(views)
1454
- trash = bucket.get("trash")
1455
- for entry in (trash.values() if isinstance(trash, dict)
1456
- else trash if isinstance(trash, list) else ()):
 
1457
  if isinstance(entry, dict) and isinstance(entry.get("view"), dict):
1458
  yield entry["view"]
1459
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1460
  cleared = 0
1461
- for bucket in (doc.values() if isinstance(doc, dict) else ()):
1462
- if not isinstance(bucket, dict):
1463
- continue
1464
- for view in views_of(bucket):
1465
  changed = False
1466
  for key in ("config", "presentation"):
1467
  config = view.get(key) if isinstance(view, dict) else None
@@ -1711,7 +1919,7 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None,
1711
  doc=_clean_doc(raw); app=doc["applications"].get(app_id)
1712
  if not app: raise ApplicationError("application does not exist")
1713
  if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
1714
- if not _database_free_code(app):
1715
  raise ApplicationError("application does not exist")
1716
  if not may_edit(app,user,is_admin): raise PermissionError("application belongs to another user")
1717
  if confirm_name is not _ANY_NAME and (
@@ -1732,6 +1940,20 @@ def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None,
1732
  result["viewsCleared"] = cleared
1733
  if failed:
1734
  result["viewsNotCleared"] = "; ".join(failed)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1735
  return result
1736
 
1737
 
 
8
  from core import shares, write_execution
9
  from core.request_timing import raise_if_resource_error
10
  import app_rules
11
+ import app_templates as _app_templates
12
 
13
  from modules.finance_apps import (finance_application as _finance_application,
14
  finance_applications as _finance_applications,
 
20
  # Finance descriptors remain available to the governed fact routes.
21
  FINANCE_TEMPLATE_KIND = "finance"
22
  _KEY = "applications_v1"
23
+ #: D-8392 (R58): the most live Apps one account may create. The cap counts per account, never per tenant.
24
  _MAX = 500
25
  _MAX_DISPLAY_BYTES = 512 * 1024
26
 
 
68
  },
69
  }
70
 
71
+ # C1 (S79-T01): every contract carries its catalogue row's wire metadata.
72
+ _C1_WIRE_KEYS = ("connector", "purpose", "process", "mark", "view")
73
+ # R33: Routing and Catalog read their C1 source themselves, so the wizard asks for no database.
74
+ _BUNDLED_SOURCE_LABELS = {"customer_data": "Odoo customers", "product_data": "Odoo products"}
75
+ for _kind, _contract in _TEMPLATE_CONTRACTS.items():
76
+ _contract.update({key: _app_templates.BY_KIND[_kind][key] for key in _C1_WIRE_KEYS})
77
+ _contract.update(sourceDatabase=_app_templates.source(_kind),
78
+ sourceLabel=_BUNDLED_SOURCE_LABELS[_app_templates.source(_kind)])
79
+
80
  # Keep stored copy identities readable after a template stops accepting new Apps.
81
  APPLICATION_WIZARD_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
82
  APPLICATION_TEMPLATE_KINDS = tuple(_TEMPLATE_CONTRACTS)
83
 
84
  #: S78-T17 (C4, R20, R21): templates offered only to a tenant that passes the kind's gate.
85
  #: They stay out of the ungated tuples above, which other gates pin to Routing and Catalog.
86
+ #: S79-T01 (C1): every connector-gated kind in `app_templates.TEMPLATES` is one of them.
87
  BRAND_ANALYTICS_DATABASE = "ut_amzn_brand_analytics"
88
+ # Wire fields a C1 row does not carry, per gated kind.
89
+ _GATED_WIRE = {
90
  "competitor_watch": {
 
 
 
 
 
91
  "fieldRoles": [
92
  {"key": "term", "label": "Search term", "required": True, "types": []},
93
  ],
 
 
94
  "sourceLabel": "Amazon brand analytics",
95
  },
96
  }
97
+
98
+
99
+ def _source_label(database):
100
+ """The connector table's label, so a refusal never names a `ut_` key."""
101
+ try:
102
+ from core import connector_spec
103
+ return connector_spec.spec_for(database).label
104
+ except Exception: # noqa: BLE001
105
+ return None
106
+
107
+
108
+ def _gated_contract(row):
109
+ # The screen reads the kind's own routes; the source names its connector database.
110
+ contract = {"kind": row["kind"], "label": row["label"],
111
+ "applicationType": "map", "displayMode": "map", "fieldRoles": [],
112
+ "defaultPlacement": {"section": "app", "order": 0},
113
+ **{key: row[key] for key in _C1_WIRE_KEYS}}
114
+ if row["sources"]:
115
+ contract["sourceDatabase"] = row["sources"][0]
116
+ contract["sourceLabel"] = _source_label(row["sources"][0])
117
+ contract.update(copy.deepcopy(_GATED_WIRE.get(row["kind"], {})))
118
+ return contract
119
+
120
+
121
+ _GATED_TEMPLATE_CONTRACTS = {row["kind"]: _gated_contract(row) for row in _app_templates.TEMPLATES
122
+ if row["kind"] not in _TEMPLATE_CONTRACTS}
123
  GATED_TEMPLATE_KINDS = tuple(_GATED_TEMPLATE_CONTRACTS)
124
 
125
 
 
172
  return False
173
 
174
 
175
+ def _connector_gate(connector):
176
+ """C1 gate: the reader has the kind's connector (F25: the connector key, never a `ut_` prefix)."""
177
+ def gate(session):
178
+ return _app_templates.connector_present(connector, session)
179
+ gate.connector = connector
180
+ return gate
181
+
182
+
183
+ # Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
184
+ _TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
185
+ else _connector_gate(contract["connector"])
186
+ for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
187
+ #: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
188
+ _BUNDLED_GATES = {kind: _connector_gate(contract["connector"])
189
+ for kind, contract in _TEMPLATE_CONTRACTS.items()}
190
+
191
+
192
+ def template_offered(kind, session):
193
+ """C1 offer predicate: the module imports, the router mounted and the kind's gate holds."""
194
+ gate = _TEMPLATE_GATES.get(kind) or _BUNDLED_GATES.get(kind)
195
+ if gate is None or _app_templates.held(kind) or not _app_templates.seams_ready(kind):
196
+ return False
197
+ try:
198
+ return bool(gate(session))
199
+ except Exception as exc: # noqa: BLE001
200
+ raise_if_resource_error(exc)
201
+ return False
202
 
203
 
204
  class ApplicationError(ValueError):
205
  pass
206
 
207
 
208
+ def account_capacity():
209
+ """The per-account App cap. `AIOS_APP_CAPACITY` lowers it for one server, as a lane proof does."""
210
+ import os
211
+ raw = str(os.environ.get("AIOS_APP_CAPACITY") or "").strip()
212
+ return min(int(raw), _MAX) if raw.isdigit() else _MAX
213
+
214
+
215
+ def capacity_refusal(doc, user):
216
+ """D-8392: the sentence naming the cap when `user` already holds it, else None."""
217
+ cap = account_capacity()
218
+ held = sum(1 for app in (doc.get("applications") or {}).values()
219
+ if isinstance(app, dict) and app.get("createdBy") == user and app.get("retired") is not True)
220
+ if held < cap:
221
+ return None
222
+ return f"Your account can hold up to {cap} App{'' if cap == 1 else 's'}. Delete an App to add another."
223
+
224
+
225
  class ApplicationRuleError(ApplicationError):
226
  """Expose an App rule refusal through every writer's existing error handler."""
227
  code = "app_rule"
 
251
  if kind == FINANCE_TEMPLATE_KIND:
252
  key = str(app.get("financeKey") or "")
253
  return f"{FINANCE_TEMPLATE_KIND}:{key}" if key else None
254
+ # S79-T05: a connector-gated copy is grouped too, so the App directory can show Added.
255
+ return kind if kind in APPLICATION_WIZARD_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS else None
256
 
257
 
258
  def _own_template_app(applications, user, identity):
 
284
  def template_contracts(session=None):
285
  """The trusted renderer contracts offered by the Application creation flow.
286
 
287
+ With a session, every template is offered only when the reader has its connector (instr 20).
288
  """
289
+ if session is None:
290
+ return _ungated_template_contracts()
291
+ have = _app_templates.connectors_present(session)
292
+ rows = [copy.deepcopy(contract) for contract in _TEMPLATE_CONTRACTS.values()
293
+ if contract["connector"] in have]
294
+ rows.extend(copy.deepcopy(contract) for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()
295
+ if template_offered(kind, session))
296
  return rows
297
 
298
 
 
445
 
446
 
447
  def load(runtime):
448
+ """The read door: a Routing or Catalog App carries its C1 binding (`app_templates.bound`)."""
449
+ try: doc = _clean_doc(runtime.get(_KEY))
450
  except Exception: return blank_doc()
451
+ doc["applications"] = {key: _app_templates.bound(app)
452
+ for key, app in doc["applications"].items()}
453
+ return doc
454
 
455
 
456
  def _server_origin(origin_door=None, via=None):
 
519
  and not app.get("compatibleDatabases"))
520
 
521
 
522
+ def _database_free(app):
523
+ """A code App, or an App whose C1 kind names no source, opens without a database binding."""
524
+ if _database_free_code(app):
525
+ return True
526
+ row = _app_templates.BY_KIND.get(str(app.get("templateKind") or "")) if isinstance(app, dict) else None
527
+ return bool(row and not row["sources"] and not app.get("compatibleDatabases"))
528
+
529
+
530
  def _finance_permission(app):
531
  try:
532
  return _finance_application(str(app.get("financeKey") or ""))["permissionKey"]
 
802
 
803
  def _template_definition(kind, payload, *, semantic_records=None, field_contracts=None,
804
  allowed_databases=None):
805
+ """R33: a template App binds its C1 source and fixed field map; the caller names neither."""
806
  contract = _TEMPLATE_CONTRACTS.get(kind) or _GATED_TEMPLATE_CONTRACTS.get(kind)
807
  if contract is None:
808
  raise ApplicationError("application template kind is not recognized")
809
  template_kind = str(payload.get("templateKind") or "").strip().lower()
810
  if template_kind != kind:
811
  raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
812
+ named = str(payload.get("sourceDatabase") or "").strip()[:120]
813
+ source = _app_templates.source(kind) or named
814
+ # S79-T05: a C1 kind whose `sources` is empty (amazon_operator) reads its own routes, no database.
815
+ sourceless = kind in _app_templates.BY_KIND and not _app_templates.BY_KIND[kind]["sources"]
816
+ if sourceless:
817
+ if named or payload.get("compatibleDatabases"):
818
+ raise ApplicationError(f"{contract['label']} reads no database. Remove the source database.")
819
+ elif not source:
820
  raise ApplicationError("sourceDatabase is required")
821
+ elif named and named != source:
822
  raise ApplicationError(f"{contract['label']} reads "
823
+ f"{contract.get('sourceLabel') or source} only. "
824
  "Choose it as the source database.")
825
+ elif source not in set(allowed_databases or ()):
826
  raise PermissionError("an Application cannot bind a database you cannot open")
827
+ if not sourceless and "compatibleDatabases" in payload:
828
  compatible = _compat(payload.get("compatibleDatabases"))
829
  if compatible != [source]:
830
  raise ApplicationError("a template Application has exactly one sourceDatabase")
831
+ # A legacy caller may still send a field map: the fixed binding replaces it.
832
+ binding = {} if sourceless else _app_templates.field_binding(kind)
833
+ if binding:
834
+ available = _contract_fields(source, semantic_records, field_contracts)
835
+ if not available:
836
+ raise ApplicationError("the source database has no readable field contract")
837
+ unreadable = sorted(role["label"] for role in contract["fieldRoles"]
838
+ if role["required"] and binding.get(role["key"]) not in available)
839
+ if unreadable:
840
+ raise ApplicationError(f"{contract['label']} needs fields your account cannot read: "
841
+ + ", ".join(unreadable) + ". Ask an administrator for access.")
842
+
843
+ raw_placement = payload.get("navPlacement") or contract["defaultPlacement"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
844
  if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
845
  raise ApplicationError("navPlacement must contain section and order")
846
  section = str(raw_placement.get("section") or "").strip().lower()
 
849
  raise ApplicationError("template Applications must be placed in App")
850
  if isinstance(order, bool) or not isinstance(order, int) or not 0 <= order <= 999:
851
  raise ApplicationError("navPlacement order must be an integer from 0 to 999")
852
+ definition = {"templateKind": kind, "compatibleDatabases": [] if sourceless else [source],
853
+ "navPlacement": {"section": "app", "order": order}}
854
+ if kind not in _app_templates.BOUND_KINDS and not sourceless:
855
+ definition.update(sourceDatabase=source, fieldMap=binding)
856
+ return definition
857
+
858
+
859
+ def bound_application(app, available):
860
+ """The App with its field map read against the caller's `{field: type}` schema.
861
+
862
+ A C1 binding drops an optional role the caller cannot read. A stored map stays strict.
863
+ """
864
+ stored = bool(isinstance(app, dict) and app.get("fieldMap"))
865
+ row = _app_templates.bound(app)
866
+ contract = _template_contract(str(row.get("templateKind") or "")) or {}
867
+ roles = {role["key"]: role for role in contract.get("fieldRoles") or ()}
868
+ resolved = {}
869
+ for role, field in (row.get("fieldMap") or {}).items():
870
+ spec = roles.get(role)
871
+ if spec is None:
872
+ raise ApplicationError(f"fieldMap role {role!r} is outside the template contract")
873
+ types = set(spec.get("types") or ())
874
+ if field in available and (not types or available[field] in types):
875
+ resolved[role] = field
876
+ elif stored or spec["required"]:
877
+ raise ApplicationError(f"fieldMap role {role!r} does not name a readable source field")
878
+ return {**row, "fieldMap": resolved}
879
 
880
 
881
  def validate_application(payload, tenant, *, semantic_records=None, field_contracts=None,
 
925
 
926
  def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
927
  is_default=False):
928
+ row=copy.deepcopy(_app_templates.bound(app))
929
  row.setdefault("origin", None)
930
  if row.get("type") == "code":
931
  # Only the authenticated code door returns the body.
 
939
  "default":bool(is_admin)}
940
  row["starred"] = user in (app.get("starredBy") or ())
941
  row["isDefault"] = bool(is_default)
942
+ catalogue = _app_templates.BY_KIND.get(str(row.get("templateKind") or ""))
943
+ if catalogue and catalogue["view"]:
944
+ # C2: the client mounts apps/<templateView>/AppView.tsx and fills the two marks.
945
+ row.update(templateView=catalogue["view"], templateMark=catalogue["mark"],
946
+ templateConnector=catalogue["connector"])
947
  if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
948
  # A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
949
  row["capabilities"]["default"] = False
 
964
  """S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
965
  user's Views in that database's workspace. The Delete App dialog names this count before
966
  confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
967
+
968
+ D-2233: it walks the Views `delete` clears, nested and `presentation` ones too, trash excepted.
969
  """
970
  from core.view_templates import workspace_key
971
  key = workspace_key(database)
 
976
  except Exception:
977
  return None
978
  counts = {}
979
+ users = {member: bucket for member, bucket in (doc.items() if isinstance(doc, dict) else ())
980
+ if member not in _RESERVED_WORKSPACE_MEMBERS}
981
+ for view in _workspace_views(users, trash=False):
982
+ for app_id in _view_app_ids(view):
983
+ counts[app_id] = counts.get(app_id, 0) + 1
 
 
 
 
 
984
  return counts
985
 
986
 
 
990
  doc = load(runtime)
991
  rows=[]
992
  for app in doc["applications"].values():
993
+ if _app_templates.held(app):
994
+ continue
995
  if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
996
  continue
997
  if database:
 
999
  if not may_read(app,user,is_admin): continue
1000
  if database not in (app.get("compatibleDatabases") or ()): continue
1001
  elif not _listed(app, user): continue
1002
+ if (not _database_free(app)
1003
  and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue
1004
  if kind and app.get("type") != kind: continue
1005
  if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
 
1046
  name=" ".join(str(payload.get("name") or "").split())[:120]
1047
  if not name: raise ApplicationError("application name is required")
1048
  app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat()
1049
+ compatible = (list(definition["compatibleDatabases"]) if definition
1050
  else _compat(payload.get("compatibleDatabases"), allow_empty=kind == "code"))
1051
  if _allowed(compatible, allowed_databases) != set(compatible):
1052
  raise PermissionError("an Application cannot bind a database you cannot open")
 
1134
  identity = template_identity(app)
1135
  def apply(raw):
1136
  doc=_clean_doc(raw)
1137
+ refusal = capacity_refusal(doc, user)
1138
+ if refusal: raise ApplicationError(refusal)
1139
  # R216 and R247 allow every template to have several independent copies.
1140
  doc["applications"][app_id]=app; return doc
1141
  _update_with_activity(runtime, app_id, user, apply, via=via, on_behalf_of=on_behalf_of)
 
1166
  """
1167
  result = _ensure_route_seed(runtime, tenant)
1168
  copies = ensure_user_app_copies(runtime, tenant, accounts=accounts, wall=wall)
1169
+ bindings = migrate_c1_bindings(runtime, tenant)
1170
  folded = dict(result)
1171
  folded["userCopies"] = copies
1172
+ folded["c1Bindings"] = bindings
1173
  note = "user copies " + copies["outcome"] + (
1174
  ": " + copies["reason"] if copies.get("reason") else "")
1175
+ note += "; C1 bindings " + bindings["outcome"] + (
1176
+ ": " + bindings["reason"] if bindings.get("reason") else "")
1177
  folded["reason"] = result["reason"] + "; " + note if result.get("reason") else note
1178
  return folded
1179
 
 
1210
  app_id = existing.get("id") if existing is not None else None
1211
  outcome = "existing" if existing is not None else "no_routes"
1212
  if existing is None and route_count:
1213
+ refusal = capacity_refusal(doc, "loopable")
1214
+ if refusal:
1215
+ raise ApplicationError(refusal)
1216
  app = _build_application(
1217
  tenant, "loopable",
1218
  {"name": "Routing", "templateKind": "route", "type": "route",
 
 
1219
  "navPlacement": {"section": "app", "order": 0}},
1220
  field_contracts={"customer_data": [
1221
  {"key": "customer", "type": "text"}, {"key": "lat", "type": "text"},
 
1233
  return result
1234
 
1235
 
1236
+ #: Marker of the R33 migration: the binding each rewritten App held before, and the Apps it kept.
1237
+ C1_BINDINGS_MARKER = "c1Bindings"
1238
+
1239
+
1240
+ def _c1_rewrite(app):
1241
+ """A Routing or Catalog App rewritten to its C1 binding, or None when it must stay as stored.
1242
+
1243
+ An App whose source or field map differs from C1 (`own_binding` non-empty) is never rewritten.
1244
+ """
1245
+ if not isinstance(app, dict) or app.get("templateKind") not in _app_templates.BOUND_KINDS:
1246
+ return None
1247
+ if "sourceDatabase" not in app and "fieldMap" not in app:
1248
+ return None
1249
+ if _app_templates.own_binding(app):
1250
+ return None
1251
+ source = _app_templates.source(app["templateKind"])
1252
+ row = copy.deepcopy(app)
1253
+ row.pop("sourceDatabase", None)
1254
+ row.pop("fieldMap", None)
1255
+ row["compatibleDatabases"] = [source, *(database for database in app.get("compatibleDatabases") or ()
1256
+ if database != source)]
1257
+ return row
1258
+
1259
+
1260
+ def _c1_plan(doc, app_ids):
1261
+ """`{id: rewritten App}` plus the stored Apps that keep a binding of their own, for one document."""
1262
+ rewrites, kept = {}, []
1263
+ for app_id, app in sorted(doc["applications"].items()):
1264
+ if app_ids is not None and app_id not in app_ids:
1265
+ continue
1266
+ if not isinstance(app, dict) or app.get("templateKind") not in _app_templates.BOUND_KINDS:
1267
+ continue
1268
+ rewritten = _c1_rewrite(app)
1269
+ if rewritten is not None:
1270
+ rewrites[app_id] = rewritten
1271
+ elif _app_templates.own_binding(app):
1272
+ kept.append({"id": app_id, "templateKind": app["templateKind"], "name": app.get("name"),
1273
+ "sourceDatabase": app.get("sourceDatabase"),
1274
+ "fieldMap": copy.deepcopy(app.get("fieldMap"))})
1275
+ return rewrites, kept
1276
+
1277
+
1278
+ def migrate_c1_bindings(runtime, tenant, *, app_ids=None):
1279
+ """R33: rewrite every stored Routing and Catalog App to its C1 binding.
1280
+
1281
+ Writes only when an App still stores a binding equal to C1, so a second run changes nothing.
1282
+ An App with its own source or field map is kept and listed. A proof run
1283
+ passes `app_ids` to touch only the named Apps; the boot call never passes it.
1284
+ """
1285
+ scope = None if app_ids is None else set(app_ids)
1286
+ try:
1287
+ doc = _clean_doc(_read_strict(runtime, _KEY))
1288
+ except Exception as exc:
1289
+ return {"outcome": "skipped", "reason": f"registry read failed: {type(exc).__name__}: {exc}",
1290
+ "rewritten": [], "kept": []}
1291
+ rewrites, kept = _c1_plan(doc, scope)
1292
+ result = {"outcome": "unchanged", "rewritten": [], "kept": kept}
1293
+ if rewrites:
1294
+ def apply(raw):
1295
+ current = _clean_doc(raw)
1296
+ planned, held = _c1_plan(current, scope)
1297
+ result["kept"] = held
1298
+ if not planned:
1299
+ return raw
1300
+ marker = current.setdefault("migrations", {}).setdefault(
1301
+ C1_BINDINGS_MARKER, {"rewritten": {}, "kept": []})
1302
+ for app_id, app in planned.items():
1303
+ before = current["applications"][app_id]
1304
+ marker["rewritten"][app_id] = {key: copy.deepcopy(before[key]) for key in
1305
+ ("sourceDatabase", "fieldMap", "compatibleDatabases")
1306
+ if key in before}
1307
+ current["applications"][app_id] = app
1308
+ marker["kept"] = sorted({row["id"] for row in held} | set(marker.get("kept") or ()))
1309
+ marker["at"] = write_execution.utcnow().isoformat()
1310
+ result.update(outcome="migrated", rewritten=sorted(planned))
1311
+ return current
1312
+ _update_document(runtime, apply)
1313
+ if result["rewritten"]:
1314
+ try:
1315
+ from catalog_store import migrate_snapshots
1316
+ result["catalogSnapshots"] = migrate_snapshots(runtime, result["rewritten"])
1317
+ except Exception as exc:
1318
+ result["catalogSnapshots"] = f"skipped: {type(exc).__name__}"
1319
+ parts = [f"{len(result['rewritten'])} rewritten"] if result["rewritten"] else []
1320
+ if result["kept"]:
1321
+ parts.append("kept with their own binding: " + ", ".join(row["id"] for row in result["kept"]))
1322
+ result["reason"] = "; ".join(parts)
1323
+ return result
1324
+
1325
+
1326
  _COPIES = "userAppCopies"
1327
  _APP_INDEX_KEY = "module_index_app"
1328
 
 
1524
  doc=_clean_doc(raw); app=doc["applications"].get(app_id)
1525
  if not app: raise ApplicationError("application does not exist")
1526
  if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
1527
+ if not _database_free(app):
1528
  raise ApplicationError("application does not exist")
1529
  if (app.get("templateKind") in APPLICATION_TEMPLATE_KINDS + GATED_TEMPLATE_KINDS
1530
  and "compatibleDatabases" in patch):
 
1559
  _refuse_legacy_catalog_changes(app, config)
1560
  app["config"] = config
1561
  if "fieldMap" in patch:
1562
+ # R33: a template App reads the fields its template names.
1563
+ raise ApplicationError("This App reads the fields its template names. "
1564
+ "Its field map cannot change.")
 
 
 
 
 
1565
  compatible = (_compat(patch["compatibleDatabases"], allow_empty=app["type"] == "code")
1566
  if "compatibleDatabases" in patch else app.get("compatibleDatabases") or [])
1567
  if _allowed(compatible, allowed_databases) != set(compatible):
 
1607
  app=load(runtime)["applications"].get(app_id)
1608
  if not may_read(app,user,is_admin): raise ApplicationError("application does not exist")
1609
  if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
1610
+ if not _database_free(app):
1611
  raise ApplicationError("application does not exist")
1612
  payload={"type":app["type"],"name":f"{app['name']} copy",
1613
  "compatibleDatabases":app.get("compatibleDatabases"),
1614
  "semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}
1615
+ # R33: a template copy binds C1 itself, so neither the source nor the field map is copied.
1616
+ for key in ("templateKind", "navPlacement", "financeKey"):
1617
  if key in app: payload[key]=copy.deepcopy(app[key])
1618
+ if app.get("templateKind") not in _app_templates.BOUND_KINDS and "sourceDatabase" in app:
1619
+ payload["sourceDatabase"] = app["sourceDatabase"]
1620
  return create(runtime,tenant,user,payload,
1621
  semantic_records=semantic_records, field_contracts=field_contracts,
1622
  allowed_databases=allowed_databases, via=via, on_behalf_of=on_behalf_of,
 
1626
  _ANY_NAME = object()
1627
 
1628
 
1629
+ def _workspace_views(doc, *, trash=True):
1630
+ """Every View in one workspace document: each member's Views, reserved members included.
1631
 
1632
+ Descends into nested folders. With `trash`, each owner's trash entries are walked too.
 
1633
  """
1634
  def nested_views(rows):
1635
  for row in (rows.values() if isinstance(rows, dict) else rows if isinstance(rows, list) else ()):
 
1638
  elif isinstance(row, (dict, list)):
1639
  yield from nested_views(row)
1640
 
1641
+ for bucket in (doc.values() if isinstance(doc, dict) else ()):
1642
+ if not isinstance(bucket, dict):
1643
+ continue
1644
+ yield from nested_views(bucket.get("views"))
1645
+ held = bucket.get("trash") if trash else None
1646
+ for entry in (held.values() if isinstance(held, dict)
1647
+ else held if isinstance(held, list) else ()):
1648
  if isinstance(entry, dict) and isinstance(entry.get("view"), dict):
1649
  yield entry["view"]
1650
 
1651
+
1652
+ def _view_app_ids(view):
1653
+ """The distinct App ids a View's `config` or `presentation` names."""
1654
+ ids = []
1655
+ for key in ("config", "presentation"):
1656
+ config = view.get(key) if isinstance(view, dict) else None
1657
+ ref = config.get("application") if isinstance(config, dict) else None
1658
+ app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
1659
+ if app_id and app_id not in ids:
1660
+ ids.append(app_id)
1661
+ return ids
1662
+
1663
+
1664
+ def _strip_view_refs(doc, app_id):
1665
+ """Remove `config.application` naming `app_id` from every View in one workspace document.
1666
+
1667
+ Walks every top-level member, reserved ones included, and each owner's trash entries.
1668
+ Returns the number of Views changed. Mutates `doc` in place.
1669
+ """
1670
  cleared = 0
1671
+ if isinstance(doc, dict):
1672
+ for view in _workspace_views(doc):
 
 
1673
  changed = False
1674
  for key in ("config", "presentation"):
1675
  config = view.get(key) if isinstance(view, dict) else None
 
1919
  doc=_clean_doc(raw); app=doc["applications"].get(app_id)
1920
  if not app: raise ApplicationError("application does not exist")
1921
  if not _is_finance(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases):
1922
+ if not _database_free(app):
1923
  raise ApplicationError("application does not exist")
1924
  if not may_edit(app,user,is_admin): raise PermissionError("application belongs to another user")
1925
  if confirm_name is not _ANY_NAME and (
 
1940
  result["viewsCleared"] = cleared
1941
  if failed:
1942
  result["viewsNotCleared"] = "; ".join(failed)
1943
+ kind = removed.get("templateKind")
1944
+ contract = _app_templates.BY_KIND.get(kind)
1945
+ if contract and contract.get("module"):
1946
+ try:
1947
+ if not any(row.get("templateKind") == kind for row in load(runtime)["applications"].values()):
1948
+ from importlib import import_module
1949
+ module = import_module(contract["module"])
1950
+ cleanup = next((getattr(module, name) for name in
1951
+ ("remove_app_data", "on_app_removed", "remove")
1952
+ if callable(getattr(module, name, None))), None)
1953
+ if cleanup is not None:
1954
+ cleanup(removed.get("tenant") or runtime.key, st=runtime)
1955
+ except Exception as exc:
1956
+ result["appDataNotCleared"] = type(exc).__name__
1957
  return result
1958
 
1959
 
api/assistant_stream.py CHANGED
@@ -1,10 +1,10 @@
1
  """Expose one Query turn as server-sent events with same-turn error recovery."""
2
  import asyncio
3
  import contextvars
4
- import gzip
5
- import io
6
  import json
7
  import threading
 
8
 
9
  import anyio
10
 
@@ -39,34 +39,24 @@ class StreamGZipMiddleware:
39
  await responder(scope, receive, send)
40
 
41
 
42
- from starlette.middleware.gzip import GZipResponder, IdentityResponder
43
 
44
 
45
- class _OffLoopGZipResponder(GZipResponder):
46
  """Retain Starlette's header and chunk behavior with compression outside the event loop."""
47
 
 
 
48
  def __init__(self, app, minimum_size, compresslevel):
49
- IdentityResponder.__init__(self, app, minimum_size)
50
  self.compresslevel = compresslevel
51
- self.gzip_file = None
52
-
53
- async def __call__(self, scope, receive, send):
54
- try:
55
- await IdentityResponder.__call__(self, scope, receive, send)
56
- finally:
57
- if self.gzip_file is not None:
58
- def close():
59
- self.gzip_file.close()
60
- self.gzip_buffer.close()
61
- with anyio.CancelScope(shield=True):
62
- await anyio.to_thread.run_sync(close)
63
 
64
  def _compress(self, body, more):
65
- if self.gzip_file is None:
66
- self.gzip_buffer = io.BytesIO()
67
- self.gzip_file = gzip.GzipFile(mode="wb", fileobj=self.gzip_buffer,
68
- compresslevel=self.compresslevel)
69
- return GZipResponder.apply_compression(self, body, more_body=more)
70
 
71
  async def send_with_compression(self, message):
72
  if (message["type"] == "http.response.body"
@@ -77,8 +67,13 @@ class _OffLoopGZipResponder(GZipResponder):
77
  lambda: self._compress(body, more))
78
  await super().send_with_compression(message)
79
 
80
- def apply_compression(self, body, *, more_body):
81
- return self._compressed_body
 
 
 
 
 
82
 
83
 
84
  def _event(name, data):
 
1
  """Expose one Query turn as server-sent events with same-turn error recovery."""
2
  import asyncio
3
  import contextvars
4
+ import inspect
 
5
  import json
6
  import threading
7
+ import zlib
8
 
9
  import anyio
10
 
 
39
  await responder(scope, receive, send)
40
 
41
 
42
+ from starlette.middleware.gzip import IdentityResponder
43
 
44
 
45
+ class _OffLoopGZipResponder(IdentityResponder):
46
  """Retain Starlette's header and chunk behavior with compression outside the event loop."""
47
 
48
+ content_encoding = "gzip"
49
+
50
  def __init__(self, app, minimum_size, compresslevel):
51
+ super().__init__(app, minimum_size)
52
  self.compresslevel = compresslevel
53
+ self._compressor = None
 
 
 
 
 
 
 
 
 
 
 
54
 
55
  def _compress(self, body, more):
56
+ if self._compressor is None:
57
+ self._compressor = zlib.compressobj(self.compresslevel, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
58
+ out = self._compressor.compress(body)
59
+ return out if more else out + self._compressor.flush()
 
60
 
61
  async def send_with_compression(self, message):
62
  if (message["type"] == "http.response.body"
 
67
  lambda: self._compress(body, more))
68
  await super().send_with_compression(message)
69
 
70
+ # Starlette 1.4 awaits apply_compression and 1.3 calls it, so match the installed base.
71
+ if inspect.iscoroutinefunction(IdentityResponder.apply_compression):
72
+ async def apply_compression(self, body, *, more_body):
73
+ return self._compressed_body
74
+ else:
75
+ def apply_compression(self, body, *, more_body):
76
+ return self._compressed_body
77
 
78
 
79
  def _event(name, data):
api/catalog_store.py CHANGED
@@ -7,6 +7,7 @@ import hashlib
7
  import json
8
 
9
  import app_rules
 
10
  import application_registry as applications
11
  from core import write_execution
12
  from deps import err
@@ -116,7 +117,33 @@ def normalize_pages(pages, *, legacy=False, identity=""):
116
 
117
 
118
  def _snapshot(app):
119
- return {key: copy.deepcopy(value) for key, value in app.items() if key in _SNAPSHOT_KEYS}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
 
121
 
122
  def _insert(doc, record):
@@ -194,7 +221,8 @@ def require_app(session, app_id, *, write=False, allow_orphan=False):
194
  may = applications.may_edit if write else applications.may_read
195
  if not may(app, session.uname, session.admin):
196
  raise err(403, "catalog_forbidden", "Your account cannot access this Catalog App. Ask its owner for access.")
197
- source = app.get("sourceDatabase")
 
198
  _assert_database_scope(session, source)
199
  session.require(source)
200
  if not perm_scope.may_read(session.user, source, st=runtime):
@@ -202,12 +230,12 @@ def require_app(session, app_id, *, write=False, allow_orphan=False):
202
  if write:
203
  _assert_write_scope(session, "catalogs")
204
  contracts = applications.chart_field_contracts(session, [source])
 
205
  try:
206
- applications._template_definition("catalog", app, field_contracts=contracts,
207
- allowed_databases={source})
208
  except applications.ApplicationError:
209
  raise err(403, "catalog_fields_forbidden", "The Catalog App binds unavailable fields. Ask its owner to rebind readable fields.") from None
210
- return app, applications._contract_fields(source, field_contracts=contracts)
211
 
212
 
213
  def _binding_wall(record, app, fields):
 
7
  import json
8
 
9
  import app_rules
10
+ import app_templates
11
  import application_registry as applications
12
  from core import write_execution
13
  from deps import err
 
117
 
118
 
119
  def _snapshot(app):
120
+ # R33: a snapshot keeps a source or field map only when it differs from the C1 binding.
121
+ kept = _SNAPSHOT_KEYS - {"sourceDatabase", "fieldMap"}
122
+ row = {key: copy.deepcopy(value) for key, value in app.items() if key in kept}
123
+ row.update(copy.deepcopy(app_templates.own_binding(app)))
124
+ return row
125
+
126
+
127
+ def migrate_snapshots(runtime, app_ids):
128
+ """R33: drop a C1 binding from the named Apps' catalog snapshots. Writes only on a change."""
129
+ wanted = set(app_ids)
130
+
131
+ def stale(doc):
132
+ return sorted(app_id for app_id, row in doc["appSnapshots"].items()
133
+ if app_id in wanted and isinstance(row, dict)
134
+ and applications._c1_rewrite(row) is not None)
135
+
136
+ if not stale(_load(runtime)):
137
+ return "unchanged"
138
+
139
+ def apply(doc):
140
+ changed = stale(doc)
141
+ for app_id in changed:
142
+ row = applications._c1_rewrite(doc["appSnapshots"][app_id])
143
+ doc["appSnapshots"][app_id] = {key: value for key, value in row.items() if key in _SNAPSHOT_KEYS}
144
+ return changed
145
+
146
+ return f"{len(_update(runtime, apply))} rewritten"
147
 
148
 
149
  def _insert(doc, record):
 
221
  may = applications.may_edit if write else applications.may_read
222
  if not may(app, session.uname, session.admin):
223
  raise err(403, "catalog_forbidden", "Your account cannot access this Catalog App. Ask its owner for access.")
224
+ # R33: Catalog reads Odoo products through its C1 binding unless the record holds its own.
225
+ source = app_templates.bound(app).get("sourceDatabase")
226
  _assert_database_scope(session, source)
227
  session.require(source)
228
  if not perm_scope.may_read(session.user, source, st=runtime):
 
230
  if write:
231
  _assert_write_scope(session, "catalogs")
232
  contracts = applications.chart_field_contracts(session, [source])
233
+ fields = applications._contract_fields(source, field_contracts=contracts)
234
  try:
235
+ app = applications.bound_application(app, fields)
 
236
  except applications.ApplicationError:
237
  raise err(403, "catalog_fields_forbidden", "The Catalog App binds unavailable fields. Ask its owner to rebind readable fields.") from None
238
+ return app, fields
239
 
240
 
241
  def _binding_wall(record, app, fields):
api/connectors_amazon_ads.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """connectors_amazon_ads.py: the Amazon Ads connector's API layer (S79-T13): the probe and the card state.
2
+
3
+ The transport is `platform/harness/amazon_ads_client.py`. This module adds the liveness probe the
4
+ keychain calls through `register_prober`, and the card state the Connectors directory reads.
5
+
6
+ ⛔ READ ONLY. No route here writes. The probe makes one Login with Amazon token exchange and one
7
+ profile listing GET, and only when an administrator tests a stored credential.
8
+ ⛔ `AMAZON_ADS_FIXTURE=1` answers from the fixture tokens below and opens no socket.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import sys
14
+ import urllib.request
15
+ from pathlib import Path
16
+
17
+ _PLATFORM = Path(__file__).resolve().parents[2] / "platform"
18
+ if str(_PLATFORM) not in sys.path:
19
+ sys.path.insert(0, str(_PLATFORM))
20
+
21
+ from harness import amazon_ads_client as client # noqa: E402
22
+
23
+ KEY = client.KEY
24
+ LABEL = client.LABEL
25
+ #: Must be a `core.keychain.ENTRY_TYPES` member, or storing the credential 400s.
26
+ ENTRY_TYPE = client.ENTRY_TYPE
27
+ CREDENTIAL_FIELDS = client.CREDENTIAL_FIELDS
28
+
29
+ #: The one read the probe makes after the token exchange. GET only, one path.
30
+ PROFILES_PATH = "v2/profiles"
31
+
32
+ #: Fixture refresh tokens. A token with the prefix passes; the expired one fails.
33
+ FIXTURE_TOKEN_PREFIX = "Atzr|fixture-ads"
34
+ FIXTURE_REFRESH_TOKEN = FIXTURE_TOKEN_PREFIX + "-ok"
35
+ FIXTURE_EXPIRED_TOKEN = FIXTURE_TOKEN_PREFIX + "-expired"
36
+ FIXTURE_PROFILE_ID = "1234567890"
37
+
38
+ #: Messages a person reads. No dash, no provider text.
39
+ OK_MESSAGE = "Amazon Ads confirmed profile access."
40
+ EXPIRED_MESSAGE = "The Amazon Ads token expired. Connect again."
41
+ NO_PROFILE_MESSAGE = "The token does not reach this profile. Check the profile ID."
42
+ NOT_FIXTURE_MESSAGE = "Fixture mode accepts only the fixture token."
43
+ SIGN_IN_MESSAGE = "Amazon Ads did not accept the sign in. Check the fields and connect again."
44
+
45
+
46
+ def fixture_fields(*, expired=False):
47
+ """The fixture credential an administrator pastes on staging. -> the four fields."""
48
+ return {"client_id": "amzn1.application-oa2-client.fixture",
49
+ "client_secret": "fixture-ads-secret",
50
+ "refresh_token": FIXTURE_EXPIRED_TOKEN if expired else FIXTURE_REFRESH_TOKEN,
51
+ "profile_id": FIXTURE_PROFILE_ID}
52
+
53
+
54
+ def missing_fields(fields):
55
+ """The credential fields that are absent or blank, in declaration order."""
56
+ f = fields if isinstance(fields, dict) else {}
57
+ return [key for key in CREDENTIAL_FIELDS if not isinstance(f.get(key), str) or not f[key].strip()]
58
+
59
+
60
+ def _fixture_probe(fields):
61
+ token = fields["refresh_token"].strip()
62
+ if token == FIXTURE_EXPIRED_TOKEN:
63
+ return {"ok": False, "message": EXPIRED_MESSAGE}
64
+ if not token.startswith(FIXTURE_TOKEN_PREFIX):
65
+ return {"ok": False, "message": NOT_FIXTURE_MESSAGE}
66
+ return {"ok": True, "message": OK_MESSAGE}
67
+
68
+
69
+ def _profiles(access_token, fields, timeout):
70
+ """The advertiser's profile ids. One GET; any failure raises `client.AdsError`."""
71
+ host = client.HOSTS.get(str(fields.get("region") or client.DEFAULT_REGION).lower(),
72
+ client.HOSTS[client.DEFAULT_REGION])
73
+ req = urllib.request.Request(f"{host}/{PROFILES_PATH}", method="GET", headers={
74
+ "Authorization": f"Bearer {access_token}",
75
+ "Amazon-Advertising-API-ClientId": fields["client_id"], "Accept": "application/json"})
76
+ try:
77
+ with urllib.request.urlopen(req, timeout=timeout) as r:
78
+ body = json.loads(r.read().decode("utf-8", "replace") or "[]")
79
+ except Exception as exc: # noqa: BLE001
80
+ raise client.AdsError(f"Amazon Ads profiles could not be read ({type(exc).__name__})") from None
81
+ return {str(p.get("profileId")) for p in body if isinstance(p, dict)} if isinstance(body, list) else set()
82
+
83
+
84
+ def probe_credential(fields, timeout=8):
85
+ """`{ok, message}` for a stored `amazon_ads` credential: the keychain's liveness probe."""
86
+ f = fields if isinstance(fields, dict) else {}
87
+ missing = missing_fields(f)
88
+ if missing:
89
+ return {"ok": False, "message": "Missing " + ", ".join(missing) + ". Enter every field."}
90
+ if client.fixture_mode():
91
+ return _fixture_probe(f)
92
+ try:
93
+ profiles = _profiles(client._token(f), f, timeout)
94
+ except client.AdsError:
95
+ return {"ok": False, "message": SIGN_IN_MESSAGE}
96
+ if f["profile_id"].strip() not in profiles:
97
+ return {"ok": False, "message": NO_PROFILE_MESSAGE}
98
+ return {"ok": True, "message": OK_MESSAGE}
99
+
100
+
101
+ def register():
102
+ """Fill the keychain's probe slot for `amazon_ads`. Never raises."""
103
+ try:
104
+ import core.keychain as kc
105
+ return kc.register_prober(ENTRY_TYPE, probe_credential)
106
+ except Exception: # noqa: BLE001
107
+ return False
108
+
109
+
110
+ def held(rt, entry_id):
111
+ """True when this entry's current ciphertext carries a passing probe proof. No decrypt, no I/O.
112
+
113
+ A failed test, an expired token or a replaced secret clears the proof, so the card reads Connect.
114
+ """
115
+ try:
116
+ from core import keychain as kc
117
+ return bool(kc.entry_verified(rt, entry_id))
118
+ except Exception: # noqa: BLE001
119
+ return False
120
+
121
+
122
+ def card_state(verified):
123
+ """The directory state: `connected` (Settings) only with a verified credential, else `available`."""
124
+ return "connected" if verified else "available"
125
+
126
+
127
+ __all__ = ["KEY", "LABEL", "ENTRY_TYPE", "CREDENTIAL_FIELDS", "PROFILES_PATH", "FIXTURE_TOKEN_PREFIX",
128
+ "FIXTURE_REFRESH_TOKEN", "FIXTURE_EXPIRED_TOKEN", "FIXTURE_PROFILE_ID", "OK_MESSAGE",
129
+ "EXPIRED_MESSAGE", "fixture_fields", "missing_fields", "probe_credential", "register",
130
+ "held", "card_state"]
api/deps.py CHANGED
@@ -1661,3 +1661,60 @@ def module_gate(module_key):
1661
  session.require(module_key)
1662
  return session
1663
  return _dep
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1661
  session.require(module_key)
1662
  return session
1663
  return _dep
1664
+
1665
+
1666
+ def _bearer_header(request: Request):
1667
+ value = request.headers.get("authorization")
1668
+ return value if isinstance(value, str) and value.strip() else None
1669
+
1670
+
1671
+ def _cookie_session(request: Request, response: Response) -> Session:
1672
+ """`require_session`, or the app's override of it, so a gate's override still applies here."""
1673
+ app = request.scope.get("app")
1674
+ override = (getattr(app, "dependency_overrides", None) or {}).get(require_session)
1675
+ if override is None:
1676
+ return require_session(request, response)
1677
+ import inspect
1678
+ names = inspect.signature(override).parameters
1679
+ return override(**{key: value for key, value in (("request", request), ("response", response))
1680
+ if key in names})
1681
+
1682
+
1683
+ def require_session_or_bearer(request: Request, response: Response) -> Session:
1684
+ """The cookie session when a cookie is sent, else an external bearer under its own key's grants.
1685
+
1686
+ `require_session` stays cookie only. A bearer session never reissues the cookie.
1687
+ """
1688
+ bearer = _bearer_header(request)
1689
+ if bearer is None or request.cookies.get(aios_session.COOKIE_NAME):
1690
+ try:
1691
+ return _cookie_session(request, response)
1692
+ except HTTPException as exc:
1693
+ if bearer is None or exc.status_code != 401:
1694
+ raise
1695
+ import routes_external # routes_external imports deps, so this import stays lazy.
1696
+ return routes_external.require_external_analyst(
1697
+ routes_external.require_external_session(bearer, request))
1698
+
1699
+
1700
+ def is_bearer(session) -> bool:
1701
+ """True for a session an external API key or OAuth token resolved."""
1702
+ return (getattr(session, "claims", None) or {}).get("external") is True
1703
+
1704
+
1705
+ def require_bearer_write(session, tool_name):
1706
+ """Refuse a read-only or refresh-only credential on a write route. A cookie session passes."""
1707
+ if is_bearer(session):
1708
+ import routes_external
1709
+ routes_external._assert_write_scope(session, tool_name)
1710
+
1711
+
1712
+ def module_gate_or_bearer(module_key, database=None):
1713
+ """`module_gate` that also admits a bearer, walled to its key's database scope."""
1714
+ def _dep(session: Session = Depends(require_session_or_bearer)) -> Session:
1715
+ session.require(module_key)
1716
+ if is_bearer(session):
1717
+ import routes_external
1718
+ routes_external._assert_database_scope(session, database or module_key)
1719
+ return session
1720
+ return _dep
api/error_sink.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Store secret-free exception groups and send one daily operations digest."""
2
+ from __future__ import annotations
3
+
4
+ from datetime import date, datetime, timedelta, timezone
5
+ from functools import partial
6
+ import hashlib
7
+ import json
8
+ import logging
9
+ from pathlib import Path
10
+ import re
11
+ import sys
12
+ from uuid import uuid4
13
+
14
+ LOG = logging.getLogger(__name__)
15
+
16
+
17
+ def utc_now():
18
+ return datetime.now(timezone.utc)
19
+
20
+
21
+ def store_backend():
22
+ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "platform"))
23
+ import core.store as store
24
+ return store
25
+
26
+
27
+ def day_key(day):
28
+ return "ops_error_sink_" + date.fromisoformat(day).isoformat()
29
+
30
+
31
+ def stack_hash(error):
32
+ frames = []
33
+ trace = error.__traceback__
34
+ while trace:
35
+ code = trace.tb_frame.f_code
36
+ frames.append((code.co_filename, code.co_name, trace.tb_lineno))
37
+ trace = trace.tb_next
38
+ signature = (type(error).__module__, type(error).__qualname__, frames)
39
+ return hashlib.sha256(json.dumps(signature).encode()).hexdigest()
40
+
41
+
42
+ def verified_tenant(scope):
43
+ from starlette.requests import Request
44
+ request = Request(scope)
45
+ authorization = request.headers.get("authorization")
46
+ if authorization:
47
+ external = sys.modules.get("api.routes_external") or sys.modules.get("routes_external")
48
+ if external is None:
49
+ return "unknown"
50
+ try:
51
+ value = external.require_external_session(authorization=authorization, request=request).tenant
52
+ except Exception:
53
+ return "unknown"
54
+ else:
55
+ import aios_session
56
+ claims = aios_session.read(request.cookies.get(aios_session.COOKIE_NAME, ""))
57
+ value = (claims or {}).get("t", "unknown")
58
+ return value if isinstance(value, str) and re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", value) else "unknown"
59
+
60
+
61
+ def record_error(scope, error, *, status=500, backend=None, now=None, tenant_resolver=None):
62
+ backend = backend if backend is not None else store_backend()
63
+ now = now or utc_now()
64
+ if now.utcoffset() is None:
65
+ raise ValueError("error timestamps must include a timezone")
66
+ now = now.astimezone(timezone.utc)
67
+ route = getattr(scope.get("route"), "path", None) or scope.get("_timing_route") or "<unmatched>"
68
+ tenant = (tenant_resolver or verified_tenant)(scope)
69
+ event = {"route": route, "status": int(status), "tenant": tenant, "stack_hash": stack_hash(error)}
70
+ group = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest()
71
+ stamp = now.isoformat()
72
+
73
+ def update(document):
74
+ document = document or {"events": {}}
75
+ events = document.setdefault("events", {})
76
+ previous = events.get(group, {})
77
+ events[group] = {**event, "count": previous.get("count", 0) + 1,
78
+ "first": previous.get("first", stamp), "last": stamp}
79
+ return document
80
+
81
+ backend.update(day_key(now.date().isoformat()), update)
82
+ return group
83
+
84
+
85
+ class ErrorSinkMiddleware:
86
+ def __init__(self, app, *, backend=None, tenant_resolver=None, clock=None):
87
+ self.app, self.backend = app, backend
88
+ self.tenant_resolver, self.clock = tenant_resolver, clock or utc_now
89
+
90
+ async def __call__(self, scope, receive, send):
91
+ if scope["type"] != "http":
92
+ return await self.app(scope, receive, send)
93
+ try:
94
+ return await self.app(scope, receive, send)
95
+ except Exception as error:
96
+ if not scope.get("_error_sink_recorded"):
97
+ scope["_error_sink_recorded"] = True
98
+ try:
99
+ from starlette.concurrency import run_in_threadpool
100
+ await run_in_threadpool(partial(record_error, scope, error, backend=self.backend,
101
+ now=self.clock(), tenant_resolver=self.tenant_resolver))
102
+ except Exception:
103
+ LOG.error("Error sink persistence failed; original request exception preserved")
104
+ raise
105
+
106
+
107
+ def daily_digest(*, sender, day=None, backend=None, now=None):
108
+ backend = backend if backend is not None else store_backend()
109
+ now = (now or utc_now()).astimezone(timezone.utc)
110
+ day = day or (now.date() - timedelta(days=1)).isoformat()
111
+ key, claim = day_key(day), uuid4().hex
112
+
113
+ def acquire(document):
114
+ document = document or {"events": {}}
115
+ previous = document.get("digest") or {}
116
+ if not document.get("events") or previous.get("state") in {"sent", "sending", "uncertain"}:
117
+ return document
118
+ document["digest"] = {"state": "sending", "claim": claim, "at": now.isoformat()}
119
+ return document
120
+
121
+ document = backend.update(key, acquire)
122
+ receipt = document.get("digest") or {}
123
+ if receipt.get("claim") != claim:
124
+ pending = receipt.get("state") in {"sending", "uncertain"}
125
+ return {"ok": not pending, "sent": False, "day": day,
126
+ "reason": "delivery_needs_reconciliation" if pending else "empty_or_already_sent"}
127
+ events = sorted(document["events"].values(), key=lambda event: (-event["count"], event["stack_hash"]))
128
+ total = sum(event["count"] for event in events)
129
+ lines = [f"Loopable errors for {day}: {total} events in {len(events)} groups"]
130
+ for event in events:
131
+ line = f"{event['count']} x {event['status']} {event['route']} [{event['tenant']}] {event['stack_hash'][:12]}"
132
+ if len(("\n".join(lines) + "\n" + line).encode()) > 3500:
133
+ lines.append(f"Remaining groups: {len(events) - len(lines) + 1}. Full groups are in the error sink.")
134
+ break
135
+ lines.append(line)
136
+
137
+ def finish(current, state):
138
+ if (current.get("digest") or {}).get("claim") == claim:
139
+ current["digest"] = {"state": state, "claim": claim, "at": now.isoformat()}
140
+ return current
141
+
142
+ try:
143
+ delivered = sender("\n".join(lines))
144
+ except Exception:
145
+ backend.update(key, lambda current: finish(current, "uncertain"))
146
+ raise
147
+ if delivered is False:
148
+ backend.update(key, lambda current: finish(current, "failed"))
149
+ raise RuntimeError("digest sender refused delivery")
150
+ backend.update(key, lambda current: finish(current, "sent"))
151
+ return {"sent": True, "day": day, "groups": len(events), "events": total}
api/external_activity.py CHANGED
@@ -3,12 +3,20 @@
3
  from contextlib import contextmanager
4
  from contextvars import ContextVar
5
  import json
 
 
6
  from core import audit, external_access
7
 
8
 
9
  RETAINED_MAX = audit.MAX_EVENTS
10
  _CALL = ContextVar("external_activity_call", default=None)
11
 
 
 
 
 
 
 
12
 
13
  def _preview(session):
14
  return bool(getattr(session.runtime, "preview_only", False)
@@ -179,13 +187,7 @@ def _text(value):
179
  def _detail(session, database, outcome, code):
180
  claims = session.claims or {}
181
  key_id = _text(claims.get("external_key_id") or claims.get("external_credential_id"))
182
- label = ""
183
- if key_id:
184
- try:
185
- rows = external_access.list_for_user(session.runtime, session.uname, session.tenant)
186
- label = next((row.get("label") for row in rows if row.get("id") == key_id), "")
187
- except external_access.ExternalAccessError:
188
- pass
189
  return {
190
  "key_id": key_id, "key_label": _text(label), "database": _text(database),
191
  "outcome": outcome, "code": _text(code),
@@ -193,6 +195,30 @@ def _detail(session, database, outcome, code):
193
  }
194
 
195
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
196
  def _record(session, database, outcome, code, *, tool="query_database", action="query", via=None, call=None,
197
  method=None):
198
  if _preview(session) or _CALL.get() is not None:
 
3
  from contextlib import contextmanager
4
  from contextvars import ContextVar
5
  import json
6
+ import threading
7
+ import time
8
  from core import audit, external_access
9
 
10
 
11
  RETAINED_MAX = audit.MAX_EVENTS
12
  _CALL = ContextVar("external_activity_call", default=None)
13
 
14
+ #: (tenant, user, key id) -> (label, monotonic stamp). A rename shows in the audit within the TTL.
15
+ _LABELS = {}
16
+ _LABELS_LOCK = threading.Lock()
17
+ LABEL_TTL_SECONDS = 60.0
18
+ LABEL_CACHE_MAX = 512
19
+
20
 
21
  def _preview(session):
22
  return bool(getattr(session.runtime, "preview_only", False)
 
187
  def _detail(session, database, outcome, code):
188
  claims = session.claims or {}
189
  key_id = _text(claims.get("external_key_id") or claims.get("external_credential_id"))
190
+ label = _key_label(session, key_id) if key_id else ""
 
 
 
 
 
 
191
  return {
192
  "key_id": key_id, "key_label": _text(label), "database": _text(database),
193
  "outcome": outcome, "code": _text(code),
 
195
  }
196
 
197
 
198
+ def _key_label(session, key_id):
199
+ """The key's label, read from the store at most once per key per TTL in this process."""
200
+ owner = (str(session.tenant or "").lower(), str(session.uname or "").lower())
201
+ now = time.monotonic()
202
+ with _LABELS_LOCK:
203
+ hit = _LABELS.get((*owner, key_id))
204
+ if hit is not None and now - hit[1] < LABEL_TTL_SECONDS:
205
+ return hit[0]
206
+ try:
207
+ rows = external_access.list_for_user(session.runtime, session.uname, session.tenant)
208
+ except external_access.ExternalAccessError:
209
+ return ""
210
+ found = {str(row.get("id")): str(row.get("label") or "") for row in rows if row.get("id")}
211
+ with _LABELS_LOCK:
212
+ if len(_LABELS) + len(found) + 1 > LABEL_CACHE_MAX:
213
+ _LABELS.clear()
214
+ for row_id, row_label in found.items():
215
+ _LABELS[(*owner, row_id)] = (row_label, now)
216
+ # A key with no row (an OAuth credential, a revoked key) caches its miss for the TTL too.
217
+ if key_id not in found:
218
+ _LABELS[(*owner, key_id)] = ("", now)
219
+ return found.get(key_id, "")
220
+
221
+
222
  def _record(session, database, outcome, code, *, tool="query_database", action="query", via=None, call=None,
223
  method=None):
224
  if _preview(session) or _CALL.get() is not None:
api/external_freshness.py CHANGED
@@ -67,6 +67,21 @@ _AMAZON_DERIVED_INPUTS = {"sku_keywords": ("listing_items", "search_query_perfor
67
  _AMAZON_DERIVED_RECEIPTS = {"keyword_ranks": "amzn_apify_keyword_ranks"}
68
 
69
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  def amazon_sync_state(cur):
71
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
72
  return {str(entity): at for entity, at in cur.execute(
@@ -94,9 +109,7 @@ def amazon_sync_stamp(cur, table_key, state=None):
94
  R11: a refusal overwrites the raw receipt and a pass with no window writes no mark, so the
95
  reconciliation proofs also count. The time is the receipt's own, never a restore or read time.
96
  """
97
- import amazon_relational
98
-
99
- raw = amazon_relational.SOURCE.get(table_key)
100
  if not raw or (cur is None and state is None):
101
  return None
102
  if state is None:
@@ -161,7 +174,7 @@ def amazon_nav(session, pages):
161
  bases = {str(name) for (name,) in cur.execute(
162
  "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' "
163
  "AND table_type = 'BASE TABLE' AND starts_with(table_name, 'amzn_')").fetchall()}
164
- raws = {page["key"]: amazon_relational.SOURCE.get(page["key"]) for page in rows}
165
  probed = sorted({raw for raw in raws.values() if raw in bases})
166
  held = {}
167
  if probed:
 
67
  _AMAZON_DERIVED_RECEIPTS = {"keyword_ranks": "amzn_apify_keyword_ranks"}
68
 
69
 
70
+ def _amazon_raw(table_key, source=None):
71
+ """The mirror table behind one `ut_amzn_*` key: the relational map, else an Ads spec (S79-T13)."""
72
+ if source is None:
73
+ import amazon_relational
74
+ source = amazon_relational.SOURCE
75
+ raw = source.get(table_key)
76
+ if raw:
77
+ return raw
78
+ from core import connector_spec
79
+ spec = connector_spec.spec_for(table_key)
80
+ if spec is not None and connector_spec.connector_of(table_key) == connector_spec.AMAZON_ADS_CONNECTOR:
81
+ return spec.source
82
+ return None
83
+
84
+
85
  def amazon_sync_state(cur):
86
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
87
  return {str(entity): at for entity, at in cur.execute(
 
109
  R11: a refusal overwrites the raw receipt and a pass with no window writes no mark, so the
110
  reconciliation proofs also count. The time is the receipt's own, never a restore or read time.
111
  """
112
+ raw = _amazon_raw(table_key)
 
 
113
  if not raw or (cur is None and state is None):
114
  return None
115
  if state is None:
 
174
  bases = {str(name) for (name,) in cur.execute(
175
  "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' "
176
  "AND table_type = 'BASE TABLE' AND starts_with(table_name, 'amzn_')").fetchall()}
177
+ raws = {page["key"]: _amazon_raw(page["key"], amazon_relational.SOURCE) for page in rows}
178
  probed = sorted({raw for raw in raws.values() if raw in bases})
179
  held = {}
180
  if probed:
api/external_openapi.py CHANGED
@@ -43,8 +43,8 @@ def _field_schema(field, source_kind):
43
  kind = _JSON_TYPE.get(field.get("type"), "string")
44
  types = [kind]
45
  # Materialised table writes retain numeric cells as strings; reads preserve those bytes.
46
- if source_kind == "materialised_user_table" and kind in {"number", "integer"}:
47
- types.append("string")
48
  return {"type": [*types, "null"]}
49
 
50
 
 
43
  kind = _JSON_TYPE.get(field.get("type"), "string")
44
  types = [kind]
45
  # Materialised table writes retain numeric cells as strings; reads preserve those bytes.
46
+ if kind in {"number", "integer"}:
47
+ types = ["number", "string"]
48
  return {"type": [*types, "null"]}
49
 
50
 
api/loopy_catalog_tools.py CHANGED
@@ -11,6 +11,7 @@ def _build_refusal(code, message):
11
  def _build_app(ctx, args):
12
  import application_registry as applications
13
  import catalog_store
 
14
  from fastapi import HTTPException
15
 
16
  source = args["sourceDatabase"]
@@ -23,8 +24,8 @@ def _build_app(ctx, args):
23
  if app["sourceDatabase"] != source:
24
  _build_refusal("catalog_source_mismatch", "Choose a Catalog App over the requested source database.")
25
  else:
26
- candidates = [app for app in applications.load(ctx.session.runtime)["applications"].values()
27
- if app.get("templateKind") == "catalog" and app.get("sourceDatabase") == source
28
  and app.get("retired") is not True
29
  and applications.may_read(app, ctx.session.uname, ctx.session.admin)
30
  and applications.may_edit(app, ctx.session.uname, ctx.session.admin)]
@@ -40,8 +41,8 @@ def _build_app(ctx, args):
40
  if app is None:
41
  _build_refusal("catalog_app_unavailable", (
42
  "No editable Catalog App is available for this source. Create one with apps.apply_template "
43
- "using templateKind catalog and a fieldMap that maps key and title to field keys of this source, "
44
- "for example {\"key\": \"sku\", \"title\": \"name\"}. apps.templates lists the optional roles."))
45
  fields = {field["key"] for field in schema["fields"]}
46
  if any(field not in fields for field in app["fieldMap"].values()):
47
  _build_refusal("catalog_fields_forbidden", "The Catalog App uses unavailable fields. Rebind it to readable source fields.")
@@ -116,6 +117,7 @@ def _build_rows(ctx, source, fields, filters, sorts):
116
  def catalog_build(ctx, args):
117
  """Build a C1 record through the caller's staged runtime and permissioned query callbacks."""
118
  import catalog_store
 
119
  from core import external_tools, write_execution
120
  from loopy_file_tools import _secure_link
121
  from loopy_links import application_link
@@ -220,6 +222,22 @@ def build_schema():
220
  "required": ["sourceDatabase"], "additionalProperties": False}
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  def catalog_pdf(session, args):
224
  import assistant_files
225
  import catalog_render
@@ -234,6 +252,7 @@ def catalog_pdf(session, args):
234
  _secure_link(links.app_link("#/assistant"))
235
  _secure_link(links.artifact_link("catalog-preview"))
236
  data, record, app = catalog_render.render_catalog(session, args["catalog_id"], ai=True)
 
237
  title = _no_dashes(record["name"]).strip()[:200]
238
  saved = assistant_files.save(session.runtime, session.uname, title=title, data=data,
239
  mime="application/pdf", door=door,
@@ -287,6 +306,7 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
287
  import sys
288
  import threading
289
  import catalog_store
 
290
 
291
  render = render or sys.modules[__name__].catalog_pdf
292
  budget = export_budget() if budget is None else budget
 
11
  def _build_app(ctx, args):
12
  import application_registry as applications
13
  import catalog_store
14
+ import app_templates
15
  from fastapi import HTTPException
16
 
17
  source = args["sourceDatabase"]
 
24
  if app["sourceDatabase"] != source:
25
  _build_refusal("catalog_source_mismatch", "Choose a Catalog App over the requested source database.")
26
  else:
27
+ candidates = [app_templates.bound(app) for app in applications.load(ctx.session.runtime)["applications"].values()
28
+ if app.get("templateKind") == "catalog" and app_templates.bound(app).get("sourceDatabase") == source
29
  and app.get("retired") is not True
30
  and applications.may_read(app, ctx.session.uname, ctx.session.admin)
31
  and applications.may_edit(app, ctx.session.uname, ctx.session.admin)]
 
41
  if app is None:
42
  _build_refusal("catalog_app_unavailable", (
43
  "No editable Catalog App is available for this source. Create one with apps.apply_template "
44
+ "using templateKind catalog. "
45
+ "The App binds its Odoo products automatically."))
46
  fields = {field["key"] for field in schema["fields"]}
47
  if any(field not in fields for field in app["fieldMap"].values()):
48
  _build_refusal("catalog_fields_forbidden", "The Catalog App uses unavailable fields. Rebind it to readable source fields.")
 
117
  def catalog_build(ctx, args):
118
  """Build a C1 record through the caller's staged runtime and permissioned query callbacks."""
119
  import catalog_store
120
+ import app_templates
121
  from core import external_tools, write_execution
122
  from loopy_file_tools import _secure_link
123
  from loopy_links import application_link
 
222
  "required": ["sourceDatabase"], "additionalProperties": False}
223
 
224
 
225
+ def _register_render_read(session, source):
226
+ """Record the render's product read in the caller's producing turn, so the PDF keeps its source (D-9854).
227
+
228
+ `assistant_files.save` keeps only sources the turn proved it read. The render reads the source
229
+ under this caller even when the producing turn holds another session object for the same user.
230
+ """
231
+ import agent_runtime
232
+
233
+ producing = agent_runtime._PRODUCING_TURN.get()
234
+ if producing is None:
235
+ return
236
+ caller, turn = producing
237
+ if caller.runtime is session.runtime and caller.uname == session.uname:
238
+ agent_runtime._merge_sources(caller, turn, [source])
239
+
240
+
241
  def catalog_pdf(session, args):
242
  import assistant_files
243
  import catalog_render
 
252
  _secure_link(links.app_link("#/assistant"))
253
  _secure_link(links.artifact_link("catalog-preview"))
254
  data, record, app = catalog_render.render_catalog(session, args["catalog_id"], ai=True)
255
+ _register_render_read(session, app["sourceDatabase"])
256
  title = _no_dashes(record["name"]).strip()[:200]
257
  saved = assistant_files.save(session.runtime, session.uname, title=title, data=data,
258
  mime="application/pdf", door=door,
 
306
  import sys
307
  import threading
308
  import catalog_store
309
+ import app_templates
310
 
311
  render = render or sys.modules[__name__].catalog_pdf
312
  budget = export_budget() if budget is None else budget
api/main.py CHANGED
@@ -146,7 +146,8 @@ from request_timing import RequestTiming # noqa: E402
146
  class _TimedFastAPI(FastAPI):
147
  def build_middleware_stack(self):
148
  # Include error responses, freeze refusals, admission and compressed body sends.
149
- return RequestTiming(super().build_middleware_stack())
 
150
 
151
 
152
  app = _TimedFastAPI(title="AIOS web API")
@@ -208,7 +209,23 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
208
 
209
  async def _resource_unavailable(request: Request, exc: Exception):
210
  from request_timing import resource_response
211
- return resource_response(exc)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
212
 
213
 
214
  @app.exception_handler(Exception)
@@ -216,28 +233,31 @@ async def _unexpected_error(request: Request, exc: Exception):
216
  from request_timing import resource_response
217
  busy = resource_response(exc)
218
  if busy is not None:
219
- return busy
220
  return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}},
221
  status_code=500)
222
 
223
 
224
- from core.odoo import OdooPoolTimeout
225
- from core.store_pg import SharedReadTimeout
226
  from core.request_timing import RequestDeadlineExceeded
227
  from request_timing import DeadlineCapacityExceeded
228
  from duckdb import OutOfMemoryException
229
- for _resource_error in (OdooPoolTimeout, SharedReadTimeout, RequestDeadlineExceeded,
230
- DeadlineCapacityExceeded, OutOfMemoryException):
 
231
  app.add_exception_handler(_resource_error, _resource_unavailable)
232
  try:
233
  from psycopg_pool import PoolTimeout, TooManyRequests
234
  from psycopg import OperationalError, InterfaceError
235
  from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout
 
236
  except ImportError:
237
  pass
238
  else:
239
  for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable,
240
- OperationalError, InterfaceError, IdleInTransactionSessionTimeout):
 
241
  app.add_exception_handler(_resource_error, _resource_unavailable)
242
 
243
 
@@ -381,6 +401,35 @@ async def health(challenge: str = ""):
381
  return {"ok": True}
382
 
383
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
384
  # W59-T43: boot stays available while a cross-lane module is absent, but never silently.
385
  # verify_wiring.W59_PENDING_CROSS_LANE_HALVES owns the shrink-only declarations and rejects
386
  # any failure outside them, including a missing dependency INSIDE a declared router.
@@ -605,10 +654,13 @@ app.include_router(routes_tickets.router)
605
  # rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden).
606
  app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers)
607
  app.include_router(routes_external.router)
608
- app.include_router(routes_competitor_watch.router) # S78-T16 / S78-T17 competitor watch
 
 
609
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
610
  _mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors.
611
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
 
612
 
613
 
614
  # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
@@ -1041,7 +1093,7 @@ def _restore_amazon_mirror_once(why, tenant_key):
1041
  # Publish measured definitions before any Odoo read can block the resident writer.
1042
  import process_role as _process_role
1043
  # The relational rebuild writes the store, so only the sync worker runs it.
1044
- if (result.get("state") != "absent" and _process_role.allows("heavy") and
1045
  rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
1046
  state = _rebuild_amazon_relational(why, rt)
1047
  if state.get("changed") and not state.get("applied") and not state.get("known"):
@@ -1198,6 +1250,11 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
1198
  return detail
1199
  classifier, sentence = _sync_now_cause(cause)
1200
  print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
 
 
 
 
 
1201
  suffix = f" (cause code: {classifier})"
1202
  # finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
1203
  budget = 300 - len(suffix)
@@ -1277,8 +1334,13 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
1277
  odoo._tlocal.tenant_ctx = None
1278
  from connector_onboarding import selected_mirror_entities
1279
  selected_entities = selected_mirror_entities(rt)
 
 
 
 
1280
  report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
1281
- log=lambda *a, **k: None)
 
1282
  finally:
1283
  odoo._tlocal.client = previous_client
1284
  odoo._tlocal.tenant_ctx = previous_context
@@ -1301,9 +1363,12 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
1301
  "refresh_loaded": True} if first_load else {}
1302
  report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
1303
  scheduled=False if targeted else receipt.get("requestedBy") == "first load",
1304
- **target_options, **options, log=lambda *a, **k: None)
 
1305
  print("[aios-api] amazon sync done (sync desk): " + ", ".join(
1306
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
 
 
1307
  for table, row in (report.get("tables") or {}).items()))
1308
  from connectors_amazon import assert_safe_status
1309
  first_load_progress = None if targeted else assert_safe_status(report.get("firstLoad"))
@@ -2216,8 +2281,10 @@ def _resync_mirror_pass(manual, passes, entities=None):
2216
  print(f"[aios-api] store resync: the absent tables first failed "
2217
  f"({type(exc).__name__}); the full pass below still syncs them")
2218
  _resync_log("full pass start")
 
2219
  sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
2220
- entities=entities, log=_resync_log)
 
2221
  _log_entities("full", sync_report)
2222
  for key, row in first.items():
2223
  if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
@@ -2412,20 +2479,71 @@ def _resync_confirm_fresh(tenant, cadence_seconds):
2412
  print(f"[aios-api] odoo freshness stamp not saved ({type(e).__name__}: {e})")
2413
 
2414
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2415
  def _capture_resync_source_counts(tenant):
2416
  """Count the default connection's Odoo source right after its mirror pull, or None.
2417
 
2418
  The vendor pulls run between the mirror pull and the rebuild, so a count taken at rebuild
2419
- time describes a later source than the mirror holds (D-9131). Never raises.
 
2420
  """
2421
  try:
2422
  if not tenant:
2423
  return None
2424
  import odoo_relational as _rel_counts
2425
  from harness import runtime as _runtime_counts
2426
- counts = _rel_counts.capture_source_counts(_runtime_counts.get_runtime(tenant))
 
 
 
 
 
 
 
 
 
2427
  print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
2428
- f"database(s) at mirror-pull time", flush=True)
 
2429
  return counts or None
2430
  except Exception as e: # noqa: BLE001
2431
  print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
@@ -2701,7 +2819,8 @@ def _store_resync_loop():
2701
  # D-9131: count the source now, before the vendor pulls, so reconciliation compares the
2702
  # mirror with the source as of the same moment. A pass without a good pull counts nothing.
2703
  _source_counts = (_capture_resync_source_counts(_cad["tenant"])
2704
- if _err is None and sync_report and not _manual else None)
 
2705
  # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): whenever the Odoo RELATIONAL tables need
2706
  # refreshing, they run AFTER the mirror they derive from. A confirmed zero-change report
2707
  # is the W39 exception; deriving before a changed mirror advances would publish a one-cycle
@@ -2914,10 +3033,11 @@ def _prewarm():
2914
  if _process_role.role() == "worker":
2915
  # The warm below fills per-process memos, which the sync worker never serves.
2916
  return
 
 
2917
  try:
2918
  from harness import runtime as _runtime
2919
  rt = _runtime.get_runtime("royal-imports")
2920
- routes_customers.warm_default(rt)
2921
  import pages as _pages
2922
  _pages.warm_default(rt)
2923
  # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent
@@ -3098,8 +3218,8 @@ def _restore_sync_request_desks():
3098
  from core import connector_sync
3099
  from harness import runtime
3100
  import process_role as _process_role
3101
- # The web role never claims, so it must not requeue the worker's in-flight claims.
3102
- requeue = _process_role.role() != "web"
3103
  for tenant_key in runtime.known_tenants():
3104
  try:
3105
  connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue)
@@ -3110,30 +3230,90 @@ def _restore_sync_request_desks():
3110
  def _boot_mirrors():
3111
  import process_role as _role_boot
3112
  _restore_sync_request_desks()
3113
- _seed_mirror_if_absent()
 
 
3114
  if _role_boot.role() == "web":
 
 
3115
  # Every connector but Odoo reaches the web process through the worker's hand-off.
3116
  print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
3117
  # The biggest read caches are this process's own, so they warm here, before the first reader pays.
3118
  _role_boot.start("read-warm", "light", _warm_read_pools)
3119
  return
 
 
 
 
 
 
 
 
3120
  _restore_amazon_mirror("boot")
3121
  _restore_declared_mirrors("boot")
 
 
 
 
 
3122
 
3123
 
3124
- def _warm_read_pools():
3125
- """Build the consolidated customer and product pools once after boot, off the request path."""
3126
- import time as _t
3127
- t0 = _t.time()
 
 
 
 
 
 
 
3128
  try:
3129
- from harness import runtime as _runtime
3130
- import routes_products
3131
- rt = _runtime.get_runtime("royal-imports")
3132
- routes_customers.warm_default(rt)
3133
- routes_products._pool_for(rt, None)
3134
- print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
3135
- except Exception as exc: # noqa: BLE001
3136
- print(f"[aios-api] read warm skipped ({type(exc).__name__}: {exc})")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3137
 
3138
 
3139
  def _restore_declared_mirrors(why):
@@ -3183,7 +3363,7 @@ if (not data_binding.gate_process()
3183
 
3184
  if (not data_binding.gate_process()
3185
  and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
3186
- and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
3187
  and not _freeze_holds("app migrations")):
3188
  import process_role as _role_apps
3189
  _role_apps.thread(target=_boot_app_migrations, args=("boot",), daemon=True, kind="heavy",
 
146
  class _TimedFastAPI(FastAPI):
147
  def build_middleware_stack(self):
148
  # Include error responses, freeze refusals, admission and compressed body sends.
149
+ from error_sink import ErrorSinkMiddleware
150
+ return ErrorSinkMiddleware(RequestTiming(super().build_middleware_stack()))
151
 
152
 
153
  app = _TimedFastAPI(title="AIOS web API")
 
209
 
210
  async def _resource_unavailable(request: Request, exc: Exception):
211
  from request_timing import resource_response
212
+ return _with_holder(resource_response(exc), exc)
213
+
214
+
215
+ def _with_holder(response, exc):
216
+ seen = set()
217
+ while exc is not None and id(exc) not in seen:
218
+ seen.add(id(exc))
219
+ holder = getattr(exc, "aios_holder", None)
220
+ if isinstance(holder, dict) and response is not None:
221
+ import json
222
+ body = json.loads(response.body)
223
+ body["error"]["holder"] = holder
224
+ body["error"]["message"] = (
225
+ f"Another change is still saving ({holder.get('label', 'another request')}). Try again shortly.")
226
+ return JSONResponse(body, status_code=503, headers={"Retry-After": "1"})
227
+ exc = exc.__cause__ or exc.__context__
228
+ return response
229
 
230
 
231
  @app.exception_handler(Exception)
 
233
  from request_timing import resource_response
234
  busy = resource_response(exc)
235
  if busy is not None:
236
+ return _with_holder(busy, exc)
237
  return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}},
238
  status_code=500)
239
 
240
 
241
+ from core.odoo import OdooPoolTimeout, OdooRequestTimeout
242
+ from core.store_pg import SharedReadTimeout, PostgresOperationTimeout
243
  from core.request_timing import RequestDeadlineExceeded
244
  from request_timing import DeadlineCapacityExceeded
245
  from duckdb import OutOfMemoryException
246
+ # Each busy cause needs its own handler: the catch-all `Exception` one answers, then re-raises to uvicorn.
247
+ for _resource_error in (OdooPoolTimeout, OdooRequestTimeout, SharedReadTimeout, PostgresOperationTimeout,
248
+ RequestDeadlineExceeded, DeadlineCapacityExceeded, OutOfMemoryException):
249
  app.add_exception_handler(_resource_error, _resource_unavailable)
250
  try:
251
  from psycopg_pool import PoolTimeout, TooManyRequests
252
  from psycopg import OperationalError, InterfaceError
253
  from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout
254
+ from psycopg.errors import ConnectionTimeout
255
  except ImportError:
256
  pass
257
  else:
258
  for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable,
259
+ OperationalError, InterfaceError, IdleInTransactionSessionTimeout,
260
+ ConnectionTimeout):
261
  app.add_exception_handler(_resource_error, _resource_unavailable)
262
 
263
 
 
401
  return {"ok": True}
402
 
403
 
404
+ def _mirror_answers():
405
+ """The resident Odoo mirror's live state for readiness. Never raises."""
406
+ try:
407
+ from harness import datastore as _ready_ds
408
+ return bool(_ready_ds.ready())
409
+ except Exception: # noqa: BLE001
410
+ return False
411
+
412
+
413
+ @app.get("/api/v1/ready")
414
+ def readiness_check():
415
+ """C9: 200 only when the mirror is installed, the pools are open and the first reads answered.
416
+
417
+ Unauthenticated, because the deploy switch polls it before anyone signs in. It carries the
418
+ version on purpose (C9), unlike /api/health.
419
+ """
420
+ report = process_role.readiness(mirror_live=_mirror_answers())
421
+ body = {"ready": report["ready"], "version": os.environ.get("AIOS_VERSION") or "unknown",
422
+ "checks": report["checks"]}
423
+ headers = {"Cache-Control": "no-store"}
424
+ if report["ready"]:
425
+ return JSONResponse(body, headers=headers)
426
+ body["failing"] = report["failing"]
427
+ body["error"] = {"code": "not_ready",
428
+ "message": "Not ready: " + ", ".join(report["failing"]) + ". "
429
+ + " ".join(report["detail"].values())}
430
+ return JSONResponse(body, status_code=503, headers=headers)
431
+
432
+
433
  # W59-T43: boot stays available while a cross-lane module is absent, but never silently.
434
  # verify_wiring.W59_PENDING_CROSS_LANE_HALVES owns the shrink-only declarations and rejects
435
  # any failure outside them, including a missing dependency INSIDE a declared router.
 
654
  # rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden).
655
  app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers)
656
  app.include_router(routes_external.router)
657
+ import app_routes
658
+ app.include_router(routes_competitor_watch.router, # S78-T16 / S78-T17 competitor watch; held by R11
659
+ dependencies=app_routes.held_dependencies("competitor_watch"))
660
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
661
  _mount_router("routes_amazon_profit") # S78-T03: the Amazon SKU profit refresh and settings doors.
662
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
663
+ app_routes.mount_app_routers(app)
664
 
665
 
666
  # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
 
1093
  # Publish measured definitions before any Odoo read can block the resident writer.
1094
  import process_role as _process_role
1095
  # The relational rebuild writes the store, so only the sync worker runs it.
1096
+ if (result.get("state") != "absent" and _process_role.may_write() and
1097
  rt.datastore_path.resolve() == _ds.DB_PATH.resolve()):
1098
  state = _rebuild_amazon_relational(why, rt)
1099
  if state.get("changed") and not state.get("applied") and not state.get("known"):
 
1250
  return detail
1251
  classifier, sentence = _sync_now_cause(cause)
1252
  print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
1253
+ if isinstance(cause, BaseException):
1254
+ import traceback
1255
+ locations = " > ".join(f"{os.path.basename(frame.filename)}:{frame.lineno}:{frame.name}"
1256
+ for frame in traceback.extract_tb(cause.__traceback__))
1257
+ print(f"[aios-api] sync failure location {type(cause).__name__}: {locations}")
1258
  suffix = f" (cause code: {classifier})"
1259
  # finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
1260
  budget = 300 - len(suffix)
 
1334
  odoo._tlocal.tenant_ctx = None
1335
  from connector_onboarding import selected_mirror_entities
1336
  selected_entities = selected_mirror_entities(rt)
1337
+ # Sync now counts each entity's source beside its copy, as the full pass does.
1338
+ # Looked up by name: extracted test fixtures of this function omit the counter.
1339
+ _counter = globals().get("_entity_source_counter")
1340
+ _after_entity = _counter(datastore) if _counter else None
1341
  report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
1342
+ log=lambda *a, **k: None,
1343
+ **({"after_entity": _after_entity} if _after_entity else {}))
1344
  finally:
1345
  odoo._tlocal.client = previous_client
1346
  odoo._tlocal.tenant_ctx = previous_context
 
1363
  "refresh_loaded": True} if first_load else {}
1364
  report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
1365
  scheduled=False if targeted else receipt.get("requestedBy") == "first load",
1366
+ **target_options, **options,
1367
+ log=lambda message: print(f"[aios-api] amazon sync desk: {message}", flush=True))
1368
  print("[aios-api] amazon sync done (sync desk): " + ", ".join(
1369
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
1370
+ + (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
1371
+ if "resync" in row else "")
1372
  for table, row in (report.get("tables") or {}).items()))
1373
  from connectors_amazon import assert_safe_status
1374
  first_load_progress = None if targeted else assert_safe_status(report.get("firstLoad"))
 
2281
  print(f"[aios-api] store resync: the absent tables first failed "
2282
  f"({type(exc).__name__}); the full pass below still syncs them")
2283
  _resync_log("full pass start")
2284
+ _after_entity = _entity_source_counter(_ds)
2285
  sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
2286
+ entities=entities, log=_resync_log,
2287
+ **({"after_entity": _after_entity} if _after_entity else {}))
2288
  _log_entities("full", sync_report)
2289
  for key, row in first.items():
2290
  if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
 
2479
  print(f"[aios-api] odoo freshness stamp not saved ({type(e).__name__}: {e})")
2480
 
2481
 
2482
+ #: tenant -> {bucket: (count, mode)} counted beside each entity's copy during the last mirror pass.
2483
+ _PASS_SOURCE_COUNTS = {}
2484
+
2485
+
2486
+ def _entity_source_counter(ds):
2487
+ """A `sync_all` hook that counts an entity's source buckets right after its copy, or None.
2488
+
2489
+ None when `sync_all` takes no `after_entity` hook: the loop then counts once after the pass.
2490
+ """
2491
+ import inspect as _inspect_counts
2492
+ try:
2493
+ if "after_entity" not in _inspect_counts.signature(ds.sync_all).parameters:
2494
+ return None
2495
+ rt = ds._resident_odoo_runtime()
2496
+ except Exception: # noqa: BLE001
2497
+ return None
2498
+ tenant = getattr(rt, "key", None)
2499
+ if not tenant:
2500
+ return None
2501
+ counted = _PASS_SOURCE_COUNTS[tenant] = {}
2502
+
2503
+ def after_entity(key):
2504
+ # A volatile table (stock_quant) drifts between its copy and a count at the end of the pass.
2505
+ try:
2506
+ import odoo_relational as _rel_counts
2507
+ model = (ds.ENTITIES.get(key) or {}).get("model")
2508
+ buckets = _rel_counts.source_count_buckets(model) - set(counted)
2509
+ if not buckets:
2510
+ return
2511
+ got = _rel_counts.capture_source_counts(rt, buckets=buckets)
2512
+ counted.update(got)
2513
+ for bucket, (count, _mode) in sorted(got.items()):
2514
+ print(f"[aios-api] store resync: counted the Odoo source for {bucket} "
2515
+ f"beside the {key} copy: {count:,} rows", flush=True)
2516
+ except Exception as e: # noqa: BLE001
2517
+ print(f"[aios-api] store resync: the {key} source was not counted beside its copy "
2518
+ f"({type(e).__name__}); it is counted after the pass", flush=True)
2519
+ return after_entity
2520
+
2521
+
2522
  def _capture_resync_source_counts(tenant):
2523
  """Count the default connection's Odoo source right after its mirror pull, or None.
2524
 
2525
  The vendor pulls run between the mirror pull and the rebuild, so a count taken at rebuild
2526
+ time describes a later source than the mirror holds (D-9131). A bucket counted beside its
2527
+ entity's copy during the pass keeps that count. Never raises.
2528
  """
2529
  try:
2530
  if not tenant:
2531
  return None
2532
  import odoo_relational as _rel_counts
2533
  from harness import runtime as _runtime_counts
2534
+ during = dict((globals().get("_PASS_SOURCE_COUNTS") or {}).pop(tenant, None) or {})
2535
+ rt = _runtime_counts.get_runtime(tenant)
2536
+ counts = dict(during)
2537
+ if not during:
2538
+ counts.update(_rel_counts.capture_source_counts(rt))
2539
+ else:
2540
+ rest = _rel_counts.enabled_buckets(rt) - set(during)
2541
+ if rest:
2542
+ counts.update(_rel_counts.capture_source_counts(rt, buckets=rest))
2543
+ per_entity = " ".join(f"{bucket}={count:,}" for bucket, (count, _mode) in sorted(during.items()))
2544
  print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
2545
+ f"database(s) at mirror-pull time, {len(during)} beside their own copy"
2546
+ + (f"; source counts per entity: {per_entity}" if per_entity else ""), flush=True)
2547
  return counts or None
2548
  except Exception as e: # noqa: BLE001
2549
  print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
 
2819
  # D-9131: count the source now, before the vendor pulls, so reconciliation compares the
2820
  # mirror with the source as of the same moment. A pass without a good pull counts nothing.
2821
  _source_counts = (_capture_resync_source_counts(_cad["tenant"])
2822
+ if _err is None and ((sync_report and not _manual)
2823
+ or (_manual and _odoo_requested)) else None)
2824
  # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): whenever the Odoo RELATIONAL tables need
2825
  # refreshing, they run AFTER the mirror they derive from. A confirmed zero-change report
2826
  # is the W39 exception; deriving before a changed mirror advances would publish a one-cycle
 
3033
  if _process_role.role() == "worker":
3034
  # The warm below fills per-process memos, which the sync worker never serves.
3035
  return
3036
+ # D-273: the shared read warm fills both default pools in runtime.pool_cache.
3037
+ _warm_read_pools(attempts=1)
3038
  try:
3039
  from harness import runtime as _runtime
3040
  rt = _runtime.get_runtime("royal-imports")
 
3041
  import pages as _pages
3042
  _pages.warm_default(rt)
3043
  # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent
 
3218
  from core import connector_sync
3219
  from harness import runtime
3220
  import process_role as _process_role
3221
+ # Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
3222
+ requeue = _process_role.may_write()
3223
  for tenant_key in runtime.known_tenants():
3224
  try:
3225
  connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue)
 
3230
  def _boot_mirrors():
3231
  import process_role as _role_boot
3232
  _restore_sync_request_desks()
3233
+ if _role_boot.role() != "web" or _role_boot.claim_primary():
3234
+ # With several web workers only the primary restores the mirror. The others read it.
3235
+ _seed_mirror_if_absent()
3236
  if _role_boot.role() == "web":
3237
+ # Readiness also waits for the worker's boot hand-offs and the mirror's own completeness.
3238
+ _role_boot.mark_ready("mirror", True)
3239
  # Every connector but Odoo reaches the web process through the worker's hand-off.
3240
  print("[aios-api] web boot restores the Odoo mirror only; other mirrors arrive from the sync worker")
3241
  # The biggest read caches are this process's own, so they warm here, before the first reader pays.
3242
  _role_boot.start("read-warm", "light", _warm_read_pools)
3243
  return
3244
+ if _role_boot.role() == "worker":
3245
+ try:
3246
+ _restore_amazon_mirror("boot")
3247
+ _restore_declared_mirrors("boot")
3248
+ finally:
3249
+ # Every boot hand-off is on disk now; the web process's readiness waits for this line.
3250
+ _role_boot.note_boot_restore()
3251
+ return
3252
  _restore_amazon_mirror("boot")
3253
  _restore_declared_mirrors("boot")
3254
+ # all and reader have no worker: their own restores finish before readiness and the warm.
3255
+ _role_boot.mark_ready("mirror", True)
3256
+ if _role_boot.role() == "reader" or data_binding.is_space():
3257
+ # A local `all` run keeps its old boot: AIOS_PREWARM=1 warms it.
3258
+ _warm_read_pools()
3259
 
3260
 
3261
+ #: A failed boot warm retries, so one slow first read cannot leave readiness red until a restart.
3262
+ _READ_WARM_ATTEMPTS = 40
3263
+ _READ_WARM_RETRY_SECONDS = 15
3264
+
3265
+
3266
+ def _probe_read_pools(rt):
3267
+ """Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
3268
+ from harness import datastore as _probe_ds
3269
+ if not _probe_ds.DB_PATH.exists():
3270
+ return "The mirror file is missing."
3271
+ cur = _probe_ds.ro_cursor()
3272
  try:
3273
+ cur.execute("SELECT 1").fetchone()
3274
+ finally:
3275
+ cur.close()
3276
+ if rt.available() is False:
3277
+ return "The tenant store did not answer."
3278
+ return ""
3279
+
3280
+
3281
+ def _warm_read_pools(attempts=None, retry_seconds=None):
3282
+ """Open the read pools, then build the consolidated customer and product pools, off the request path.
3283
+
3284
+ Marks the `pools` and `first_reads` readiness checks. A failure never clears a check that passed.
3285
+ """
3286
+ import time as _t
3287
+ import process_role as _warm_role
3288
+ attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
3289
+ retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
3290
+ for attempt in range(1, attempts + 1):
3291
+ t0 = _t.time()
3292
+ stage = "pools"
3293
+ try:
3294
+ from harness import runtime as _runtime
3295
+ import routes_products
3296
+ rt = _runtime.get_runtime("royal-imports")
3297
+ problem = _probe_read_pools(rt)
3298
+ if problem:
3299
+ raise RuntimeError(problem)
3300
+ _warm_role.mark_ready("pools", True)
3301
+ stage = "first_reads"
3302
+ routes_customers.warm_default(rt)
3303
+ routes_products._pool_for(rt, None)
3304
+ _warm_role.mark_ready("first_reads", True)
3305
+ routes_customers.warm_bu_pools(rt)
3306
+ print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
3307
+ return True
3308
+ except Exception as exc: # noqa: BLE001
3309
+ if not _warm_role._READINESS.get(stage, {}).get("ok"):
3310
+ _warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
3311
+ f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
3312
+ print(f"[aios-api] read warm failed at {stage} ({type(exc).__name__}: {exc}); "
3313
+ f"attempt {attempt} of {attempts}")
3314
+ if attempt < attempts:
3315
+ _t.sleep(retry_seconds)
3316
+ return False
3317
 
3318
 
3319
  def _restore_declared_mirrors(why):
 
3363
 
3364
  if (not data_binding.gate_process()
3365
  and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
3366
+ and (bool(os.environ.get("SPACE_ID")) or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
3367
  and not _freeze_holds("app migrations")):
3368
  import process_role as _role_apps
3369
  _role_apps.thread(target=_boot_app_migrations, args=("boot",), daemon=True, kind="heavy",
api/odoo_relational.py CHANGED
@@ -2572,26 +2572,33 @@ class Refused(Exception):
2572
  #: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole
2573
  #: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code.
2574
  #: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits.
 
 
 
 
 
 
 
 
 
 
2575
  def payment_fields():
2576
  """Declare company-currency payment facts without materializing source rows."""
2577
- return [_preset({"key": key, "label": label, "type": kind, "source": "overlay",
2578
- "default": True}) for key, label, kind in (
2579
- ("amount_company_currency_signed", "Company currency amount", "currency"),
2580
- ("date", "Date", "date"), ("state", "State", "text"),
2581
- ("payment_type", "Payment type", "text"), ("partner_type", "Partner type", "text"),
2582
- ("is_internal_transfer", "Internal transfer", "checkbox"),
2583
- ("partner_id", "Customer ID", "int"))]
2584
 
2585
 
2586
  def payment_source_specs():
2587
  """Bind the payment database to its native payment identity and gross receipt amount."""
 
 
2588
  return {PAYMENTS_KEY: {
2589
  "table": "account_payment", "from_sql": "account_payment pay", "tables": {"pay": "account_payment"},
2590
- "alias": "pay", "id": "pay.id", "where": "",
2591
- "cols": {field["key"]: ("pay." + field["key"], converter)
2592
- for field, converter in zip(payment_fields(),
2593
- (lambda value: float(value or 0), _as_date, str, str, str, bool,
2594
- lambda value: int(value or 0)))}}}
2595
 
2596
 
2597
  TABLES = (
@@ -2716,12 +2723,19 @@ def _selection_field_contracts(fields, catalogue_row):
2716
  return out
2717
 
2718
 
2719
- def discover_readable_models(client):
 
 
 
 
 
 
2720
  """Read-only introspection over an injected Odoo client.
2721
 
2722
  ``ir.model`` supplies candidates, ``check_access_rights('read')`` narrows them to this API
2723
  account, and ``fields_get`` supplies only fields that account may inspect. No method in this
2724
  path can mutate Odoo; the production client independently blocks write-shaped method names.
 
2725
  """
2726
  errors = []
2727
  try:
@@ -2734,13 +2748,16 @@ def discover_readable_models(client):
2734
  model = str((item or {}).get("model") or "").strip().lower()
2735
  if not model:
2736
  continue
 
 
2737
  try:
2738
- readable = client.execute_kw(
2739
- model, "check_access_rights", ["read"], {"raise_exception": False})
2740
  except Exception as exc: # noqa: BLE001
2741
  errors.append(f"{model}: access: {type(exc).__name__}: {exc}")
2742
  continue
2743
- if not bool(readable):
 
 
2744
  continue
2745
  try:
2746
  fields = client.fields_get(
@@ -2764,28 +2781,125 @@ def _catalog_slot(connection_id):
2764
  return str(connection_id or DEFAULT_CONNECTION)
2765
 
2766
 
2767
- def readable_model_catalog(rt, connection_id=None):
2768
- """The last complete tenant-local catalogue; reading it never contacts Odoo."""
2769
  try:
2770
  cached = rt.get(MODEL_CATALOG_KEY) if rt is not None else None
2771
  except Exception: # noqa: BLE001
2772
  cached = None
2773
  if not isinstance(cached, dict) or cached.get("version") != MODEL_CATALOG_VERSION:
2774
- return {"version": MODEL_CATALOG_VERSION, "complete": False, "capturedAt": "",
2775
- "models": [], "errors": []}
2776
- cached = (cached.get("connections") or {}).get(_catalog_slot(connection_id)) or {}
 
 
 
 
2777
  return {"version": MODEL_CATALOG_VERSION,
2778
- "complete": cached.get("complete") is True,
2779
- "capturedAt": str(cached.get("capturedAt") or ""),
2780
- "models": [dict(row) for row in (cached.get("models") or ())
2781
  if isinstance(row, dict) and row.get("model")],
2782
- "errors": [str(value) for value in (cached.get("errors") or ())][:50]}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2783
 
2784
 
2785
  def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False):
2786
  """Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure."""
2787
- current = readable_model_catalog(rt, connection_id)
2788
- if (current["capturedAt"] and not force
 
 
 
 
2789
  and not any(_missing_selection_metadata(row) for row in current["models"])):
2790
  try:
2791
  captured = _dt.datetime.fromisoformat(current["capturedAt"])
@@ -2802,46 +2916,42 @@ def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=Fa
2802
  source = None
2803
  if source is None:
2804
  return current
 
2805
  try:
2806
  if client is None:
2807
  from core.odoo import get_odoo
2808
  with source._bound(): # tenant credential + scope
2809
- rows, errors = discover_readable_models(get_odoo())
2810
  else:
2811
- rows, errors = discover_readable_models(client)
2812
  except Exception as exc: # noqa: BLE001
2813
  rows, errors = [], [f"catalogue: {type(exc).__name__}: {exc}"]
 
2814
  # A partial retry is not evidence that a previously readable model disappeared. Preserve a
2815
  # complete, stamped catalogue in full whenever the new pass reports any error; otherwise one
2816
  # refused model could silently shrink the enablement roster. A first-ever partial pass may be
2817
  # retained for diagnostics, but an entirely failed pass still has no catalogue to publish.
2818
- if errors and current["complete"] and current["capturedAt"]:
2819
- return current
2820
- if not rows and errors:
 
2821
  return current
2822
  captured = _dt.datetime.now(_dt.timezone.utc).isoformat()
2823
  result = {"version": MODEL_CATALOG_VERSION, "complete": not errors,
2824
- "capturedAt": captured, "models": rows, "errors": errors[:50]}
2825
- slot = _catalog_slot(connection_id)
2826
-
2827
- def _cache(current_cache):
2828
- current_cache = dict(current_cache) if isinstance(current_cache, dict) else {}
2829
- connections = dict(current_cache.get("connections") or {})
2830
- connections[slot] = result
2831
- return {"version": MODEL_CATALOG_VERSION, "connections": connections}
2832
-
2833
- rt.update(MODEL_CATALOG_KEY, _cache, flush="sync")
2834
- return result
2835
 
2836
 
2837
  def refresh_model_catalog_entry(rt, model, connection_id=None, client=None):
2838
  """Refresh one model's fields with two read-only calls, without enumerating other models."""
2839
- current = readable_model_catalog(rt, connection_id)
 
 
 
2840
 
2841
  def _read_fields(bound_client):
2842
- readable = bound_client.execute_kw(
2843
- model, "check_access_rights", ["read"], {"raise_exception": False})
2844
- if not bool(readable):
2845
  return None
2846
  fields = bound_client.fields_get(
2847
  model, attributes=_ODOO_FIELD_ATTRIBUTES)
@@ -2867,28 +2977,40 @@ def refresh_model_catalog_entry(rt, model, connection_id=None, client=None):
2867
  # No answer (including unavailable credentials) is not evidence of denied model access.
2868
  return current
2869
  if fields is None:
 
 
 
 
 
 
 
 
 
 
 
2870
  raise Refused(f"This Odoo account cannot read the model {model}. "
2871
  "Ask your Odoo administrator to grant read access in Odoo, not here.")
2872
- slot = _catalog_slot(connection_id)
2873
 
2874
- def _cache(current_cache):
2875
- current_cache = dict(current_cache) if isinstance(current_cache, dict) else {}
2876
- connections = dict(current_cache.get("connections") or {})
2877
- result = dict(connections.get(slot) or current)
2878
  rows = list(result.get("models") or ())
2879
- previous = next((row for row in rows if row.get("model") == model), None)
2880
  row = {"model": model,
2881
- "name": " ".join(str((previous or {}).get("name") or model).split()),
2882
  "fields": fields}
2883
  result["models"] = [row if item.get("model") == model else item for item in rows]
2884
- if previous is None:
2885
  result["models"].append(row)
2886
  # One model neither captures a whole catalogue nor changes its completeness. Preserve
2887
  # complete and capturedAt exactly, along with every unrelated row and connection slot.
2888
- connections[slot] = result
2889
- return {"version": MODEL_CATALOG_VERSION, "connections": connections}
2890
-
2891
- rt.update(MODEL_CATALOG_KEY, _cache, flush="sync")
 
 
 
 
2892
  return readable_model_catalog(rt, connection_id)
2893
 
2894
 
@@ -3255,6 +3377,10 @@ ODOO_SPEC = (
3255
  "pinned": "invoice_no", "identity": (_ROW_ID_COL, "odoo_id"),
3256
  "links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),)},
3257
 
 
 
 
 
3258
  # ⚠ RETIRED FROM `TABLES` BY W33-T44, DECLARED HERE ANYWAY, AND THAT IS THE POINT OF A SPEC.
3259
  # `RETIRED_KEYS` pops this key and `PRODUCTS_KEY` from a tenant's document on every sync, but
3260
  # the key constants, the field builders and the readers in `_READERS` all still exist, the
@@ -3788,12 +3914,18 @@ def _source_domain_count(client, bucket):
3788
  raise Refused("the database source-count mode is not supported")
3789
 
3790
 
3791
- def capture_source_counts(rt, connection_id=None):
 
 
 
 
 
3792
  """`{bucket: (count, mode)}` for each enabled curated database, counted at the source now.
3793
 
3794
  The resync loop calls this right after the mirror pull, so reconciliation compares the mirror
3795
- with the source as of the same moment. A bucket whose count raises is left out, and that
3796
- bucket counts fresh at reconciliation. No source or no client returns `{}`.
 
3797
  """
3798
  try:
3799
  source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
@@ -3810,7 +3942,10 @@ def capture_source_counts(rt, connection_id=None):
3810
  return {}
3811
  if client is None:
3812
  return {}
3813
- for bucket in sorted(enabled_buckets(rt, connection_id)):
 
 
 
3814
  try:
3815
  counts[bucket] = _source_domain_count(client, bucket)
3816
  except Exception: # noqa: BLE001
@@ -5341,27 +5476,37 @@ def grid_choices(rt, connection_id=None):
5341
  by a curated table is covered there; every remaining readable model receives a stable raw-model
5342
  key and defaults off. Stored choices survive catalogue refreshes because config is keyed by the
5343
  stable database key, never by its position.
 
 
 
 
 
5344
  """
5345
  chosen = read_config(rt)["grids"]
5346
  namespace = connection_namespace(rt, connection_id)[1] if connection_id else None
5347
- catalogue = readable_model_catalog(rt, namespace)
 
 
 
5348
  fixed = [{"key": key, "label": label, "model": ODOO_TABLE_MODELS.get(key, ""),
5349
  "relevant": True, "defaultEnabled": True,
5350
  "enabled": bool(chosen.get(key, True))}
5351
  for _bucket, key, label, _fields in TABLES]
 
 
5352
  # Before discovery, keep the established curated defaults. A complete stamped catalogue is
5353
  # authoritative for this API account, so a curated database whose model is absent must not be
5354
  # offered: relevance never widens access.
5355
- if catalogue["complete"] and catalogue["capturedAt"]:
5356
  readable = {str(row.get("model") or "") for row in catalogue["models"]}
5357
- fixed = [row for row in fixed if not row["model"] or row["model"] in readable]
5358
  covered = {row["model"] for row in fixed if row["model"]}
5359
  dynamic = []
5360
  try:
5361
  from core import connector_spec
5362
- for item in catalogue["models"]:
5363
  model = str(item.get("model") or "")
5364
- if not model or model in covered:
5365
  continue
5366
  key = connector_spec.odoo_model_table_key(model)
5367
  dynamic.append({"key": key,
@@ -5795,7 +5940,7 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
5795
  existing = {}
5796
  if rt is not None:
5797
  try:
5798
- existing = dict(rt.get(ut.STORE_KEY) or {})
5799
  except Exception: # noqa: BLE001
5800
  existing = {}
5801
  # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and
@@ -5814,12 +5959,10 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
5814
  # be eligible FOR), but the law is written without an exception, so this lends the evaluator
5815
  # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so
5816
  # what is lent is the document as it stands the moment this plan applies.
5817
- lent = _LentDoc({**existing,
5818
- **{table_key_for(k, connection_id): {
5819
- **(existing.get(table_key_for(k, connection_id)) or {}),
5820
- "readThrough": True}
5821
- for k in READ_THROUGH_KEYS}},
5822
- ut.STORE_KEY)
5823
 
5824
  built, limits = {}, {}
5825
  caps = {}
@@ -6192,9 +6335,14 @@ def apply_plan(rt, built, username="automation", today=None, report=None,
6192
  generated = connector_spec.system_link_fields("odoo", live_columns)
6193
  for key, label, fields, rows in plans:
6194
  existing = {field["key"] for field in fields}
 
 
 
6195
  fields = [*fields, *namespaced_fields(
6196
  [_preset(field) for field in generated.get(base_table_key(key), ())
6197
- if field["key"] not in existing], connection_id)]
 
 
6198
  written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp)
6199
  from core import connector_sync
6200
  cur[key]["connector"] = {
@@ -6590,7 +6738,7 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
6590
  if not selected:
6591
  return {}
6592
  ut = _ut()
6593
- existing = dict(rt.get(ut.STORE_KEY) or {})
6594
  # ⛔ HOTFIX 2026-09-24: a model the store's byte budget already refused is NOT pulled again.
6595
  # Without this, every resync re-read all of it from Odoo (account.invoice.line.agent: 250,000
6596
  # rows) only for the store door to discard the rows. The refusal and its fix are on the wire
 
2572
  #: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole
2573
  #: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code.
2574
  #: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits.
2575
+ #: (key, label, type, converter) per payment fact, in column order.
2576
+ _PAYMENT_FACTS = (
2577
+ ("amount_company_currency_signed", "Company currency amount", "currency",
2578
+ lambda value: float(value or 0)),
2579
+ ("date", "Date", "date", _as_date), ("state", "State", "text", str),
2580
+ ("payment_type", "Payment type", "text", str), ("partner_type", "Partner type", "text", str),
2581
+ ("is_internal_transfer", "Internal transfer", "checkbox", bool),
2582
+ ("partner_id", "Customer ID", "int", lambda value: int(value or 0)))
2583
+
2584
+
2585
  def payment_fields():
2586
  """Declare company-currency payment facts without materializing source rows."""
2587
+ # The pinned identity column is what ODOO_SPEC's payments row names (pinned and identity).
2588
+ return [_preset({"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
2589
+ "default": True, "pinned": True,
2590
+ "description": "The `account.payment` id. Also this row's id."})] + [
2591
+ _preset({"key": key, "label": label, "type": kind, "source": "overlay", "default": True})
2592
+ for key, label, kind, _convert in _PAYMENT_FACTS]
 
2593
 
2594
 
2595
  def payment_source_specs():
2596
  """Bind the payment database to its native payment identity and gross receipt amount."""
2597
+ cols = {"odoo_id": ("pay.id", lambda value: int(value or 0))}
2598
+ cols.update({key: ("pay." + key, convert) for key, _label, _kind, convert in _PAYMENT_FACTS})
2599
  return {PAYMENTS_KEY: {
2600
  "table": "account_payment", "from_sql": "account_payment pay", "tables": {"pay": "account_payment"},
2601
+ "alias": "pay", "id": "pay.id", "where": "", "cols": cols}}
 
 
 
 
2602
 
2603
 
2604
  TABLES = (
 
2723
  return out
2724
 
2725
 
2726
+ def _read_access(client, model):
2727
+ """Odoo's own read verdict for this credential; raises when Odoo gives no answer."""
2728
+ return bool(client.execute_kw(
2729
+ model, "check_access_rights", ["read"], {"raise_exception": False}))
2730
+
2731
+
2732
+ def discover_readable_models(client, verdicts=None):
2733
  """Read-only introspection over an injected Odoo client.
2734
 
2735
  ``ir.model`` supplies candidates, ``check_access_rights('read')`` narrows them to this API
2736
  account, and ``fields_get`` supplies only fields that account may inspect. No method in this
2737
  path can mutate Odoo; the production client independently blocks write-shaped method names.
2738
+ A ``verdicts`` dict receives every candidate: True, False (an explicit refusal) or None.
2739
  """
2740
  errors = []
2741
  try:
 
2748
  model = str((item or {}).get("model") or "").strip().lower()
2749
  if not model:
2750
  continue
2751
+ if verdicts is not None:
2752
+ verdicts[model] = None
2753
  try:
2754
+ readable = _read_access(client, model)
 
2755
  except Exception as exc: # noqa: BLE001
2756
  errors.append(f"{model}: access: {type(exc).__name__}: {exc}")
2757
  continue
2758
+ if verdicts is not None:
2759
+ verdicts[model] = readable
2760
+ if not readable:
2761
  continue
2762
  try:
2763
  fields = client.fields_get(
 
2781
  return str(connection_id or DEFAULT_CONNECTION)
2782
 
2783
 
2784
+ def _read_catalog_slot(rt, connection_id=None):
2785
+ """One read of this connection's raw catalogue slot, or {} before any discovery."""
2786
  try:
2787
  cached = rt.get(MODEL_CATALOG_KEY) if rt is not None else None
2788
  except Exception: # noqa: BLE001
2789
  cached = None
2790
  if not isinstance(cached, dict) or cached.get("version") != MODEL_CATALOG_VERSION:
2791
+ return {}
2792
+ slot = (cached.get("connections") or {}).get(_catalog_slot(connection_id))
2793
+ return slot if isinstance(slot, dict) else {}
2794
+
2795
+
2796
+ def _catalog_view(slot):
2797
+ slot = slot if isinstance(slot, dict) else {}
2798
  return {"version": MODEL_CATALOG_VERSION,
2799
+ "complete": slot.get("complete") is True,
2800
+ "capturedAt": str(slot.get("capturedAt") or ""),
2801
+ "models": [dict(row) for row in (slot.get("models") or ())
2802
  if isinstance(row, dict) and row.get("model")],
2803
+ "errors": [str(value) for value in (slot.get("errors") or ())][:50]}
2804
+
2805
+
2806
+ def _model_names(values):
2807
+ return sorted({str(value).strip().lower() for value in (values or ())
2808
+ if str(value or "").strip()})
2809
+
2810
+
2811
+ def _access_view(slot):
2812
+ """Who answered this slot, and which models Odoo refused or does not expose."""
2813
+ slot = slot if isinstance(slot, dict) else {}
2814
+ return {"source": str(slot.get("source") or ""),
2815
+ "denied": _model_names(slot.get("denied")),
2816
+ "absent": _model_names(slot.get("absent"))}
2817
+
2818
+
2819
+ def readable_model_catalog(rt, connection_id=None):
2820
+ """The last complete tenant-local catalogue; reading it never contacts Odoo."""
2821
+ return _catalog_view(_read_catalog_slot(rt, connection_id))
2822
+
2823
+
2824
+ def model_access(rt, connection_id=None):
2825
+ """This slot's explicit access facts (D-748, R70); reading them never contacts Odoo."""
2826
+ return _access_view(_read_catalog_slot(rt, connection_id))
2827
+
2828
+
2829
+ def _credential_identity(rt, connection_id=None):
2830
+ """The stored credential that answers for this slot, or "" when unknown. Decrypts nothing."""
2831
+ slot = _catalog_slot(connection_id)
2832
+ if slot != DEFAULT_CONNECTION:
2833
+ return "keychain:" + slot
2834
+ default = default_connection_id(rt)
2835
+ if default:
2836
+ return "keychain:" + str(default)
2837
+ return "env" if getattr(rt, "key", None) == "royal-imports" else ""
2838
+
2839
+
2840
+ def _same_credential(stamped, current):
2841
+ """Evidence stamped by another credential is not this credential's; an unknown side matches."""
2842
+ return not stamped or not current or stamped == current
2843
+
2844
+
2845
+ def _merged_access(previous, verdicts, identity):
2846
+ """Explicit verdicts replace earlier ones; a model Odoo did not answer keeps its last verdict."""
2847
+ previous = previous or {"source": "", "denied": [], "absent": []}
2848
+ denied = set(previous["denied"])
2849
+ for model, verdict in verdicts.items():
2850
+ if verdict is True:
2851
+ denied.discard(model)
2852
+ elif verdict is False:
2853
+ denied.add(model)
2854
+ # An answered ir.model listing is the exposure fact; without one the last fact stands.
2855
+ absent = ({model for model in ODOO_TABLE_MODELS.values() if model and model not in verdicts}
2856
+ if verdicts else set(previous["absent"]))
2857
+ return {"source": identity or previous["source"], "denied": sorted(denied),
2858
+ "absent": sorted(absent)}
2859
+
2860
+
2861
+ def _write_catalog_slot(rt, connection_id, fn):
2862
+ """Rewrite one connection's slot through `fn`; every other slot is carried unchanged."""
2863
+ slot_key = _catalog_slot(connection_id)
2864
+
2865
+ def _cache(current_cache):
2866
+ current_cache = dict(current_cache) if isinstance(current_cache, dict) else {}
2867
+ connections = dict(current_cache.get("connections") or {})
2868
+ connections[slot_key] = fn(dict(connections.get(slot_key) or {}))
2869
+ return {"version": MODEL_CATALOG_VERSION, "connections": connections}
2870
+
2871
+ rt.update(MODEL_CATALOG_KEY, _cache, flush="sync")
2872
+
2873
+
2874
+ def _narrow_catalog(rt, connection_id, current, previous, access):
2875
+ """Withdraw refused and unexposed models from a retained slot; nothing here widens it."""
2876
+ withdrawn = set(access["denied"]) | set(access["absent"])
2877
+ models = [row for row in current["models"] if row.get("model") not in withdrawn]
2878
+ if (models == current["models"] and access["denied"] == previous["denied"]
2879
+ and access["absent"] == previous["absent"]):
2880
+ return current
2881
+
2882
+ def _narrowed(stored):
2883
+ # A narrowing is not a capture: complete, capturedAt, errors and the stamp stay as stored.
2884
+ if not stored:
2885
+ stored = {"source": access["source"]}
2886
+ stored["models"] = [row for row in (stored.get("models") or ())
2887
+ if not (isinstance(row, dict) and row.get("model") in withdrawn)]
2888
+ stored["denied"], stored["absent"] = access["denied"], access["absent"]
2889
+ return stored
2890
+
2891
+ _write_catalog_slot(rt, connection_id, _narrowed)
2892
+ return {**current, "models": models}
2893
 
2894
 
2895
  def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False):
2896
  """Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure."""
2897
+ slot = _read_catalog_slot(rt, connection_id)
2898
+ current, previous = _catalog_view(slot), _access_view(slot)
2899
+ identity = _credential_identity(rt, connection_id)
2900
+ same = _same_credential(previous["source"], identity)
2901
+ # A slot another credential answered is never fresh for this one, whatever its age.
2902
+ if (current["capturedAt"] and not force and same
2903
  and not any(_missing_selection_metadata(row) for row in current["models"])):
2904
  try:
2905
  captured = _dt.datetime.fromisoformat(current["capturedAt"])
 
2916
  source = None
2917
  if source is None:
2918
  return current
2919
+ verdicts = {}
2920
  try:
2921
  if client is None:
2922
  from core.odoo import get_odoo
2923
  with source._bound(): # tenant credential + scope
2924
+ rows, errors = discover_readable_models(get_odoo(), verdicts)
2925
  else:
2926
+ rows, errors = discover_readable_models(client, verdicts)
2927
  except Exception as exc: # noqa: BLE001
2928
  rows, errors = [], [f"catalogue: {type(exc).__name__}: {exc}"]
2929
+ access = _merged_access(previous if same else None, verdicts, identity)
2930
  # A partial retry is not evidence that a previously readable model disappeared. Preserve a
2931
  # complete, stamped catalogue in full whenever the new pass reports any error; otherwise one
2932
  # refused model could silently shrink the enablement roster. A first-ever partial pass may be
2933
  # retained for diagnostics, but an entirely failed pass still has no catalogue to publish.
2934
+ # R70: Odoo's explicit refusal, or a model it does not expose, still withdraws that model.
2935
+ if errors and same and (not rows or (current["complete"] and current["capturedAt"])):
2936
+ return _narrow_catalog(rt, connection_id, current, previous, access)
2937
+ if errors and not rows:
2938
  return current
2939
  captured = _dt.datetime.now(_dt.timezone.utc).isoformat()
2940
  result = {"version": MODEL_CATALOG_VERSION, "complete": not errors,
2941
+ "capturedAt": captured, "models": rows, "errors": errors[:50], **access}
2942
+ _write_catalog_slot(rt, connection_id, lambda _stored: result)
2943
+ return _catalog_view(result)
 
 
 
 
 
 
 
 
2944
 
2945
 
2946
  def refresh_model_catalog_entry(rt, model, connection_id=None, client=None):
2947
  """Refresh one model's fields with two read-only calls, without enumerating other models."""
2948
+ slot = _read_catalog_slot(rt, connection_id)
2949
+ current, previous = _catalog_view(slot), _access_view(slot)
2950
+ identity = _credential_identity(rt, connection_id)
2951
+ same = _same_credential(previous["source"], identity)
2952
 
2953
  def _read_fields(bound_client):
2954
+ if not _read_access(bound_client, model):
 
 
2955
  return None
2956
  fields = bound_client.fields_get(
2957
  model, attributes=_ODOO_FIELD_ATTRIBUTES)
 
2977
  # No answer (including unavailable credentials) is not evidence of denied model access.
2978
  return current
2979
  if fields is None:
2980
+ # R70: this credential's explicit refusal withdraws the model from its own offer.
2981
+ if same and (model not in previous["denied"]
2982
+ or any(row.get("model") == model for row in current["models"])):
2983
+ def _deny(stored):
2984
+ stored = stored or {**current, "source": identity}
2985
+ stored["models"] = [row for row in (stored.get("models") or ())
2986
+ if not (isinstance(row, dict) and row.get("model") == model)]
2987
+ stored["denied"] = _model_names([*(stored.get("denied") or ()), model])
2988
+ return stored
2989
+
2990
+ _write_catalog_slot(rt, connection_id, _deny)
2991
  raise Refused(f"This Odoo account cannot read the model {model}. "
2992
  "Ask your Odoo administrator to grant read access in Odoo, not here.")
 
2993
 
2994
+ def _merge(stored):
2995
+ result = stored or {**current, "source": identity}
 
 
2996
  rows = list(result.get("models") or ())
2997
+ prior = next((row for row in rows if row.get("model") == model), None)
2998
  row = {"model": model,
2999
+ "name": " ".join(str((prior or {}).get("name") or model).split()),
3000
  "fields": fields}
3001
  result["models"] = [row if item.get("model") == model else item for item in rows]
3002
+ if prior is None:
3003
  result["models"].append(row)
3004
  # One model neither captures a whole catalogue nor changes its completeness. Preserve
3005
  # complete and capturedAt exactly, along with every unrelated row and connection slot.
3006
+ # Its explicit read verdict does clear an earlier refusal for this credential.
3007
+ if same:
3008
+ for name in ("denied", "absent"):
3009
+ if name in result:
3010
+ result[name] = [value for value in _model_names(result[name]) if value != model]
3011
+ return result
3012
+
3013
+ _write_catalog_slot(rt, connection_id, _merge)
3014
  return readable_model_catalog(rt, connection_id)
3015
 
3016
 
 
3377
  "pinned": "invoice_no", "identity": (_ROW_ID_COL, "odoo_id"),
3378
  "links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),)},
3379
 
3380
+ # Read-through like the line grains; payments name customers and vendors alike, so no link.
3381
+ {"key": PAYMENTS_KEY, "source": "account_payment", "label": "Odoo payments",
3382
+ "pinned": "odoo_id", "identity": (_ROW_ID_COL, "odoo_id"), "links": ()},
3383
+
3384
  # ⚠ RETIRED FROM `TABLES` BY W33-T44, DECLARED HERE ANYWAY, AND THAT IS THE POINT OF A SPEC.
3385
  # `RETIRED_KEYS` pops this key and `PRODUCTS_KEY` from a tenant's document on every sync, but
3386
  # the key constants, the field builders and the readers in `_READERS` all still exist, the
 
3914
  raise Refused("the database source-count mode is not supported")
3915
 
3916
 
3917
+ def source_count_buckets(model):
3918
+ """The curated buckets whose source population is counted on one Odoo model."""
3919
+ return {bucket for bucket, spec in _SOURCE_COUNT_SPECS.items() if spec["model"] == model}
3920
+
3921
+
3922
+ def capture_source_counts(rt, connection_id=None, buckets=None):
3923
  """`{bucket: (count, mode)}` for each enabled curated database, counted at the source now.
3924
 
3925
  The resync loop calls this right after the mirror pull, so reconciliation compares the mirror
3926
+ with the source as of the same moment. `buckets` limits the count to those buckets. A bucket
3927
+ whose count raises is left out, and that bucket counts fresh at reconciliation. No source or
3928
+ no client returns `{}`.
3929
  """
3930
  try:
3931
  source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
 
3942
  return {}
3943
  if client is None:
3944
  return {}
3945
+ chosen = enabled_buckets(rt, connection_id)
3946
+ if buckets is not None:
3947
+ chosen = chosen & set(buckets)
3948
+ for bucket in sorted(chosen):
3949
  try:
3950
  counts[bucket] = _source_domain_count(client, bucket)
3951
  except Exception: # noqa: BLE001
 
5476
  by a curated table is covered there; every remaining readable model receives a stable raw-model
5477
  key and defaults off. Stored choices survive catalogue refreshes because config is keyed by the
5478
  stable database key, never by its position.
5479
+
5480
+ S79-T66 / R70 (D-748): a model is withdrawn only on this credential's own evidence. Odoo's
5481
+ explicit read refusal, or a curated model it does not expose, withdraws it at any catalogue
5482
+ age or completeness; no answer keeps the curated default. A slot another credential answered
5483
+ offers no discovered model at all.
5484
  """
5485
  chosen = read_config(rt)["grids"]
5486
  namespace = connection_namespace(rt, connection_id)[1] if connection_id else None
5487
+ slot = _read_catalog_slot(rt, namespace)
5488
+ catalogue, access = _catalog_view(slot), _access_view(slot)
5489
+ own = _same_credential(access["source"], _credential_identity(rt, namespace))
5490
+ withdrawn = set(access["denied"]) | set(access["absent"]) if own else set()
5491
  fixed = [{"key": key, "label": label, "model": ODOO_TABLE_MODELS.get(key, ""),
5492
  "relevant": True, "defaultEnabled": True,
5493
  "enabled": bool(chosen.get(key, True))}
5494
  for _bucket, key, label, _fields in TABLES]
5495
+ # A row with no model names nothing the connector exposes, so it is never offered.
5496
+ fixed = [row for row in fixed if row["model"] and row["model"] not in withdrawn]
5497
  # Before discovery, keep the established curated defaults. A complete stamped catalogue is
5498
  # authoritative for this API account, so a curated database whose model is absent must not be
5499
  # offered: relevance never widens access.
5500
+ if own and catalogue["complete"] and catalogue["capturedAt"]:
5501
  readable = {str(row.get("model") or "") for row in catalogue["models"]}
5502
+ fixed = [row for row in fixed if row["model"] in readable]
5503
  covered = {row["model"] for row in fixed if row["model"]}
5504
  dynamic = []
5505
  try:
5506
  from core import connector_spec
5507
+ for item in (catalogue["models"] if own else ()):
5508
  model = str(item.get("model") or "")
5509
+ if not model or model in covered or model in withdrawn:
5510
  continue
5511
  key = connector_spec.odoo_model_table_key(model)
5512
  dynamic.append({"key": key,
 
5940
  existing = {}
5941
  if rt is not None:
5942
  try:
5943
+ existing = rt.get(ut.STORE_KEY) or {}
5944
  except Exception: # noqa: BLE001
5945
  existing = {}
5946
  # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and
 
5959
  # be eligible FOR), but the law is written without an exception, so this lends the evaluator
5960
  # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so
5961
  # what is lent is the document as it stands the moment this plan applies.
5962
+ for k in READ_THROUGH_KEYS:
5963
+ tk = table_key_for(k, connection_id)
5964
+ existing[tk] = {**(existing.get(tk) or {}), "readThrough": True}
5965
+ lent = _LentDoc(existing, ut.STORE_KEY)
 
 
5966
 
5967
  built, limits = {}, {}
5968
  caps = {}
 
6335
  generated = connector_spec.system_link_fields("odoo", live_columns)
6336
  for key, label, fields, rows in plans:
6337
  existing = {field["key"] for field in fields}
6338
+ # A declared link on the same column to the same target already serves that edge.
6339
+ declared = {((field.get("link") or {}).get("table"), (field.get("link") or {}).get("from"))
6340
+ for field in fields if field.get("type") == "link"}
6341
  fields = [*fields, *namespaced_fields(
6342
  [_preset(field) for field in generated.get(base_table_key(key), ())
6343
+ if field["key"] not in existing
6344
+ and (field["link"]["table"], field["link"]["from"]) not in declared],
6345
+ connection_id)]
6346
  written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp)
6347
  from core import connector_sync
6348
  cur[key]["connector"] = {
 
6738
  if not selected:
6739
  return {}
6740
  ut = _ut()
6741
+ existing = rt.get(ut.STORE_KEY) or {}
6742
  # ⛔ HOTFIX 2026-09-24: a model the store's byte budget already refused is NOT pulled again.
6743
  # Without this, every resync re-read all of it from Odoo (account.invoice.line.agent: 250,000
6744
  # rows) only for the store door to discard the rows. The refusal and its fix are on the wire
api/process_role.py CHANGED
@@ -1,8 +1,15 @@
1
  """One process rule for every database: heavy background work runs only in the sync worker.
2
 
3
  Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs every heavy job,
4
- `all` does both in one process (local runs and gates). Kinds: `heavy` moves source data or writes
5
- the store in the background; `light` serves this process's own reads.
 
 
 
 
 
 
 
6
  """
7
  from __future__ import annotations
8
 
@@ -10,6 +17,7 @@ import atexit
10
  import json
11
  import os
12
  import shutil
 
13
  import subprocess
14
  import sys
15
  import threading
@@ -17,13 +25,16 @@ import time
17
  import uuid
18
  from pathlib import Path
19
 
20
- ROLES = ("web", "worker", "all")
21
  KINDS = ("heavy", "light")
 
22
  API_DIR = Path(__file__).resolve().parent
23
  POLL_SECONDS = 15
24
  HEARTBEAT_SECONDS = 30
25
  HEARTBEAT_FILE = "worker.heartbeat.json"
26
  READY_FILE = "READY.json"
 
 
27
  INSTALL_ATTEMPTS = 3
28
  INCOMPLETE_GRACE_SECONDS = 3600
29
 
@@ -45,7 +56,7 @@ ROLE, ROLE_SOURCE = resolve_role()
45
  os.environ["AIOS_PROCESS_ROLE"] = ROLE
46
  if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
47
  print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
48
- if ROLE == "web":
49
  os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB")
50
  os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2")
51
 
@@ -95,9 +106,14 @@ class _Refused:
95
 
96
 
97
  def thread(*, target, name, kind, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
98
- """An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in."""
 
 
 
99
  if not admit(name, kind, role_name):
100
  return _Refused(name)
 
 
101
  if delay is None:
102
  job = threading.Thread(target=target, args=tuple(args), kwargs=dict(kwargs or {}),
103
  daemon=daemon, name=name)
@@ -130,6 +146,305 @@ def started_jobs():
130
  return {name: dict(row) for name, row in _STARTED.items()}
131
 
132
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  # ----------------------------------------------------------------------------- the hand-off dir
134
  def handoff_dir():
135
  return Path(os.environ.get("AIOS_MIRROR_HANDOFF_DIR") or "/tmp/aios-mirror-handoff")
@@ -200,9 +515,15 @@ def ready_handoffs(root=None):
200
  def _install_default(connector, tenant, ready):
201
  from harness import mirror_store, runtime
202
  rt = runtime.get_runtime(tenant)
 
203
  paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
204
- installed = mirror_store.install_handoff(connector, tenant, rt, ready["manifest"],
205
- ready.get("stateRows") or [], paths)
 
 
 
 
 
206
  try:
207
  import routes_odoo_tables
208
  routes_odoo_tables.invalidate_sources()
@@ -211,6 +532,14 @@ def _install_default(connector, tenant, ready):
211
  return installed
212
 
213
 
 
 
 
 
 
 
 
 
214
  def _remove_dir(path):
215
  shutil.rmtree(path, ignore_errors=True)
216
 
@@ -336,6 +665,7 @@ def _web_watch_loop():
336
  while not _STOP.wait(POLL_SECONDS):
337
  try:
338
  install_ready()
 
339
  except Exception as exc: # noqa: BLE001
340
  print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
341
  try:
@@ -359,7 +689,7 @@ def write_heartbeat(now=None):
359
  resync = dict(getattr(app_main, "RESYNC_STATE", None) or {})
360
  _write_json(handoff_dir() / HEARTBEAT_FILE,
361
  {"pid": os.getpid(), "at": time.time() if now is None else now,
362
- "catchUp": catch_up, "resync": resync})
363
 
364
 
365
  def _heartbeat_loop(parent_pid):
@@ -382,6 +712,8 @@ def start_worker_services():
382
  mirror_store.register_publish_listener(write_handoff)
383
  parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
384
  start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
 
 
385
  start("sync-desk-poll", "heavy", _worker_desk_loop)
386
 
387
 
@@ -391,6 +723,8 @@ def _worker_env():
391
  AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
392
  for name in ("AIOS_DUCKDB_MEMORY", "AIOS_DUCKDB_THREADS"):
393
  env.pop(name, None) # the worker sets its own caps
 
 
394
  return env
395
 
396
 
@@ -406,6 +740,7 @@ def _supervise():
406
  if proc is not None:
407
  _WORKER.update(proc=proc, pid=proc.pid, startedAt=started)
408
  print(f"[process-role] sync worker started pid={proc.pid}", flush=True)
 
409
  code = proc.wait()
410
  _WORKER["lastExit"] = code
411
  else:
@@ -435,11 +770,21 @@ def stop_worker():
435
 
436
 
437
  def start_web_services():
438
- """Web role only: forward Sync now, supervise the worker, install its snapshots."""
 
 
 
439
  if ROLE != "web" or _WORKER.get("supervisor") is not None:
440
  return False
441
  from core import connector_sync
 
442
  connector_sync.register_request_listener(_forward_request)
 
 
 
 
 
 
443
  _WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise)
444
  start("mirror-handoff-watcher", "light", _web_watch_loop)
445
  atexit.register(stop_worker)
@@ -460,7 +805,8 @@ def status():
460
  age = None
461
  if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
462
  age = round(max(0.0, time.time() - float(beat["at"])), 1)
463
- return {"role": ROLE, "roleSource": ROLE_SOURCE,
464
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
465
  "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
466
- "lastInstallError": dict(LAST_INSTALL_ERROR) or None}}
 
 
1
  """One process rule for every database: heavy background work runs only in the sync worker.
2
 
3
  Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs every heavy job,
4
+ `all` does both in one process (local runs and gates), `reader` serves reads, spawns no worker and
5
+ never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
6
+ store in the background; `light` serves this process's own reads.
7
+
8
+ One sync writer per store: a worker runs its heavy jobs only while it holds the store's writer
9
+ lease, so a second Space's worker waits instead of becoming a second writer.
10
+
11
+ Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock and
12
+ supervises the sync worker and installs hand-offs. The others only read the mirror read-only.
13
  """
14
  from __future__ import annotations
15
 
 
17
  import json
18
  import os
19
  import shutil
20
+ import socket
21
  import subprocess
22
  import sys
23
  import threading
 
25
  import uuid
26
  from pathlib import Path
27
 
28
+ ROLES = ("web", "worker", "all", "reader")
29
  KINDS = ("heavy", "light")
30
+ SERVING_ROLES = ("web", "all", "reader")
31
  API_DIR = Path(__file__).resolve().parent
32
  POLL_SECONDS = 15
33
  HEARTBEAT_SECONDS = 30
34
  HEARTBEAT_FILE = "worker.heartbeat.json"
35
  READY_FILE = "READY.json"
36
+ PRIMARY_LOCK = "web-primary.lock"
37
+ PRIMARY_FILE = "web-primary.json"
38
  INSTALL_ATTEMPTS = 3
39
  INCOMPLETE_GRACE_SECONDS = 3600
40
 
 
56
  os.environ["AIOS_PROCESS_ROLE"] = ROLE
57
  if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
58
  print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
59
+ if ROLE in ("web", "reader"):
60
  os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB")
61
  os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2")
62
 
 
106
 
107
 
108
  def thread(*, target, name, kind, daemon=True, args=(), kwargs=None, delay=None, role_name=None):
109
+ """An unstarted Thread, or a Timer when delay is given. A refused job gets an inert stand-in.
110
+
111
+ While the writer lease is armed, a heavy job's target first waits until this process holds it.
112
+ """
113
  if not admit(name, kind, role_name):
114
  return _Refused(name)
115
+ if kind == "heavy" and _WRITER["armed"]:
116
+ target = _after_writer_lease(target, name)
117
  if delay is None:
118
  job = threading.Thread(target=target, args=tuple(args), kwargs=dict(kwargs or {}),
119
  daemon=daemon, name=name)
 
146
  return {name: dict(row) for name, row in _STARTED.items()}
147
 
148
 
149
+ # ----------------------------------------------------------------------------- the writer lease
150
+ WRITER_LEASE_DOC = "sync_writer_lease"
151
+ WRITER_LEASE_SECONDS = 180
152
+ WRITER_POLL_SECONDS = 15
153
+ _WRITER = {"armed": False, "holder": None, "renewedAt": None, "lostAt": None, "waitingSince": None,
154
+ "error": None}
155
+ _WRITER_HELD = threading.Event()
156
+
157
+
158
+ def writer_holder(env=None):
159
+ """This container's lease identity, so a restarted worker in it takes its predecessor's lease."""
160
+ env = os.environ if env is None else env
161
+ return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
162
+
163
+
164
+ def acquire_writer(rt, holder=None, *, now=None, ttl=None):
165
+ """Take or renew the store's writer lease. True when `holder` holds it after the call.
166
+
167
+ The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
168
+ """
169
+ holder = holder or writer_holder()
170
+ stamp = time.time() if now is None else float(now)
171
+ ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
172
+
173
+ def take(doc):
174
+ doc = dict(doc or {})
175
+ current = doc.get("holder")
176
+ expired = stamp - float(doc.get("renewedAt") or 0) > ttl
177
+ if current in (None, "", holder) or expired:
178
+ if current != holder:
179
+ doc.update(acquiredAt=stamp, previous=current)
180
+ doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
181
+ return doc
182
+
183
+ doc = rt.update(WRITER_LEASE_DOC, take)
184
+ if not isinstance(doc, dict):
185
+ doc = rt.get(WRITER_LEASE_DOC, fresh=True)
186
+ return isinstance(doc, dict) and doc.get("holder") == holder
187
+
188
+
189
+ def release_writer(rt, holder=None):
190
+ """Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
191
+ holder = holder or writer_holder()
192
+
193
+ def drop(doc):
194
+ doc = dict(doc or {})
195
+ if doc.get("holder") == holder:
196
+ doc.update(holder=None, renewedAt=0, previous=holder)
197
+ return doc
198
+
199
+ rt.update(WRITER_LEASE_DOC, drop)
200
+
201
+
202
+ def holds_writer():
203
+ return _WRITER_HELD.is_set()
204
+
205
+
206
+ def may_write(role_name=None):
207
+ """allows('heavy'), plus the writer lease when armed. For a store write outside a heavy thread."""
208
+ return allows("heavy", role_name) and (not _WRITER["armed"] or _WRITER_HELD.is_set())
209
+
210
+
211
+ def _after_writer_lease(target, name):
212
+ """Wrap a heavy job so it starts only after this process holds the writer lease."""
213
+ def gated(*args, **kwargs):
214
+ if not _WRITER_HELD.is_set():
215
+ print(f"[process-role] {name} waits for the sync writer lease", flush=True)
216
+ _WRITER_HELD.wait()
217
+ return target(*args, **kwargs)
218
+ gated.__name__ = getattr(target, "__name__", name)
219
+ return gated
220
+
221
+
222
+ def writer_lease_tick(rt_factory, holder, *, now=None, ttl=None):
223
+ """One acquire or renew. Returns 'held', 'waiting' or 'lost'. 'lost' means stop writing now."""
224
+ stamp = time.time() if now is None else float(now)
225
+ ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
226
+ try:
227
+ held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl)
228
+ except Exception as exc: # noqa: BLE001
229
+ held = None
230
+ if _WRITER.get("error") != type(exc).__name__:
231
+ # Fail closed: no lease, no heavy job. Set AIOS_SYNC_WRITER_TENANT to a writable tenant.
232
+ print(f"[process-role] sync writer lease unreadable ({type(exc).__name__}); heavy jobs wait",
233
+ flush=True)
234
+ _WRITER["error"] = type(exc).__name__
235
+ else:
236
+ _WRITER["error"] = None
237
+ if held:
238
+ if not _WRITER_HELD.is_set():
239
+ print(f"[process-role] sync writer lease held by {holder}", flush=True)
240
+ _WRITER.update(renewedAt=stamp, waitingSince=None)
241
+ _WRITER_HELD.set()
242
+ return "held"
243
+ if _WRITER_HELD.is_set():
244
+ # Another holder took it, or renewals failed long enough that one may take it next.
245
+ if held is False or stamp - float(_WRITER["renewedAt"] or 0) > ttl - 2 * WRITER_POLL_SECONDS:
246
+ _WRITER_HELD.clear()
247
+ _WRITER["lostAt"] = stamp
248
+ print("[process-role] sync writer lease lost; this worker stops writing", flush=True)
249
+ return "lost"
250
+ return "held"
251
+ if _WRITER["waitingSince"] is None:
252
+ _WRITER["waitingSince"] = stamp
253
+ print("[process-role] another process holds the sync writer lease; heavy jobs wait", flush=True)
254
+ return "waiting"
255
+
256
+
257
+ def _writer_lease_loop(rt_factory, holder, exit_fn):
258
+ while True:
259
+ if writer_lease_tick(rt_factory, holder) == "lost":
260
+ exit_fn(3)
261
+ return
262
+ if _STOP.wait(WRITER_POLL_SECONDS):
263
+ return
264
+
265
+
266
+ def _writer_runtime():
267
+ from harness import runtime
268
+ return runtime.get_runtime(os.environ.get("AIOS_SYNC_WRITER_TENANT") or "royal-imports")
269
+
270
+
271
+ def _release_at_exit():
272
+ if _WRITER_HELD.is_set():
273
+ try:
274
+ release_writer(_writer_runtime(), _WRITER["holder"])
275
+ except Exception: # noqa: BLE001
276
+ pass
277
+
278
+
279
+ def arm_writer_lease(rt_factory=None, holder=None, exit_fn=None):
280
+ """Worker only: every later heavy job waits for the lease. Call before any heavy job starts."""
281
+ if _WRITER["armed"]:
282
+ return False
283
+ _WRITER.update(armed=True, holder=holder or writer_holder())
284
+ # Light: it must run before the lease is held, and it writes only the lease document.
285
+ start("sync-writer-lease", "light", _writer_lease_loop,
286
+ args=(rt_factory or _writer_runtime, _WRITER["holder"], exit_fn or os._exit))
287
+ if rt_factory is None:
288
+ atexit.register(_release_at_exit)
289
+ return True
290
+
291
+
292
+ # ----------------------------------------------------------------------------- readiness (C9)
293
+ READY_CHECKS = ("mirror", "pools", "first_reads")
294
+ _READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
295
+ "pools": "The read pools are not open yet.",
296
+ "first_reads": "The first reads have not answered yet."}
297
+ _READINESS = {}
298
+ _BOOT_RESTORE = {}
299
+ # The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
300
+ # must not pull a warm web process out of rotation.
301
+ _READY_LATCH = {"at": None}
302
+
303
+
304
+ def reset_readiness():
305
+ _READY_LATCH["at"] = None
306
+ _READINESS.clear()
307
+ _READINESS.update({name: {"ok": False, "detail": text, "at": None} for name, text in _READY_DEFAULT.items()})
308
+
309
+
310
+ reset_readiness()
311
+
312
+
313
+ def mark_ready(check, ok, detail=""):
314
+ """Record one readiness check. A check passes only after its own step finished in this process."""
315
+ if check not in _READY_DEFAULT:
316
+ raise ValueError(f"unknown readiness check {check!r}")
317
+ _READINESS[check] = {"ok": bool(ok), "detail": "" if ok else (detail or _READY_DEFAULT[check]),
318
+ "at": time.time()}
319
+
320
+
321
+ def note_boot_restore(state="done"):
322
+ """Worker side: its boot restore finished, so every hand-off it made is already on disk."""
323
+ _BOOT_RESTORE.update(state=state, at=time.time(), pid=os.getpid())
324
+ try:
325
+ write_heartbeat()
326
+ except Exception as exc: # noqa: BLE001
327
+ print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
328
+
329
+
330
+ def worker_boot_restored(root=None):
331
+ """Web side: the current worker's heartbeat says its boot restore finished."""
332
+ beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
333
+ restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
334
+ # A secondary web worker supervises nothing: the primary names the current sync worker.
335
+ pid = _WORKER.get("pid") if is_primary() else primary_state(root).get("workerPid")
336
+ return bool(restore.get("state")) and pid is not None and beat.get("pid") == pid
337
+
338
+
339
+ def _installed_map(root=None):
340
+ if is_primary():
341
+ return dict(_INSTALLED)
342
+ installed = primary_state(root).get("installed") or {}
343
+ return {tuple(key.split("/", 1)): snap for key, snap in installed.items() if "/" in key}
344
+
345
+
346
+ def pending_handoffs(root=None):
347
+ """`tenant/connector` for each newest READY hand-off not installed yet (by the primary)."""
348
+ installed = _installed_map(root)
349
+ return sorted(f"{key[0]}/{key[1]}" for key, rows in ready_handoffs(root).items()
350
+ if installed.get(key) != (rows[-1][2].get("snapshotId") or rows[-1][1].name))
351
+
352
+
353
+ # ----------------------------------------------------------------------------- several web workers
354
+ _PRIMARY = {"fd": None}
355
+
356
+
357
+ def web_workers(env=None):
358
+ """How many uvicorn workers serve this container, from AIOS_UVICORN_WORKERS. 1 when unset."""
359
+ env = os.environ if env is None else env
360
+ try:
361
+ return max(1, int(env.get("AIOS_UVICORN_WORKERS") or 1))
362
+ except ValueError:
363
+ return 1
364
+
365
+
366
+ def is_primary():
367
+ """True when this web process supervises the sync worker and installs. Always for one worker."""
368
+ return web_workers() == 1 or _PRIMARY["fd"] is not None
369
+
370
+
371
+ def claim_primary(root=None):
372
+ """Take the container's primary lock without waiting. True when this process holds it.
373
+
374
+ The lock is a kernel flock: it drops when its holder dies, so a standby can take over.
375
+ """
376
+ if is_primary():
377
+ return True
378
+ try:
379
+ import fcntl
380
+ except ImportError:
381
+ return False
382
+ path = Path(root or handoff_dir()) / PRIMARY_LOCK
383
+ path.parent.mkdir(parents=True, exist_ok=True)
384
+ fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o600)
385
+ try:
386
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
387
+ except OSError:
388
+ os.close(fd)
389
+ return False
390
+ _PRIMARY["fd"] = fd
391
+ print(f"[process-role] web worker pid={os.getpid()} is the primary: it supervises the sync worker "
392
+ f"and installs hand-offs", flush=True)
393
+ write_primary_state(root)
394
+ return True
395
+
396
+
397
+ def write_primary_state(root=None):
398
+ """Primary only: the sync worker pid and every installed snapshot, for the secondaries."""
399
+ if web_workers() == 1 or _PRIMARY["fd"] is None:
400
+ return False
401
+ try:
402
+ from harness import datastore
403
+ version = datastore.serving_file()[1]
404
+ except Exception: # noqa: BLE001
405
+ version = None
406
+ _write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
407
+ "pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
408
+ "installed": {f"{key[0]}/{key[1]}": snap for key, snap in _INSTALLED.items()},
409
+ "mirrorVersion": version})
410
+ return True
411
+
412
+
413
+ def primary_state(root=None):
414
+ state = _read_json(Path(root or handoff_dir()) / PRIMARY_FILE)
415
+ return state if isinstance(state, dict) else {}
416
+
417
+
418
+ def _standby_loop():
419
+ while not _STOP.wait(POLL_SECONDS):
420
+ if claim_primary():
421
+ start_web_services()
422
+ return
423
+
424
+
425
+ def readiness(mirror_live=None, role_name=None, root=None):
426
+ """{ready, checks: {name: bool}, failing: [names], detail: {name: text}} for GET /api/v1/ready."""
427
+ role_name = role_name or ROLE
428
+ checks, detail = {}, {}
429
+ for name in READY_CHECKS:
430
+ row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
431
+ checks[name], detail[name] = bool(row["ok"]), row["detail"]
432
+ boot = role_name == "web" and _READY_LATCH["at"] is None
433
+ if "mirror" in checks and checks["mirror"]:
434
+ if mirror_live is False:
435
+ checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
436
+ elif boot and not worker_boot_restored(root):
437
+ checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
438
+ elif boot and pending_handoffs(root):
439
+ checks["mirror"] = False
440
+ detail["mirror"] = "Mirror hand-offs are not installed yet: " + ", ".join(pending_handoffs(root)) + "."
441
+ failing = [name for name in checks if not checks[name]]
442
+ if not failing and _READY_LATCH["at"] is None:
443
+ _READY_LATCH["at"] = time.time()
444
+ return {"ready": not failing, "checks": checks, "failing": failing,
445
+ "detail": {name: detail[name] for name in failing}}
446
+
447
+
448
  # ----------------------------------------------------------------------------- the hand-off dir
449
  def handoff_dir():
450
  return Path(os.environ.get("AIOS_MIRROR_HANDOFF_DIR") or "/tmp/aios-mirror-handoff")
 
515
  def _install_default(connector, tenant, ready):
516
  from harness import mirror_store, runtime
517
  rt = runtime.get_runtime(tenant)
518
+ from harness import datastore
519
  paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
520
+ resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
521
+ with (datastore.versioned_write(tenant) if resident else _no_version()) as version:
522
+ installed = mirror_store.install_handoff(connector, tenant, rt, ready["manifest"],
523
+ ready.get("stateRows") or [], paths)
524
+ if version is not None:
525
+ # Already resident: keep serving the current version, publish nothing.
526
+ version["publish"] = installed is not False
527
  try:
528
  import routes_odoo_tables
529
  routes_odoo_tables.invalidate_sources()
 
532
  return installed
533
 
534
 
535
+ class _no_version:
536
+ def __enter__(self):
537
+ return None
538
+
539
+ def __exit__(self, *exc):
540
+ return False
541
+
542
+
543
  def _remove_dir(path):
544
  shutil.rmtree(path, ignore_errors=True)
545
 
 
665
  while not _STOP.wait(POLL_SECONDS):
666
  try:
667
  install_ready()
668
+ write_primary_state()
669
  except Exception as exc: # noqa: BLE001
670
  print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
671
  try:
 
689
  resync = dict(getattr(app_main, "RESYNC_STATE", None) or {})
690
  _write_json(handoff_dir() / HEARTBEAT_FILE,
691
  {"pid": os.getpid(), "at": time.time() if now is None else now,
692
+ "catchUp": catch_up, "resync": resync, "bootRestore": dict(_BOOT_RESTORE)})
693
 
694
 
695
  def _heartbeat_loop(parent_pid):
 
712
  mirror_store.register_publish_listener(write_handoff)
713
  parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
714
  start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
715
+ # Before the first heavy job: each one now waits until this worker holds the writer lease.
716
+ arm_writer_lease()
717
  start("sync-desk-poll", "heavy", _worker_desk_loop)
718
 
719
 
 
723
  AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
724
  for name in ("AIOS_DUCKDB_MEMORY", "AIOS_DUCKDB_THREADS"):
725
  env.pop(name, None) # the worker sets its own caps
726
+ # The worker writes its own file read-write and is never a web worker.
727
+ env.update(AIOS_MIRROR_VERSIONS="0", AIOS_UVICORN_WORKERS="1")
728
  return env
729
 
730
 
 
740
  if proc is not None:
741
  _WORKER.update(proc=proc, pid=proc.pid, startedAt=started)
742
  print(f"[process-role] sync worker started pid={proc.pid}", flush=True)
743
+ write_primary_state()
744
  code = proc.wait()
745
  _WORKER["lastExit"] = code
746
  else:
 
770
 
771
 
772
  def start_web_services():
773
+ """Web role only: forward Sync now, supervise the worker, install its snapshots.
774
+
775
+ With several uvicorn workers only the primary does this. The others stand by and read.
776
+ """
777
  if ROLE != "web" or _WORKER.get("supervisor") is not None:
778
  return False
779
  from core import connector_sync
780
+ # Every web worker forwards the Sync now it accepts: the desk lives in the store.
781
  connector_sync.register_request_listener(_forward_request)
782
+ if not claim_primary():
783
+ if _WORKER.get("standby") is None:
784
+ _WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
785
+ print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
786
+ f"the primary supervises the sync worker", flush=True)
787
+ return False
788
  _WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise)
789
  start("mirror-handoff-watcher", "light", _web_watch_loop)
790
  atexit.register(stop_worker)
 
805
  age = None
806
  if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
807
  age = round(max(0.0, time.time() - float(beat["at"])), 1)
808
+ return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
809
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
810
  "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
811
+ "lastInstallError": dict(LAST_INSTALL_ERROR) or None},
812
+ "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
api/providers.py CHANGED
@@ -34,7 +34,9 @@ lets a tenant be moved off a vendor without a release.
34
  from __future__ import annotations
35
 
36
  import json
 
37
  import os
 
38
  import time
39
  from dataclasses import dataclass, field
40
  from datetime import datetime, timezone
@@ -573,7 +575,86 @@ def llm_chain(capability="llm_tool_calling"):
573
  raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip()
574
  names = [x.strip() for x in raw.split(",") if x.strip() in LLM_DEFAULT_ORDER] or list(LLM_DEFAULT_ORDER)
575
  return [LLM_PROVIDERS[n] for n in names
576
- if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
577
 
578
 
579
  #: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND
@@ -743,6 +824,20 @@ def anthropic_send(req, timeout=None, use_sdk=None):
743
  later check in that process running on the wrong path and passing for the wrong reason.
744
  """
745
  sdk = None if use_sdk is False else anthropic_sdk()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
746
  payload = dict(req.get("json") or {})
747
  if sdk is None:
748
  r = requests.post(req["url"], headers=req["headers"], json=payload,
@@ -754,7 +849,8 @@ def anthropic_send(req, timeout=None, use_sdk=None):
754
 
755
  key = (req.get("headers") or {}).get("x-api-key") or ""
756
  try:
757
- client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60))
 
758
  msg = client.messages.create(**payload)
759
  # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same
760
  # wire-shaped dict the raw POST parses out of the response body.
 
34
  from __future__ import annotations
35
 
36
  import json
37
+ import math
38
  import os
39
+ import threading
40
  import time
41
  from dataclasses import dataclass, field
42
  from datetime import datetime, timezone
 
575
  raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip()
576
  names = [x.strip() for x in raw.split(",") if x.strip() in LLM_DEFAULT_ORDER] or list(LLM_DEFAULT_ORDER)
577
  return [LLM_PROVIDERS[n] for n in names
578
+ if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)
579
+ and not llm_breaker_open(n)]
580
+
581
+
582
+ # ═══════════ LLM CIRCUIT BREAKER, ONE PER PROVIDER NAME (S79-T43) ═══════════════════════════════
583
+ # After N consecutive failures (transport, 408, 5xx) a provider opens for a cooldown and is skipped.
584
+ # After the cooldown it half-opens: the next recorded outcome closes it or re-opens it.
585
+ # ⛔ 400, 401, 402, 404, 413 and 429 are ANSWERS, so they never count. Credit has `_NO_CREDIT`.
586
+ # ⚠ Process-local like `_NO_CREDIT`: a latency guard, not a record.
587
+ _monotonic = time.monotonic # Gates patch this clock so no leg sleeps.
588
+ _LLM_BREAKERS: dict[str, dict] = {} # name -> {"failures": int, "opened_at": float | None, "trial": bool}
589
+ _LLM_BREAKER_LOCK = threading.Lock()
590
+
591
+
592
+ def _llm_breaker_setting(name, default, minimum):
593
+ try:
594
+ value = float(os.environ.get(name) or default)
595
+ return max(minimum, value) if math.isfinite(value) else float(default)
596
+ except ValueError:
597
+ return float(default)
598
+
599
+
600
+ def _llm_breaker_cooling(state):
601
+ """Is an open breaker still inside its cooldown?"""
602
+ cooldown = _llm_breaker_setting("AIOS_LLM_BREAKER_COOLDOWN_S", 30, 0)
603
+ return _monotonic() - state["opened_at"] < cooldown
604
+
605
+
606
+ def llm_breaker_record(name, status):
607
+ """Record one call outcome. 0 is a transport exception, 2xx closes, 0, 408 and 5xx count."""
608
+ code = int(status or 0)
609
+ with _LLM_BREAKER_LOCK:
610
+ if 200 <= code <= 299:
611
+ _LLM_BREAKERS.pop(str(name), None)
612
+ return
613
+ if not (code == 0 or code == 408 or 500 <= code <= 599):
614
+ if str(name) in _LLM_BREAKERS:
615
+ _LLM_BREAKERS[str(name)]["trial"] = False # An answer ends a trial, count unchanged.
616
+ return
617
+ state = _LLM_BREAKERS.setdefault(str(name), {"failures": 0, "opened_at": None, "trial": False})
618
+ state["failures"] += 1
619
+ limit = _llm_breaker_setting("AIOS_LLM_BREAKER_FAILURES", 5, 1)
620
+ if state["opened_at"] is not None or state["failures"] >= limit:
621
+ state["opened_at"] = _monotonic()
622
+ state["trial"] = False
623
+
624
+
625
+ def llm_breaker_open(name):
626
+ """Is this provider refused right now? False while closed or while a half-open trial is allowed."""
627
+ with _LLM_BREAKER_LOCK:
628
+ state = _LLM_BREAKERS.get(str(name))
629
+ if state is None or state["opened_at"] is None:
630
+ return False
631
+ return bool(state["trial"] or _llm_breaker_cooling(state))
632
+
633
+
634
+ def _llm_breaker_admit(name):
635
+ """Admit one call: True for the half-open trial, False when closed, None while open."""
636
+ with _LLM_BREAKER_LOCK:
637
+ state = _LLM_BREAKERS.get(str(name))
638
+ if state is None or state["opened_at"] is None:
639
+ return False
640
+ if state["trial"] or _llm_breaker_cooling(state):
641
+ return None
642
+ state["trial"] = True
643
+ return True
644
+
645
+
646
+ def llm_breaker_reset(name=None):
647
+ """Close one breaker, or all of them. For a gate."""
648
+ with _LLM_BREAKER_LOCK:
649
+ if name is None:
650
+ _LLM_BREAKERS.clear()
651
+ else:
652
+ _LLM_BREAKERS.pop(str(name), None)
653
+
654
+
655
+ def _llm_max_retries():
656
+ """SDK retries per call. 0 bounds one call by its own timeout."""
657
+ return int(_llm_breaker_setting("AIOS_LLM_MAX_RETRIES", 0, 0))
658
 
659
 
660
  #: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND
 
824
  later check in that process running on the wrong path and passing for the wrong reason.
825
  """
826
  sdk = None if use_sdk is False else anthropic_sdk()
827
+ transport = "http" if sdk is None else "sdk"
828
+ if _llm_breaker_admit("anthropic") is None:
829
+ return 503, {"error": {"type": "llm_breaker_open", "message": (
830
+ "Anthropic failed several times in a row, so calls pause briefly before the next try.")}}, transport
831
+ status = 0
832
+ try:
833
+ status, body, transport = _anthropic_post(sdk, req, timeout)
834
+ return status, body, transport
835
+ finally:
836
+ llm_breaker_record("anthropic", status)
837
+
838
+
839
+ def _anthropic_post(sdk, req, timeout):
840
+ """One Anthropic call on the chosen transport. A raised exception reaches the breaker as 0."""
841
  payload = dict(req.get("json") or {})
842
  if sdk is None:
843
  r = requests.post(req["url"], headers=req["headers"], json=payload,
 
849
 
850
  key = (req.get("headers") or {}).get("x-api-key") or ""
851
  try:
852
+ client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60),
853
+ max_retries=_llm_max_retries())
854
  msg = client.messages.create(**payload)
855
  # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same
856
  # wire-shaped dict the raw POST parses out of the response body.
api/registry_source.py CHANGED
@@ -185,12 +185,36 @@ def generation(session, table_key):
185
  if not callable(revision):
186
  return None # Ordinary offline readers retain their original snapshot semantics.
187
  tokens = {name: revision(name) for name in names}
188
- return copy.deepcopy((session.user, tokens))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
189
 
190
 
191
  def require_generation(session, table_key, before):
192
- if before is not None and generation(session, table_key) != before:
193
- raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
 
 
 
 
 
 
194
 
195
 
196
  class BatchStore:
@@ -225,6 +249,46 @@ class BatchStore:
225
  return {pid: value for pid, value in zip(ids, paths(self._runtime, name,
226
  [[*path, pid] for pid in ids])) if value is not None}
227
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
  def update(self, *args, **kwargs):
229
  raise err(503, 'canonical_source_read_only', 'Refresh the database definition before reading its rows.')
230
 
@@ -305,8 +369,12 @@ class BatchStore:
305
  [*operand.params, *selected]).fetchall()]
306
  else:
307
  # Metadata consumes no row memberships; retain only the curated witness.
308
- with closing(values()) as source:
309
- next(source, None)
 
 
 
 
310
  kept = []
311
  if saw_any[0] and first[0] is None:
312
  # Config sanitization drops invalid entries; [] would mean all rows.
@@ -319,9 +387,9 @@ class BatchStore:
319
  for owner, ws in meta.items():
320
  if not isinstance(ws, dict) or owner.startswith('_') or owner != self.uname:
321
  continue
322
- values = paths(self._runtime, name, [[owner, 'overlays', pid] for pid in self.row_ids])
323
  # Only this owner's selected cells have been read, so admit a new bounded document.
324
- meta[owner] = dict(ws, overlays={pid: value for pid, value in zip(self.row_ids, values)
325
  if isinstance(value, dict)})
326
  return meta
327
 
@@ -408,17 +476,23 @@ class CanonicalStore(BatchStore):
408
  [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
409
  for key, meta in definitions.items() if isinstance(meta, dict)}}
410
  elif name == 'loopable_route_system':
411
- value = object_values(self._runtime, name, omit=('routes', 'zones'))
 
 
 
 
 
 
 
 
 
412
  value['routes'], value['zones'] = [], []
413
- for batch in array_values(self._runtime, name, ['routes'], drop=('stops',)):
414
- for index, route in batch:
415
- if isinstance(route, dict):
416
- route = dict(route, stops=members(self._runtime, name, ['routes', index, 'stops'],
417
- self.row_ids, id_key='customerPid'))
418
- value['routes'].append(route)
419
- for batch in array_values(self._runtime, name, ['zones'], drop=(
420
- 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')):
421
- value['zones'].extend(meta for _, meta in batch)
422
  elif name == 'table_zones':
423
  value = {self.table_key: []}
424
  for batch in array_values(self._runtime, name, [self.table_key], drop=(
@@ -442,12 +516,13 @@ class CanonicalStore(BatchStore):
442
  return super().get_entry(name, entry)
443
 
444
 
445
- def canonical_metadata(session, table_key):
446
  """Read the canonical permitted definitions without constructing source rows."""
447
  import routes_customers as customers
448
  import routes_products as products
449
  session.require(table_key)
450
- lent = CanonicalStore(session, table_key, [], cur=None, metadata={}, needed=frozenset())
 
451
  actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
452
  build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
453
  return build(actor, scope='product' if table_key == 'product_data' else 'customer',
@@ -482,8 +557,8 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
482
  session.require(table_key)
483
  team, agent = perm_scope.derive_pool_scope(session.user, table_key)
484
  before = generation(session, table_key)
485
- metadata = {}
486
- definition = canonical_metadata(session, table_key)
487
  # Every permanent wall operand is populated before apply_row_scope judges the batch.
488
  tree = perm_scope.entry(session.user, table_key) or {}
489
  wall = tree.get('filter') or {}
 
185
  if not callable(revision):
186
  return None # Ordinary offline readers retain their original snapshot semantics.
187
  tokens = {name: revision(name) for name in names}
188
+ user = session.user if isinstance(session.user, dict) else {}
189
+ # A sign in or activity stamp never changes what the reader may see, so it never forces a rebuild.
190
+ policy = {key: value for key, value in user.items() if key not in USER_NOISE}
191
+ return copy.deepcopy((policy, tokens))
192
+
193
+
194
+ #: User record keys that change on sign in or use and never change what the reader may see.
195
+ USER_NOISE = frozenset(('last_login', 'last_active', 'updated_at', 'epoch', 'avatar', 'whatsapp'))
196
+
197
+
198
+ def moved_keys(before, after):
199
+ """Sorted names of the generation parts that differ: store keys, plus `user` for the policy."""
200
+ if not (isinstance(before, tuple) and isinstance(after, tuple)
201
+ and len(before) == len(after) == 2):
202
+ return ['unknown']
203
+ moved = ['user'] if before[0] != after[0] else []
204
+ tokens_before, tokens_after = before[1] or {}, after[1] or {}
205
+ return moved + sorted(str(name) for name in set(tokens_before) | set(tokens_after)
206
+ if tokens_before.get(name) != tokens_after.get(name))
207
 
208
 
209
  def require_generation(session, table_key, before):
210
+ if before is None:
211
+ return
212
+ after = generation(session, table_key)
213
+ if after != before:
214
+ exc = err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
215
+ # The rebuild log names the moved keys. They never reach the client.
216
+ exc.moved = moved_keys(before, after)
217
+ raise exc
218
 
219
 
220
  class BatchStore:
 
249
  return {pid: value for pid, value in zip(ids, paths(self._runtime, name,
250
  [[*path, pid] for pid in ids])) if value is not None}
251
 
252
+ def _selected_json(self, name, path, *, array=False, id_key=None):
253
+ """Reuse one request-owned SQL copy of an overlay or route membership across batches."""
254
+ import json
255
+ cur = getattr(self, '_canonical_source_cursor', None)
256
+ if cur is None or not self.metadata.get('_scan'):
257
+ if array:
258
+ return members(self._runtime, name, path, self.row_ids, id_key=id_key)
259
+ return self.selected_values(name, path, self.row_ids)
260
+ key = ('selected_json', name, tuple(path), array, id_key)
261
+ if key not in self.metadata:
262
+ from routes_odoo_tables import _RelationLease
263
+ lease = _RelationLease(cur)
264
+ table = lease.table('_json')
265
+ cur.execute(f'CREATE TEMP TABLE "{table}" (ordinal BIGINT, pid VARCHAR, body JSON)')
266
+ source = array_values(self._runtime, name, path) if array else items(self._runtime, name, path)
267
+ offset = 0
268
+ with closing(source):
269
+ for batch in source:
270
+ records = []
271
+ for pid, value in batch:
272
+ raw = value.get(id_key) if id_key and isinstance(value, dict) else value
273
+ try:
274
+ key_pid = str(int(raw)) if array else str(pid)
275
+ except (TypeError, ValueError):
276
+ continue
277
+ records.append((key_pid, json.dumps(value)))
278
+ if records:
279
+ cur.execute(f'INSERT INTO "{table}" SELECT unnest(?::BIGINT[]), '
280
+ 'unnest(?::VARCHAR[]), unnest(?::JSON[])',
281
+ [list(range(offset, offset + len(records))),
282
+ [pid for pid, _ in records], [body for _, body in records]])
283
+ offset += len(records)
284
+ self.metadata[key] = table, lease
285
+ table, _lease = self.metadata[key]
286
+ rows = cur.execute(f'SELECT pid, body FROM "{table}" WHERE pid IN '
287
+ '(SELECT unnest(?::VARCHAR[])) '
288
+ 'QUALIFY row_number() OVER (PARTITION BY pid ORDER BY ordinal) = 1 '
289
+ 'ORDER BY ordinal', [self.row_ids]).fetchall()
290
+ return [json.loads(body) for _, body in rows] if array else {pid: json.loads(body) for pid, body in rows}
291
+
292
  def update(self, *args, **kwargs):
293
  raise err(503, 'canonical_source_read_only', 'Refresh the database definition before reading its rows.')
294
 
 
369
  [*operand.params, *selected]).fetchall()]
370
  else:
371
  # Metadata consumes no row memberships; retain only the curated witness.
372
+ witness_key = ('view_witness', name, owner, vid)
373
+ if witness_key not in self.metadata:
374
+ with closing(values()) as source:
375
+ next(source, None)
376
+ self.metadata[witness_key] = first[0], saw_any[0]
377
+ first[0], saw_any[0] = self.metadata[witness_key]
378
  kept = []
379
  if saw_any[0] and first[0] is None:
380
  # Config sanitization drops invalid entries; [] would mean all rows.
 
387
  for owner, ws in meta.items():
388
  if not isinstance(ws, dict) or owner.startswith('_') or owner != self.uname:
389
  continue
390
+ values = self._selected_json(name, [owner, 'overlays'])
391
  # Only this owner's selected cells have been read, so admit a new bounded document.
392
+ meta[owner] = dict(ws, overlays={pid: value for pid, value in values.items()
393
  if isinstance(value, dict)})
394
  return meta
395
 
 
476
  [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
477
  for key, meta in definitions.items() if isinstance(meta, dict)}}
478
  elif name == 'loopable_route_system':
479
+ meta_key = ('route_definitions', name)
480
+ if meta_key not in self.metadata or fresh:
481
+ base = object_values(self._runtime, name, omit=('routes', 'zones'))
482
+ base['routes'] = [(index, route) for batch in array_values(self._runtime, name,
483
+ ['routes'], drop=('stops',)) for index, route in batch]
484
+ base['zones'] = [meta for batch in array_values(self._runtime, name, ['zones'], drop=(
485
+ 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')) for _, meta in batch]
486
+ self.metadata[meta_key] = base
487
+ saved = self.metadata[meta_key]
488
+ value = {key: copy.deepcopy(val) for key, val in saved.items() if key not in {'routes', 'zones'}}
489
  value['routes'], value['zones'] = [], []
490
+ for index, route in saved['routes']:
491
+ if isinstance(route, dict):
492
+ route = dict(route, stops=self._selected_json(name, ['routes', index, 'stops'],
493
+ array=True, id_key='customerPid'))
494
+ value['routes'].append(route)
495
+ value['zones'] = copy.deepcopy(saved['zones'])
 
 
 
496
  elif name == 'table_zones':
497
  value = {self.table_key: []}
498
  for batch in array_values(self._runtime, name, [self.table_key], drop=(
 
516
  return super().get_entry(name, entry)
517
 
518
 
519
+ def canonical_metadata(session, table_key, *, metadata=None):
520
  """Read the canonical permitted definitions without constructing source rows."""
521
  import routes_customers as customers
522
  import routes_products as products
523
  session.require(table_key)
524
+ lent = CanonicalStore(session, table_key, [], cur=None,
525
+ metadata={} if metadata is None else metadata, needed=frozenset())
526
  actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
527
  build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
528
  return build(actor, scope='product' if table_key == 'product_data' else 'customer',
 
557
  session.require(table_key)
558
  team, agent = perm_scope.derive_pool_scope(session.user, table_key)
559
  before = generation(session, table_key)
560
+ metadata = {'_scan': ids is None}
561
+ definition = canonical_metadata(session, table_key, metadata=metadata)
562
  # Every permanent wall operand is populated before apply_row_scope judges the batch.
563
  tree = perm_scope.entry(session.user, table_key) or {}
564
  wall = tree.get('filter') or {}
api/request_class.py CHANGED
@@ -1,18 +1,27 @@
1
- """Heavy request admission: a bounded queue so CPU-heavy reads never starve light requests.
2
-
3
- One web process on 2 vCPU serves everyone. `classify` names the heavy doors in ONE registry, and
4
- `HeavyGate` lets at most `AIOS_HEAVY_CONCURRENCY` of them run at once. A heavy request that waits
5
- longer than `AIOS_HEAVY_QUEUE_SECONDS` gets a 503 `server_busy`. Light requests never wait.
 
 
 
 
 
 
 
6
  """
7
  from __future__ import annotations
8
 
9
  import asyncio
 
10
  import json
11
  import os
12
  import re
13
  import sys
14
  import threading
15
  import time
 
16
  from urllib.parse import parse_qs
17
 
18
  BUSY_CODE = "server_busy"
@@ -36,23 +45,32 @@ def _sheet_read_heavy(query):
36
  return not (tabs == "meta" and not _flag(query, "opening"))
37
 
38
 
39
- # (methods, path regex, query predicate or None). A request is heavy when one row matches.
 
 
 
 
 
 
 
40
  HEAVY = (
41
- ({"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")),
42
- ({"GET"}, r"/api/v1/products", None),
43
- ({"GET"}, r"/api/v1/customers", None),
44
- ({"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None),
45
- ({"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None),
46
- ({"GET"}, r"/api/v1/tables/[^/]+/rows", None),
47
- ({"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy),
48
- ({"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None),
49
  # In the web role a refresh only queues for the sync worker; it computes only in worker or all.
50
- ({"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"),
51
- ({"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None),
52
- ({"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None),
53
- ({"POST"}, r"/api/v1/external/explorer/run", None),
54
  )
55
- _HEAVY = tuple((methods, re.compile(rx), pred) for methods, rx, pred in HEAVY)
 
 
56
 
57
  # MCP tools/call names that read sheets or query databases.
58
  MCP_HEAVY_TOOLS = {"sheet", "table", "data"}
@@ -88,18 +106,50 @@ def _mcp_heavy(body):
88
  return False
89
 
90
 
91
- def classify(method, path, query="", body=None):
92
- """Return "heavy" or "light" for one request. `query` is a raw query string or a parsed dict."""
93
  method = (method or "").upper()
94
  if isinstance(query, bytes):
95
  query = query.decode("latin-1")
96
  q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else (query or {})
97
- for methods, rx, pred in _HEAVY:
98
  if method in methods and rx.fullmatch(path or "") and (pred is None or pred(q)):
99
- return "heavy"
100
  if method == "POST" and path == MCP_PATH and _mcp_heavy(body):
101
- return "heavy"
102
- return "light"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
 
104
 
105
  class _Counters:
@@ -120,17 +170,40 @@ class _Counters:
120
 
121
 
122
  _COUNTERS = _Counters()
 
 
123
 
124
 
125
  def status():
126
  """Snapshot of heavy request admission for a settings or health surface."""
127
  c = _COUNTERS
 
 
 
 
 
 
 
 
 
128
  with c.lock:
129
- return {"limit": c.limit if c.limit is not None else _env_int("AIOS_HEAVY_CONCURRENCY", 2),
 
 
130
  "queueSeconds": (c.queue_seconds if c.queue_seconds is not None
131
  else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)),
132
  "inFlight": c.in_flight, "queued": c.queued,
133
- "busyRefusals": c.busy_refusals, "admitted": c.admitted}
 
 
 
 
 
 
 
 
 
 
134
 
135
 
136
  def _env_int(name, default):
@@ -147,8 +220,32 @@ def _env_float(name, default):
147
  return default
148
 
149
 
150
- async def _send_busy(send, cause="heavy_queue_timeout"):
151
- body = json.dumps({"error": {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE}}).encode()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
152
  await send({"type": "http.response.start", "status": 503,
153
  "headers": [(b"content-type", b"application/json"),
154
  (b"content-length", str(len(body)).encode()),
@@ -157,25 +254,143 @@ async def _send_busy(send, cause="heavy_queue_timeout"):
157
  await send({"type": "http.response.body", "body": body, "more_body": False})
158
 
159
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
160
  class HeavyGate:
161
- """Pure ASGI middleware. Heavy requests take a permit; light ones pass straight through."""
162
 
163
- def __init__(self, app, concurrency=None, queue_seconds=None, counters=None):
 
 
 
 
 
164
  self.app = app
165
  self.concurrency = concurrency or _env_int("AIOS_HEAVY_CONCURRENCY", 2)
166
  self.queue_seconds = (queue_seconds if queue_seconds is not None
167
  else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0))
168
  self.counters = counters or _COUNTERS
169
  self.counters.limit, self.counters.queue_seconds = self.concurrency, self.queue_seconds
170
- self._sem = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  self._loop = None
172
 
173
- def _semaphore(self):
174
- # A Semaphore binds to the first loop that waits on it. One per loop.
175
  loop = asyncio.get_running_loop()
176
- if self._sem is None or self._loop is not loop:
177
- self._sem, self._loop = asyncio.Semaphore(self.concurrency), loop
178
- return self._sem
 
 
 
 
179
 
180
  async def __call__(self, scope, receive, send):
181
  if scope.get("type") != "http":
@@ -189,23 +404,34 @@ class HeavyGate:
189
  return
190
  if classify(method, path, scope.get("query_string", b""), peek) != "heavy":
191
  return await self.app(scope, _replay(body_msgs, receive), send)
 
192
  if body_msgs is None:
193
  body_msgs, _ = await _read_body(receive, None)
194
  if body_msgs is None:
195
  return
196
- await self._run_heavy(scope, body_msgs, receive, send)
197
 
198
- async def _run_heavy(self, scope, body_msgs, receive, send):
199
  queued_at = time.perf_counter()
200
- sem = self._semaphore()
201
- c = self.counters
202
- c.add("queued", 1)
 
 
 
 
 
 
 
 
 
 
203
  timing = sys.modules.get("core.request_timing")
204
  budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError)
205
  acquire = asyncio.ensure_future(sem.acquire())
206
  gone = asyncio.ensure_future(_wait_disconnect(receive))
207
  try:
208
- timeout = timing.remaining_budget(self.queue_seconds) if timing else self.queue_seconds
209
  done, _ = await asyncio.wait({acquire, gone}, timeout=timeout,
210
  return_when=asyncio.FIRST_COMPLETED)
211
  if timing:
@@ -213,8 +439,8 @@ class HeavyGate:
213
  except budget_error:
214
  if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
215
  sem.release()
216
- c.add("busy_refusals", 1)
217
- await _send_busy(send, "request_deadline")
218
  return
219
  except BaseException:
220
  # Cancellation can race a successful acquire before the handler owns the permit.
@@ -226,7 +452,7 @@ class HeavyGate:
226
  timing = sys.modules.get("core.request_timing")
227
  if timing is not None:
228
  timing.record_queue((time.perf_counter() - queued_at) * 1000)
229
- c.add("queued", -1)
230
  if not acquire.done():
231
  acquire.cancel()
232
  if not gone.done():
@@ -240,15 +466,15 @@ class HeavyGate:
240
  except budget_error:
241
  if got:
242
  sem.release()
243
- c.add("busy_refusals", 1)
244
- await _send_busy(send, "request_deadline")
245
  return
246
  if acquire not in done or gone in done:
247
  if got:
248
  sem.release()
249
  if gone not in done:
250
- c.add("busy_refusals", 1)
251
- await _send_busy(send)
252
  return
253
  released = False
254
 
@@ -256,7 +482,7 @@ class HeavyGate:
256
  nonlocal released
257
  if not released:
258
  released = True
259
- c.add("in_flight", -1)
260
  sem.release()
261
 
262
  async def send_wrapped(message):
@@ -264,8 +490,8 @@ class HeavyGate:
264
  if message.get("type") == "http.response.body" and not message.get("more_body"):
265
  release()
266
 
267
- c.add("in_flight", 1)
268
- c.add("admitted", 1)
269
  try:
270
  await self.app(scope, _replay(body_msgs, receive), send_wrapped)
271
  finally:
@@ -307,3 +533,180 @@ def _replay(msgs, receive):
307
  return pending.pop(0)
308
  return await receive()
309
  return replay
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Heavy request admission: bounded queues so CPU-heavy reads never starve light requests.
2
+
3
+ `classify` names the heavy doors in ONE registry, each in a named request class (grid rows,
4
+ workspace, sheet, export, query). `HeavyGate` gives every class its own limit and queue bound in
5
+ each web process: no permit is shared between classes, so a full class answers a 503
6
+ `server_busy` naming itself while another class keeps admitting. Inside a class, waiting requests
7
+ are admitted round robin by tenant, so one tenant's burst queues behind itself and another
8
+ tenant's read is admitted within its queue bound. Light requests never wait.
9
+
10
+ Box sizing lives here too: `python request_class.py --env` reads the container's CPU and memory
11
+ limits and prints the launch values (uvicorn workers, heavy concurrency, DuckDB caps).
12
+ Run as a script it imports only the standard library.
13
  """
14
  from __future__ import annotations
15
 
16
  import asyncio
17
+ import hashlib
18
  import json
19
  import os
20
  import re
21
  import sys
22
  import threading
23
  import time
24
+ from collections import deque
25
  from urllib.parse import parse_qs
26
 
27
  BUSY_CODE = "server_busy"
 
45
  return not (tabs == "meta" and not _flag(query, "opening"))
46
 
47
 
48
+ #: The heavy request classes. Each has its own limit (`AIOS_HEAVY_CONCURRENCY_<CLASS>`, default
49
+ #: `AIOS_HEAVY_CONCURRENCY` for grid_rows and half of it for the others, see `class_limit`) and
50
+ #: queue bound (`AIOS_HEAVY_QUEUE_SECONDS_<CLASS>`, default `AIOS_HEAVY_QUEUE_SECONDS`).
51
+ CLASSES = ("grid_rows", "workspace", "sheet", "export", "query")
52
+ #: The class a heavy request with no registry row lands in (a caller that names no class).
53
+ DEFAULT_CLASS = "grid_rows"
54
+
55
+ # (class, methods, path regex, query predicate or None). A request is heavy when one row matches.
56
  HEAVY = (
57
+ ("workspace", {"GET"}, r"/api/v1/workspace", lambda q: not _flag(q, "fields_only")),
58
+ ("grid_rows", {"GET"}, r"/api/v1/products", None),
59
+ ("grid_rows", {"GET"}, r"/api/v1/customers", None),
60
+ ("grid_rows", {"GET", "POST"}, r"/api/v1/odoo-tables/[^/]+/rows", None),
61
+ ("grid_rows", {"GET", "POST"}, r"/api/v1/connected-tables/[^/]+/rows", None),
62
+ ("grid_rows", {"GET"}, r"/api/v1/tables/[^/]+/rows", None),
63
+ ("sheet", {"GET"}, r"/api/v1/sheets/(?!index$|import$|automation$)[^/]+", _sheet_read_heavy),
64
+ ("sheet", {"GET"}, r"/api/v1/sheets/[^/]+/tabs/[^/]+", None),
65
  # In the web role a refresh only queues for the sync worker; it computes only in worker or all.
66
+ ("sheet", {"POST"}, r"/api/v1/sheets/[^/]+/refresh", lambda q: os.environ.get("AIOS_PROCESS_ROLE") != "web"),
67
+ ("export", {"GET", "POST"}, r"/api/v1/.*(?:/export|-export)(?:/.*)?", None),
68
+ ("query", {"POST"}, r"/api/v1/(?:external|typed)/databases/[^/]+/query", None),
69
+ ("query", {"POST"}, r"/api/v1/external/explorer/run", None),
70
  )
71
+ _HEAVY = tuple((cls, methods, re.compile(rx), pred) for cls, methods, rx, pred in HEAVY)
72
+ #: The class of a heavy MCP tools/call (sheet reads and database queries).
73
+ MCP_CLASS = "query"
74
 
75
  # MCP tools/call names that read sheets or query databases.
76
  MCP_HEAVY_TOOLS = {"sheet", "table", "data"}
 
106
  return False
107
 
108
 
109
+ def request_class(method, path, query="", body=None):
110
+ """The heavy class of one request, or None when it is light."""
111
  method = (method or "").upper()
112
  if isinstance(query, bytes):
113
  query = query.decode("latin-1")
114
  q = parse_qs(query, keep_blank_values=True) if isinstance(query, str) else (query or {})
115
+ for cls, methods, rx, pred in _HEAVY:
116
  if method in methods and rx.fullmatch(path or "") and (pred is None or pred(q)):
117
+ return cls
118
  if method == "POST" and path == MCP_PATH and _mcp_heavy(body):
119
+ return MCP_CLASS
120
+ return None
121
+
122
+
123
+ def classify(method, path, query="", body=None):
124
+ """Return "heavy" or "light" for one request. `query` is a raw query string or a parsed dict."""
125
+ return "heavy" if request_class(method, path, query, body) else "light"
126
+
127
+
128
+ def request_tenant(scope):
129
+ """The fairness key of one request: the signed session's tenant, else a hash of its API key.
130
+
131
+ No store read: the cookie is verified with its HMAC only. An unsigned request shares one key.
132
+ """
133
+ cookie, auth = "", ""
134
+ for key, value in scope.get("headers") or ():
135
+ if key == b"cookie":
136
+ cookie = value.decode("latin-1")
137
+ elif key == b"authorization":
138
+ auth = value.decode("latin-1")
139
+ if cookie:
140
+ try:
141
+ import aios_session
142
+ for part in cookie.split(";"):
143
+ name, _, value = part.strip().partition("=")
144
+ if name == aios_session.COOKIE_NAME and value:
145
+ claims = aios_session.read(value)
146
+ if claims:
147
+ return "tenant:" + str(claims["t"]).strip().lower()
148
+ except Exception: # noqa: BLE001
149
+ pass
150
+ if auth:
151
+ return "key:" + hashlib.sha256(auth.encode("utf-8", "replace")).hexdigest()[:16]
152
+ return "anonymous"
153
 
154
 
155
  class _Counters:
 
170
 
171
 
172
  _COUNTERS = _Counters()
173
+ #: The same counters for each request class.
174
+ _CLASS_COUNTERS = {name: _Counters() for name in CLASSES}
175
 
176
 
177
  def status():
178
  """Snapshot of heavy request admission for a settings or health surface."""
179
  c = _COUNTERS
180
+ workers = web_workers()
181
+ classes = {}
182
+ for name, cc in _CLASS_COUNTERS.items():
183
+ with cc.lock:
184
+ limit, queue = class_limit(name)
185
+ classes[name] = {"limit": cc.limit if cc.limit is not None else limit,
186
+ "queueSeconds": cc.queue_seconds if cc.queue_seconds is not None else queue,
187
+ "inFlight": cc.in_flight, "queued": cc.queued,
188
+ "busyRefusals": cc.busy_refusals, "admitted": cc.admitted}
189
  with c.lock:
190
+ limit = c.limit if c.limit is not None else _env_int("AIOS_HEAVY_CONCURRENCY", 2)
191
+ # Counters are per process: N uvicorn workers admit N times `limit` across the box.
192
+ return {"limit": limit, "workers": workers, "boxLimit": limit * workers,
193
  "queueSeconds": (c.queue_seconds if c.queue_seconds is not None
194
  else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)),
195
  "inFlight": c.in_flight, "queued": c.queued,
196
+ "busyRefusals": c.busy_refusals, "admitted": c.admitted,
197
+ "tenantCap": tenant_cap_env() or None, "classes": classes}
198
+
199
+
200
+ def web_workers(env=None):
201
+ """How many uvicorn workers share this box, from AIOS_UVICORN_WORKERS. 1 when unset."""
202
+ env = os.environ if env is None else env
203
+ try:
204
+ return max(1, int(env.get("AIOS_UVICORN_WORKERS") or 1))
205
+ except ValueError:
206
+ return 1
207
 
208
 
209
  def _env_int(name, default):
 
220
  return default
221
 
222
 
223
+ def tenant_cap_env():
224
+ """AIOS_HEAVY_TENANT_CAP: most permits of one class one tenant holds. 0 or unset: the class limit."""
225
+ try:
226
+ return max(0, int(os.environ.get("AIOS_HEAVY_TENANT_CAP") or 0))
227
+ except ValueError:
228
+ return 0
229
+
230
+
231
+ def class_limit(name):
232
+ """(limit, queue seconds) of one request class from its env, else from the shared heavy env.
233
+
234
+ Unset, grid_rows takes the shared heavy limit and every other class half of it (at least 1),
235
+ so five independent classes do not multiply the heavy work one process admits.
236
+ """
237
+ shared = _env_int("AIOS_HEAVY_CONCURRENCY", 2)
238
+ limit = shared if name == DEFAULT_CLASS else max(1, shared // 2)
239
+ queue = _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0)
240
+ return (_env_int(f"AIOS_HEAVY_CONCURRENCY_{name.upper()}", limit),
241
+ _env_float(f"AIOS_HEAVY_QUEUE_SECONDS_{name.upper()}", queue))
242
+
243
+
244
+ async def _send_busy(send, cause="heavy_queue_timeout", request_class=None):
245
+ error = {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE}
246
+ if request_class:
247
+ error["requestClass"] = request_class
248
+ body = json.dumps({"error": error}).encode()
249
  await send({"type": "http.response.start", "status": 503,
250
  "headers": [(b"content-type", b"application/json"),
251
  (b"content-length", str(len(body)).encode()),
 
254
  await send({"type": "http.response.body", "body": body, "more_body": False})
255
 
256
 
257
+ class _FairSlots:
258
+ """One class's permits in one event loop. A free permit goes to the waiting tenant served
259
+ least recently, so tenants take turns. A tenant holds at most `tenant_cap` permits.
260
+
261
+ FIFO would let one tenant's burst hold every other tenant's read past the queue bound.
262
+ """
263
+
264
+ TURNS_KEPT = 1024
265
+
266
+ def __init__(self, limit, tenant_cap=None):
267
+ self.limit = max(1, int(limit))
268
+ self.tenant_cap = max(1, min(self.limit, int(tenant_cap or self.limit)))
269
+ self.running = 0
270
+ self.by_tenant = {}
271
+ self.waiting = {} # tenant -> deque of futures, oldest first
272
+ self.turn = {} # tenant -> the grant count at its last grant
273
+ self.grants = 0
274
+
275
+ @property
276
+ def _value(self):
277
+ """Free permits, named as asyncio.Semaphore names them."""
278
+ return self.limit - self.running
279
+
280
+ def _dispatch(self):
281
+ while self.running < self.limit:
282
+ best = None
283
+ for tenant in list(self.waiting):
284
+ queue = self.waiting[tenant]
285
+ while queue and queue[0].done(): # a waiter that left (timeout, disconnect)
286
+ queue.popleft()
287
+ if not queue:
288
+ del self.waiting[tenant]
289
+ continue
290
+ if self.by_tenant.get(tenant, 0) >= self.tenant_cap:
291
+ continue
292
+ if best is None or self.turn.get(tenant, -1) < self.turn.get(best, -1):
293
+ best = tenant
294
+ if best is None:
295
+ return
296
+ queue = self.waiting[best]
297
+ self.running += 1
298
+ self.by_tenant[best] = self.by_tenant.get(best, 0) + 1
299
+ self.grants += 1
300
+ self.turn[best] = self.grants
301
+ queue.popleft().set_result(True)
302
+ if not queue:
303
+ del self.waiting[best]
304
+ if len(self.turn) > self.TURNS_KEPT:
305
+ for tenant in [t for t in self.turn if t not in self.by_tenant and t not in self.waiting]:
306
+ del self.turn[tenant]
307
+
308
+ async def acquire(self, tenant):
309
+ fut = asyncio.get_running_loop().create_future()
310
+ self.waiting.setdefault(tenant, deque()).append(fut)
311
+ self._dispatch()
312
+ try:
313
+ await fut
314
+ except asyncio.CancelledError:
315
+ if fut.done() and not fut.cancelled():
316
+ self.release(tenant) # granted in the same tick as the cancel
317
+ else:
318
+ fut.cancel()
319
+ raise
320
+ return True
321
+
322
+ def release(self, tenant):
323
+ self.running -= 1
324
+ held = self.by_tenant.get(tenant, 0) - 1
325
+ if held > 0:
326
+ self.by_tenant[tenant] = held
327
+ else:
328
+ self.by_tenant.pop(tenant, None)
329
+ self._dispatch()
330
+
331
+
332
+ class _Permit:
333
+ """One tenant's handle on a class's permits: acquire() and release() as a Semaphore has them."""
334
+
335
+ def __init__(self, slots, tenant):
336
+ self.slots, self.tenant = slots, tenant
337
+
338
+ async def acquire(self):
339
+ return await self.slots.acquire(self.tenant)
340
+
341
+ def release(self):
342
+ self.slots.release(self.tenant)
343
+
344
+ @property
345
+ def _value(self):
346
+ return self.slots._value
347
+
348
+
349
  class HeavyGate:
350
+ """Pure ASGI middleware. Heavy requests take a permit of their class; light ones pass through.
351
 
352
+ `concurrency` and `queue_seconds` set every class at once (tests). Unset, each class reads
353
+ its own env, then the shared heavy env. `classes` overrides one class: {name: {limit, queueSeconds}}.
354
+ """
355
+
356
+ def __init__(self, app, concurrency=None, queue_seconds=None, counters=None, classes=None,
357
+ tenant_cap=None):
358
  self.app = app
359
  self.concurrency = concurrency or _env_int("AIOS_HEAVY_CONCURRENCY", 2)
360
  self.queue_seconds = (queue_seconds if queue_seconds is not None
361
  else _env_float("AIOS_HEAVY_QUEUE_SECONDS", 90.0))
362
  self.counters = counters or _COUNTERS
363
  self.counters.limit, self.counters.queue_seconds = self.concurrency, self.queue_seconds
364
+ # A gate given its own counters (a test) keeps its own class counters too.
365
+ self.class_counters = ({name: _Counters() for name in CLASSES} if counters is not None
366
+ else _CLASS_COUNTERS)
367
+ self.tenant_cap = tenant_cap if tenant_cap is not None else tenant_cap_env()
368
+ self.limits = {}
369
+ for name in CLASSES:
370
+ limit, queue = class_limit(name)
371
+ if concurrency:
372
+ limit = concurrency
373
+ if queue_seconds is not None:
374
+ queue = queue_seconds
375
+ override = (classes or {}).get(name) or {}
376
+ limit = max(1, int(override.get("limit") or limit))
377
+ queue = float(override["queueSeconds"]) if override.get("queueSeconds") is not None else queue
378
+ self.limits[name] = (limit, queue)
379
+ cc = self.class_counters[name]
380
+ cc.limit, cc.queue_seconds = limit, queue
381
+ self._slots = {}
382
  self._loop = None
383
 
384
+ def _semaphore(self, cls=DEFAULT_CLASS, tenant="anonymous"):
385
+ # Futures bind to the loop they wait on. One set of permits per loop.
386
  loop = asyncio.get_running_loop()
387
+ if self._loop is not loop:
388
+ self._slots, self._loop = {}, loop
389
+ slots = self._slots.get(cls)
390
+ if slots is None:
391
+ limit = self.limits.get(cls, (self.concurrency, self.queue_seconds))[0]
392
+ slots = self._slots[cls] = _FairSlots(limit, self.tenant_cap or None)
393
+ return _Permit(slots, tenant)
394
 
395
  async def __call__(self, scope, receive, send):
396
  if scope.get("type") != "http":
 
404
  return
405
  if classify(method, path, scope.get("query_string", b""), peek) != "heavy":
406
  return await self.app(scope, _replay(body_msgs, receive), send)
407
+ cls = request_class(method, path, scope.get("query_string", b""), peek) or DEFAULT_CLASS
408
  if body_msgs is None:
409
  body_msgs, _ = await _read_body(receive, None)
410
  if body_msgs is None:
411
  return
412
+ await self._run_heavy(scope, body_msgs, receive, send, cls=cls)
413
 
414
+ async def _run_heavy(self, scope, body_msgs, receive, send, cls=None, tenant=None):
415
  queued_at = time.perf_counter()
416
+ if cls is None:
417
+ cls = request_class(scope.get("method", ""), scope.get("path", ""),
418
+ scope.get("query_string", b"")) or DEFAULT_CLASS
419
+ tenant = tenant if tenant is not None else request_tenant(scope)
420
+ queue_seconds = self.limits.get(cls, (self.concurrency, self.queue_seconds))[1]
421
+ sem = self._semaphore(cls, tenant)
422
+ cc = self.class_counters.get(cls) or _Counters()
423
+
424
+ def add(name, n):
425
+ self.counters.add(name, n)
426
+ cc.add(name, n)
427
+
428
+ add("queued", 1)
429
  timing = sys.modules.get("core.request_timing")
430
  budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError)
431
  acquire = asyncio.ensure_future(sem.acquire())
432
  gone = asyncio.ensure_future(_wait_disconnect(receive))
433
  try:
434
+ timeout = timing.remaining_budget(queue_seconds) if timing else queue_seconds
435
  done, _ = await asyncio.wait({acquire, gone}, timeout=timeout,
436
  return_when=asyncio.FIRST_COMPLETED)
437
  if timing:
 
439
  except budget_error:
440
  if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
441
  sem.release()
442
+ add("busy_refusals", 1)
443
+ await _send_busy(send, "request_deadline", cls)
444
  return
445
  except BaseException:
446
  # Cancellation can race a successful acquire before the handler owns the permit.
 
452
  timing = sys.modules.get("core.request_timing")
453
  if timing is not None:
454
  timing.record_queue((time.perf_counter() - queued_at) * 1000)
455
+ add("queued", -1)
456
  if not acquire.done():
457
  acquire.cancel()
458
  if not gone.done():
 
466
  except budget_error:
467
  if got:
468
  sem.release()
469
+ add("busy_refusals", 1)
470
+ await _send_busy(send, "request_deadline", cls)
471
  return
472
  if acquire not in done or gone in done:
473
  if got:
474
  sem.release()
475
  if gone not in done:
476
+ add("busy_refusals", 1)
477
+ await _send_busy(send, f"{cls}_queue_timeout", cls)
478
  return
479
  released = False
480
 
 
482
  nonlocal released
483
  if not released:
484
  released = True
485
+ add("in_flight", -1)
486
  sem.release()
487
 
488
  async def send_wrapped(message):
 
490
  if message.get("type") == "http.response.body" and not message.get("more_body"):
491
  release()
492
 
493
+ add("in_flight", 1)
494
+ add("admitted", 1)
495
  try:
496
  await self.app(scope, _replay(body_msgs, receive), send_wrapped)
497
  finally:
 
533
  return pending.pop(0)
534
  return await receive()
535
  return replay
536
+
537
+
538
+ # ----------------------------------------------------------------------------- box sizing (R25)
539
+ #: The 2 vCPU tier: today's values, exactly. Staging (cpu-basic, 2 vCPU, 16 GB) runs these.
540
+ TODAY = {"workers": 1, "heavyConcurrency": 2, "webDuckdbMemory": "6GB", "webDuckdbThreads": 2,
541
+ "workerDuckdbMemory": "4GB", "workerDuckdbThreads": 1}
542
+ #: Launch env name for each sizing value. An explicit env value always wins over the derived one.
543
+ SIZING_ENV = {"workers": "AIOS_UVICORN_WORKERS", "heavyConcurrency": "AIOS_HEAVY_CONCURRENCY",
544
+ "webDuckdbMemory": "AIOS_WEB_DUCKDB_MEMORY", "webDuckdbThreads": "AIOS_WEB_DUCKDB_THREADS",
545
+ "workerDuckdbMemory": "AIOS_WORKER_DUCKDB_MEMORY",
546
+ "workerDuckdbThreads": "AIOS_WORKER_DUCKDB_THREADS"}
547
+ VERSIONS_ENV = "AIOS_MIRROR_VERSIONS"
548
+ MAX_WEB_WORKERS = 4
549
+ #: Share of box memory all DuckDB instances may take together (10 of 16 GB today).
550
+ DUCKDB_MEMORY_SHARE = 0.625
551
+
552
+
553
+ def _read(path):
554
+ try:
555
+ with open(path, encoding="ascii") as handle:
556
+ return handle.read().strip()
557
+ except (OSError, UnicodeError):
558
+ return None
559
+
560
+
561
+ def _cgroup_cpus(root):
562
+ raw = _read(os.path.join(root, "cpu.max")) # cgroup v2
563
+ if raw:
564
+ parts = raw.split()
565
+ if len(parts) == 2 and parts[0] != "max":
566
+ try:
567
+ return max(1, int(int(parts[0]) / int(parts[1]))), "cgroup v2 cpu.max"
568
+ except (ValueError, ZeroDivisionError):
569
+ pass
570
+ for sub in ("cpu", "cpu,cpuacct", "cpuacct,cpu"): # cgroup v1
571
+ quota = _read(os.path.join(root, sub, "cpu.cfs_quota_us"))
572
+ period = _read(os.path.join(root, sub, "cpu.cfs_period_us"))
573
+ try:
574
+ if quota and period and int(quota) > 0:
575
+ return max(1, int(int(quota) / int(period))), "cgroup v1 cpu.cfs_quota_us"
576
+ except (ValueError, ZeroDivisionError):
577
+ pass
578
+ return None, None
579
+
580
+
581
+ def _cgroup_memory(root):
582
+ raw = _read(os.path.join(root, "memory.max")) # cgroup v2
583
+ if raw and raw != "max" and raw.isdigit():
584
+ return int(raw), "cgroup v2 memory.max"
585
+ raw = _read(os.path.join(root, "memory", "memory.limit_in_bytes")) # cgroup v1
586
+ # v1 reports "no limit" as a huge page-aligned number.
587
+ if raw and raw.isdigit() and int(raw) < 1 << 60:
588
+ return int(raw), "cgroup v1 memory.limit_in_bytes"
589
+ return None, None
590
+
591
+
592
+ def _meminfo_bytes(path):
593
+ raw = _read(path) or ""
594
+ for line in raw.splitlines():
595
+ if line.startswith("MemTotal:"):
596
+ try:
597
+ return int(line.split()[1]) * 1024
598
+ except (IndexError, ValueError):
599
+ return None
600
+ return None
601
+
602
+
603
+ def detect_box(cgroup_root="/sys/fs/cgroup", meminfo="/proc/meminfo", affinity=None):
604
+ """{cpus, memoryBytes, cpuSource, memorySource} for this container.
605
+
606
+ `os.cpu_count()` reports the HOST inside a container, so the cgroup limit is read first.
607
+ """
608
+ if affinity is None:
609
+ try:
610
+ affinity = len(os.sched_getaffinity(0))
611
+ except (AttributeError, OSError):
612
+ affinity = os.cpu_count() or 1
613
+ cpus, cpu_source = _cgroup_cpus(cgroup_root)
614
+ if cpus is None or cpus > affinity:
615
+ cpus, cpu_source = affinity, "sched_getaffinity"
616
+ physical = _meminfo_bytes(meminfo)
617
+ memory, memory_source = _cgroup_memory(cgroup_root)
618
+ if memory is None or (physical and memory > physical):
619
+ memory, memory_source = physical, "/proc/meminfo"
620
+ return {"cpus": int(cpus), "memoryBytes": int(memory or 0), "cpuSource": cpu_source,
621
+ "memorySource": memory_source}
622
+
623
+
624
+ def sizing(cpus, memory_bytes, workers=None):
625
+ """Launch values for a box. Under 4 CPUs with no forced worker count: TODAY exactly.
626
+
627
+ `workers` forces the uvicorn worker count, and the per-worker values follow it.
628
+ """
629
+ cpus = max(1, int(cpus))
630
+ memory_gb = max(0.0, float(memory_bytes or 0) / 1e9)
631
+ if workers is None and cpus < 4:
632
+ return {"tier": "2 vCPU", **TODAY}
633
+ if workers is None:
634
+ workers = max(1, min(MAX_WEB_WORKERS, cpus // 2, int(memory_gb // 8)))
635
+ workers = max(1, int(workers))
636
+ worker_memory = max(4, int(memory_gb // 8))
637
+ duckdb_total = int(memory_gb * DUCKDB_MEMORY_SHARE)
638
+ return {"tier": "derived", "workers": workers,
639
+ # One heavy request per CPU across the box, split over the workers.
640
+ "heavyConcurrency": max(1, cpus // workers),
641
+ "webDuckdbMemory": f"{max(2, (duckdb_total - worker_memory) // workers)}GB",
642
+ "webDuckdbThreads": max(2, cpus // workers),
643
+ "workerDuckdbMemory": f"{worker_memory}GB",
644
+ "workerDuckdbThreads": max(1, cpus // 4)}
645
+
646
+
647
+ def launch_role(environ=None):
648
+ """The role the app will resolve, as `process_role.resolve_role` does: a valid
649
+ AIOS_PROCESS_ROLE, else web on a Space (SPACE_ID set), else all."""
650
+ environ = os.environ if environ is None else environ
651
+ raw = str(environ.get("AIOS_PROCESS_ROLE") or "").strip().lower()
652
+ if raw in ("web", "worker", "all", "reader"):
653
+ return raw
654
+ return "web" if str(environ.get("SPACE_ID") or "").strip() else "all"
655
+
656
+
657
+ def launch_env(environ=None, box=None):
658
+ """{env name: value} the launcher exports. Explicit env wins, except it never buys workers
659
+ without the read-only mirror: AIOS_MIRROR_VERSIONS=0 or a role other than web holds one worker.
660
+ """
661
+ environ = os.environ if environ is None else environ
662
+ box = box or detect_box()
663
+ explicit = {key for key, name in SIZING_ENV.items() if str(environ.get(name) or "").strip()}
664
+ forced = web_workers(environ) if "workers" in explicit else None
665
+ role = launch_role(environ)
666
+ if str(environ.get(VERSIONS_ENV) or "").strip() == "0" or role != "web":
667
+ # Only the web role versions its mirror writes. Every other role restores in place.
668
+ forced = 1
669
+ values = sizing(box["cpus"], box["memoryBytes"], workers=forced)
670
+ out = {SIZING_ENV[key]: str(values[key]) for key in SIZING_ENV
671
+ if key not in explicit and key != "workers"}
672
+ workers = int(values["workers"])
673
+ out["AIOS_UVICORN_WORKERS"] = str(workers)
674
+ # Several workers each open the mirror read-only through its version pointer.
675
+ out[VERSIONS_ENV] = "1" if workers > 1 else "0"
676
+ return out
677
+
678
+
679
+ def web_app():
680
+ """`uvicorn --factory` target for several workers. The supervisor imports only this module.
681
+
682
+ uvicorn imports a plain `api.main:app` in its supervisor process too, which would run the
683
+ app's boot threads in a process that never serves.
684
+ """
685
+ import importlib
686
+ return importlib.import_module("api.main").app
687
+
688
+
689
+ def main(argv=None):
690
+ import argparse
691
+ import shlex
692
+ parser = argparse.ArgumentParser(description="Print launch values sized to this box.")
693
+ parser.add_argument("--env", action="store_true", help="print shell export lines")
694
+ # The env defaults let a gate point the launcher at a fixture box.
695
+ parser.add_argument("--cgroup-root", default=os.environ.get("AIOS_CGROUP_ROOT") or "/sys/fs/cgroup")
696
+ parser.add_argument("--meminfo", default=os.environ.get("AIOS_MEMINFO") or "/proc/meminfo")
697
+ args = parser.parse_args(argv)
698
+ box = detect_box(args.cgroup_root, args.meminfo)
699
+ env = launch_env(os.environ, box)
700
+ print(f"[box-sizing] cpus={box['cpus']} ({box['cpuSource']}) "
701
+ f"memory={box['memoryBytes'] / 1e9:.1f}GB ({box['memorySource']}) "
702
+ + " ".join(f"{k}={v}" for k, v in env.items()), file=sys.stderr, flush=True)
703
+ if args.env:
704
+ for name, value in env.items():
705
+ print(f"export {name}={shlex.quote(value)}")
706
+ else:
707
+ print(json.dumps({"box": box, "env": env}, sort_keys=True))
708
+ return 0
709
+
710
+
711
+ if __name__ == "__main__":
712
+ sys.exit(main())
api/request_timing.py CHANGED
@@ -29,6 +29,7 @@ def request_timeout_s():
29
 
30
 
31
  _BUSY_CAUSES = {
 
32
  "PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full",
33
  "QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout",
34
  "SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout",
@@ -69,25 +70,46 @@ class DeadlineCapacityExceeded(RuntimeError):
69
 
70
 
71
  class _StatementAlarm:
 
 
 
72
  def __init__(self, owner, connection, acc):
73
  self.owner, self.connection, self.acc = owner, connection, acc
74
  self.lock = threading.Lock()
75
  self.active, self.expired = True, False
 
76
 
77
  def fire(self):
78
  with self.lock:
79
- if self.active and (self.acc.cancelled or time.monotonic() >= self.acc.deadline):
80
- self.expired = True
81
- self.active = False
 
 
 
 
 
 
 
 
 
82
  self.connection.interrupt()
 
 
 
83
 
84
  def close(self):
85
  # Serialize disarm with interrupt before the cursor can execute another statement.
86
  with self.lock:
87
  self.active = False
88
- expired = self.expired
89
  with self.owner.lock:
90
  self.owner.active.discard(self)
 
 
 
 
 
91
  return expired
92
 
93
 
 
29
 
30
 
31
  _BUSY_CAUSES = {
32
+ "OdooBreakerOpen": "odoo_breaker_open",
33
  "PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full",
34
  "QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout",
35
  "SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout",
 
70
 
71
 
72
  class _StatementAlarm:
73
+ #: Repeat interrupt() this often until disarm: DuckDB can lose a single one on a parameterized execute.
74
+ REFIRE = .05
75
+
76
  def __init__(self, owner, connection, acc):
77
  self.owner, self.connection, self.acc = owner, connection, acc
78
  self.lock = threading.Lock()
79
  self.active, self.expired = True, False
80
+ self.fired_at, self.last, self.sent = None, 0.0, 0
81
 
82
  def fire(self):
83
  with self.lock:
84
+ if not self.active:
85
+ return
86
+ now = time.monotonic()
87
+ if not self.expired:
88
+ if not (self.acc.cancelled or now >= self.acc.deadline):
89
+ return
90
+ self.expired, self.fired_at = True, now
91
+ elif now - self.last < self.REFIRE:
92
+ return
93
+ self.last = now
94
+ self.sent += 1
95
+ try:
96
  self.connection.interrupt()
97
+ except Exception:
98
+ self.active = False
99
+ raise
100
 
101
  def close(self):
102
  # Serialize disarm with interrupt before the cursor can execute another statement.
103
  with self.lock:
104
  self.active = False
105
+ expired, fired_at, sent = self.expired, self.fired_at, self.sent
106
  with self.owner.lock:
107
  self.owner.active.discard(self)
108
+ if expired and fired_at is not None:
109
+ now = time.monotonic()
110
+ LOG.warning("[deadline-interrupt] statement stopped %.0f ms after the first interrupt "
111
+ "(%d sent), %.0f ms past the request deadline", (now - fired_at) * 1000, sent,
112
+ (now - self.acc.deadline) * 1000)
113
  return expired
114
 
115
 
api/routes_account_health.py ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Account health App routes: summary, metric history, owned alerts and settings, each App scoped (C3)."""
2
+ from fastapi import APIRouter, Body, Depends, Query
3
+
4
+ from deps import Session, err, require_session
5
+
6
+ TEMPLATE_KIND = "account_health"
7
+ router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"])
8
+ SOURCE = "ut_amzn_account_health"
9
+
10
+
11
+ def _module():
12
+ from modules import account_health
13
+ return account_health
14
+
15
+
16
+ def _reader(session: Session = Depends(require_session)) -> Session:
17
+ """Reads need read access to the account health database."""
18
+ from core import perm_scope
19
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
20
+ raise err(403, "account_health_forbidden", "Your account cannot read Amazon account health.")
21
+ return session
22
+
23
+
24
+ def _writer(session: Session = Depends(_reader)) -> Session:
25
+ """Writes also need a role that may edit."""
26
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
27
+ raise err(403, "account_health_read_only", "Your account can view account health but not change it.")
28
+ return session
29
+
30
+
31
+ def _tenant(session):
32
+ return str(getattr(session.runtime, "key", "") or session.tenant)
33
+
34
+
35
+ def _call(fn, *args, **kwargs):
36
+ try:
37
+ return fn(*args, **kwargs)
38
+ except PermissionError as exc:
39
+ raise err(403, "account_health_read_only", str(exc)) from exc
40
+ except ValueError as exc:
41
+ raise err(400, "account_health_invalid", str(exc)) from exc
42
+ except KeyError as exc:
43
+ raise err(404, "account_health_not_found", "That alert or metric does not exist.") from exc
44
+ except Exception as exc: # noqa: BLE001
45
+ raise err(503, "account_health_unavailable", "Account health could not be read. Try again shortly.") from exc
46
+
47
+
48
+ def _app(session, app_id):
49
+ """The Account health App record this request names, readable by the caller (404 otherwise)."""
50
+ import application_registry
51
+ try:
52
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
53
+ except Exception as exc: # noqa: BLE001
54
+ raise err(503, "account_health_unavailable", "The App could not be read. Try again shortly.") from exc
55
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
56
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
57
+ raise err(404, "account_health_app_not_found", "That Account health App does not exist.")
58
+ return app
59
+
60
+
61
+ def _may_edit(session, app):
62
+ import application_registry
63
+ role = str(session.user.get("role") or "user").strip().lower()
64
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
65
+
66
+
67
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
68
+ """A read names the App, and the caller must be able to read that App."""
69
+ _app(session, app)
70
+ return session
71
+
72
+
73
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
74
+ """A write names the App, and only an editor of that App may change it."""
75
+ if not _may_edit(session, _app(session, app)):
76
+ raise err(403, "account_health_read_only", "You can view this App but not change it.")
77
+ return session
78
+
79
+
80
+ def owners(session):
81
+ """People who may own an alert: active, non-service, non-viewer accounts of this tenant, by display name."""
82
+ ah = _module()
83
+ try:
84
+ users = session.runtime.get("users")
85
+ except Exception: # noqa: BLE001
86
+ users = None
87
+ want = _tenant(session).strip().lower()
88
+ names = set()
89
+ for name, rec in (users.items() if isinstance(users, dict) else ()):
90
+ if (not isinstance(rec, dict) or not rec.get("active", True) or ah._is_service(name, rec)
91
+ or str(rec.get("role") or "").strip().lower() == "viewer"
92
+ or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
93
+ continue
94
+ names.add(str(rec.get("name") or name).strip())
95
+ return sorted((n for n in names if n), key=str.lower)
96
+
97
+
98
+ @router.get("/summary")
99
+ def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
100
+ ah = _module()
101
+ out = _call(ah.summary, _tenant(session), st=session.runtime)
102
+ return {**out, "canEdit": _may_edit(session, _app(session, app))}
103
+
104
+
105
+ @router.get("/history")
106
+ def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
107
+ limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
108
+ ah = _module()
109
+ return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
110
+
111
+
112
+ @router.get("/items")
113
+ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
114
+ cursor: str | None = Query(None), limit: int = Query(200, ge=1),
115
+ session: Session = Depends(_app_reader)):
116
+ ah = _module()
117
+ listed = _call(ah.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
118
+ st=session.runtime)
119
+ return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
120
+
121
+
122
+ @router.patch("/items/{item_id}")
123
+ def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
124
+ due: str | None = Body(None), note: str | None = Body(None),
125
+ session: Session = Depends(_app_writer)):
126
+ ah = _module()
127
+ return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
128
+ due=due, note=note, st=session.runtime)
129
+
130
+
131
+ @router.get("/settings")
132
+ def read_settings(session: Session = Depends(_app_reader)):
133
+ ah = _module()
134
+ return _call(ah.settings, _tenant(session), st=session.runtime)
135
+
136
+
137
+ @router.put("/settings")
138
+ def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None),
139
+ session: Session = Depends(_app_writer)):
140
+ ah = _module()
141
+ return _call(ah.set_settings, _tenant(session), alert_distance_pct=alertDistancePct,
142
+ default_owner=defaultOwner, st=session.runtime)
api/routes_admin.py CHANGED
@@ -143,7 +143,17 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
143
  # editor will open, so the two cannot disagree.
144
  "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
145
  surface_keys=surface_keys),
146
- "perms_v": int(rec.get("perms_v") or 0)}
 
 
 
 
 
 
 
 
 
 
147
 
148
 
149
  def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
@@ -454,6 +464,9 @@ def update_user(username: str, body: dict = Body(default=None),
454
  if result == "last_admin":
455
  raise err(400, "last_admin",
456
  "an active tenant must retain at least one administrator")
 
 
 
457
  if result != "updated":
458
  raise err(404, "no_such_user", f"no account named {uname!r}")
459
 
@@ -539,6 +552,166 @@ def expire_password(username: str, session: Session = Depends(admin_gate)):
539
  return Response(status_code=204)
540
 
541
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
542
  # ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
543
  #: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
544
  #: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
 
143
  # editor will open, so the two cannot disagree.
144
  "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
145
  surface_keys=surface_keys),
146
+ "perms_v": int(rec.get("perms_v") or 0),
147
+ # S79-T63: only a copied test account carries `test`, so a real account's keys stay exact.
148
+ **({"test": _test_marker(rec)} if _test_marker(rec) else {})}
149
+
150
+
151
+ def _test_marker(rec):
152
+ mark = (rec or {}).get(users.TEST_ACCOUNT_KEY)
153
+ if not isinstance(mark, dict):
154
+ return None
155
+ return {"copiedFrom": mark.get("copied_from"), "createdBy": mark.get("created_by"),
156
+ "createdAt": mark.get("created_at")}
157
 
158
 
159
  def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
 
464
  if result == "last_admin":
465
  raise err(400, "last_admin",
466
  "an active tenant must retain at least one administrator")
467
+ if result in _TEST_ACCOUNT_REFUSALS:
468
+ # R42: a test account gets no email and no admin role, refused in the core writer.
469
+ raise err(400, result, _TEST_ACCOUNT_REFUSALS[result])
470
  if result != "updated":
471
  raise err(404, "no_such_user", f"no account named {uname!r}")
472
 
 
552
  return Response(status_code=204)
553
 
554
 
555
+ # ── S79-T63 (R28, R42): test accounts that copy one member's permissions exactly ─────────────────
556
+ _TEST_ACCOUNT_REFUSALS = {
557
+ "test_account_no_email": "A test account cannot have an email address.",
558
+ "test_account_no_whatsapp": "A test account cannot connect WhatsApp.",
559
+ "test_account_not_admin": "A test account cannot be an administrator.",
560
+ }
561
+
562
+
563
+ def _copy_refusal(exc):
564
+ return err(exc.status, exc.code, str(exc))
565
+
566
+
567
+ #: The en and em dash, built from code points so this file carries no dash character (rule 2).
568
+ _DASHES = re.compile("[" + chr(0x2013) + chr(0x2014) + "]")
569
+
570
+
571
+ def _copy_failure(exc, verb):
572
+ # A store error can carry any text; rule 2 keeps dashes off the screen.
573
+ cause = _DASHES.sub(",", str(exc.cause or "")).strip().rstrip(".")
574
+ message = f"The test account was not {verb}. {cause}."
575
+ if exc.leftovers:
576
+ message += (" These parts could not be removed: " + ", ".join(exc.leftovers)
577
+ + ". Remove the test account again to finish.")
578
+ elif verb == "created":
579
+ message += " Nothing was kept."
580
+ return err(503, "test_account_failed", message)
581
+
582
+
583
+ def _test_account_steps(session):
584
+ """Copy steps for grant stores outside `core`: Connector grants, through their own route."""
585
+ import core.perm_scope as perm_scope
586
+ import routes_keychain as keychain
587
+
588
+ def held():
589
+ doc = session.runtime.get(perm_scope.CONNECTOR_GRANT_KEY, fresh=True)
590
+ if doc is not None and not isinstance(doc, dict):
591
+ raise RuntimeError("connector access could not be read")
592
+ return doc or {}
593
+
594
+ def copy_grants(username, source):
595
+ known = {kind["key"] for kind in keychain.connector_grant_kinds()}
596
+ kinds = sorted(k for k in (held().get(source) or ()) if k in known)
597
+ if kinds:
598
+ keychain.put_connector_grants(username, {"granted": kinds}, session)
599
+
600
+ def clear_grants(username):
601
+ if username in held():
602
+ keychain.put_connector_grants(username, {"granted": []}, session)
603
+
604
+ return [("connector access", copy_grants, clear_grants)]
605
+
606
+
607
+ @router.get("/admin/users/{username}/test-account/preview")
608
+ def preview_test_account(username: str, session: Session = Depends(admin_gate)):
609
+ """What a test account copied from this member would carry, before anything is written."""
610
+ from core import test_accounts, user_tables
611
+ reg = _registry()
612
+ try:
613
+ plan = test_accounts.plan(username, reg, tenant=_tenant_of(session.user),
614
+ st=session.runtime)
615
+ lent = user_tables.lend_defs(session.runtime, schema_only=True)
616
+ except test_accounts.CopyRefused as exc:
617
+ raise _copy_refusal(exc)
618
+ except Exception as exc: # noqa: BLE001
619
+ raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc
620
+ limits = _source_limits(plan["source"], reg.get(plan["source"]), session)
621
+ body = test_accounts.preview(plan, st=lent, limits=limits)
622
+ _label_grants(session, body["copy"]["grants"])
623
+ return body
624
+
625
+
626
+ def _source_limits(uname, rec, session):
627
+ """The source's row limits, phrased for the administrator; None if they cannot be read.
628
+
629
+ Field walls are lifted for the phrasing only, so a condition on a hidden field is named
630
+ instead of reading "Limited by your access". The filters themselves are the source's.
631
+ """
632
+ import core.perm_scope as perm_scope
633
+ try:
634
+ reader = deepcopy(users._public(uname, rec))
635
+ for entry in (reader.get("perms") or {}).values():
636
+ if isinstance(entry, dict):
637
+ entry.pop("hiddenFields", None)
638
+ return perm_scope.scope_limits(reader, st=session.runtime)
639
+ except Exception: # noqa: BLE001
640
+ return None
641
+
642
+
643
+ #: What a preview row says when a shared item's name cannot be read. Never a raw id.
644
+ _GRANT_FALLBACK = {"view": "A shared view", "folder": "A shared folder", "database": "A database",
645
+ "field": "A shared field", "cohort": "A locked view", "workbook": "A shared sheet"}
646
+ #: Naming stops after this many seconds; every grant is still listed, the rest by kind.
647
+ _LABEL_BUDGET_S = 3.0
648
+
649
+
650
+ def _label_grants(session, grants):
651
+ """Name each grant through the share door's own resolver (`routes_shares._object_ref`)."""
652
+ import time
653
+ try:
654
+ from routes_shares import _object_ref
655
+ except Exception: # noqa: BLE001
656
+ _object_ref = None
657
+ deadline = time.monotonic() + _LABEL_BUDGET_S
658
+ for grant in grants:
659
+ if grant.get("label"):
660
+ continue
661
+ label = None
662
+ if time.monotonic() < deadline:
663
+ try:
664
+ if grant["kind"] == "workbook":
665
+ # `_object_ref` has no workbook branch; the workbook's own name is the label.
666
+ import core.sheets as sheets
667
+ label = str(sheets.get_workbook(session.runtime, grant["oid"]).get("name") or "")
668
+ elif _object_ref is not None:
669
+ label = str(_object_ref(session, grant["kind"], grant["oid"])[0] or "")
670
+ except Exception: # noqa: BLE001
671
+ label = None
672
+ label = (label or "").strip()
673
+ grant["label"] = label or _GRANT_FALLBACK.get(grant["kind"], "A shared item")
674
+
675
+
676
+ @router.post("/admin/test-accounts", status_code=201)
677
+ def create_test_account(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
678
+ """Create a test account whose permissions copy `source` exactly. The password shows once."""
679
+ from core import test_accounts
680
+ body = body if isinstance(body, dict) else {}
681
+ _require_store()
682
+ try:
683
+ made = test_accounts.create(body.get("source"), admin=session.uname,
684
+ tenant=_tenant_of(session.user), st=session.runtime,
685
+ steps=_test_account_steps(session))
686
+ except test_accounts.CopyRefused as exc:
687
+ raise _copy_refusal(exc)
688
+ except test_accounts.CopyFailed as exc:
689
+ raise _copy_failure(exc, "created")
690
+ except Exception as exc: # noqa: BLE001
691
+ raise err(503, "store_unavailable", "The test account was not created. Try again.") from exc
692
+ rec = _registry().get(made["username"])
693
+ if not isinstance(rec, dict):
694
+ raise err(503, "store_unavailable", "The test account was not saved. Try again.")
695
+ return {"user": _view(made["username"], rec), "password": made["password"]}
696
+
697
+
698
+ @router.delete("/admin/test-accounts/{username}", status_code=204)
699
+ def delete_test_account(username: str, session: Session = Depends(admin_gate)):
700
+ """Remove a test account, its sign in and every grant naming it, in one call."""
701
+ from core import test_accounts
702
+ _require_store()
703
+ try:
704
+ test_accounts.remove(username, tenant=_tenant_of(session.user), st=session.runtime,
705
+ granter=session.uname, steps=_test_account_steps(session))
706
+ except test_accounts.CopyRefused as exc:
707
+ raise _copy_refusal(exc)
708
+ except test_accounts.CopyFailed as exc:
709
+ raise _copy_failure(exc, "fully removed")
710
+ except Exception as exc: # noqa: BLE001
711
+ raise err(503, "store_unavailable", "The test account was not removed. Try again.") from exc
712
+ return Response(status_code=204)
713
+
714
+
715
  # ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
716
  #: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
717
  #: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
api/routes_aged_stock.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Aged stock App routes (C3): summary, items, one SKU, settings, refresh and the rule 8 tie."""
2
+ from fastapi import APIRouter, Body, Depends, Query
3
+
4
+ from deps import Session, err, require_session
5
+
6
+ TEMPLATE_KIND = "aged_stock"
7
+ router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["aged-stock"])
8
+ SOURCE = "ut_amzn_storage_fees"
9
+
10
+
11
+ def _module():
12
+ from modules import aged_stock
13
+ return aged_stock
14
+
15
+
16
+ def _reader(session: Session = Depends(require_session)) -> Session:
17
+ """Reads need read access to the Amazon storage fees database."""
18
+ from core import perm_scope
19
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
20
+ raise err(403, "aged_stock_forbidden", "Your account cannot read Amazon storage fees.")
21
+ return session
22
+
23
+
24
+ def _writer(session: Session = Depends(_reader)) -> Session:
25
+ """Writes also need a role that may edit."""
26
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
27
+ raise err(403, "aged_stock_read_only", "You can view this App but not change it.")
28
+ return session
29
+
30
+
31
+ def _tenant(session):
32
+ return str(getattr(session.runtime, "key", "") or session.tenant)
33
+
34
+
35
+ def _call(fn, *args, **kwargs):
36
+ try:
37
+ return fn(*args, **kwargs)
38
+ except PermissionError as exc:
39
+ raise err(403, "aged_stock_read_only", "You can view this App but not change it.") from exc
40
+ except ValueError as exc:
41
+ raise err(400, "aged_stock_invalid", str(exc)) from exc
42
+ except KeyError as exc:
43
+ raise err(404, "aged_stock_item_not_found", "That aged stock item does not exist.") from exc
44
+ except Exception as exc: # noqa: BLE001
45
+ raise err(503, "aged_stock_unavailable", "The aged stock view could not be read. Try again shortly.") from exc
46
+
47
+
48
+ def _app(session, app_id):
49
+ """The Aged stock App record this request names, readable by the caller (404 otherwise)."""
50
+ import application_registry
51
+ try:
52
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
53
+ except Exception as exc: # noqa: BLE001
54
+ raise err(503, "aged_stock_unavailable", "The App could not be read. Try again shortly.") from exc
55
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
56
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
57
+ raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
58
+ return app
59
+
60
+
61
+ def _may_edit(session, app):
62
+ import application_registry
63
+ role = str(session.user.get("role") or "user").strip().lower()
64
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
65
+
66
+
67
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
68
+ """A read names the App, and the caller must be able to read that App."""
69
+ _app(session, app)
70
+ return session
71
+
72
+
73
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
74
+ """A write names the App, and only an editor of that App may change it."""
75
+ if not _may_edit(session, _app(session, app)):
76
+ raise err(403, "aged_stock_read_only", "You can view this App but not change it.")
77
+ return session
78
+
79
+
80
+ def owners(session):
81
+ """People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name."""
82
+ mod = _module()
83
+ try:
84
+ users = session.runtime.get("users")
85
+ except Exception: # noqa: BLE001
86
+ users = None
87
+ want = _tenant(session).strip().lower()
88
+ names = set()
89
+ for name, rec in (users.items() if isinstance(users, dict) else ()):
90
+ if (not isinstance(rec, dict) or not rec.get("active", True) or mod._is_service(name, rec)
91
+ or str(rec.get("role") or "").strip().lower() == "viewer"
92
+ or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
93
+ continue
94
+ names.add(str(rec.get("name") or name).strip())
95
+ return sorted((n for n in names if n), key=str.lower)
96
+
97
+
98
+ #: The item fields a Next tier row carries, so the screen lists it with no second read.
99
+ ROW_FIELDS = ("action", "recommended_action", "owner", "status", "due", "units")
100
+
101
+
102
+ def _summary_rows(tenant_key, st):
103
+ """The module summary with each Next tier row joined to its item's action, owner, status and due."""
104
+ mod = _module()
105
+ out = mod.summary(tenant_key, st=st)
106
+ stored = mod._doc(st, mod.KEY_ITEMS).get("items") or {}
107
+ for row in out.get("nextTier") or ():
108
+ item = stored.get(row.get("id")) or {}
109
+ row["item"] = {key: item.get(key) for key in ROW_FIELDS}
110
+ return out
111
+
112
+
113
+ @router.get("/summary")
114
+ def read_summary(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
115
+ out = _call(_summary_rows, _tenant(session), session.runtime)
116
+ return {**out, "canEdit": _may_edit(session, _app(session, app))}
117
+
118
+
119
+ @router.get("/items")
120
+ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
121
+ action: str | None = Query(None), cursor: str | None = Query(None),
122
+ limit: int = Query(200, ge=1), q: str | None = Query(None),
123
+ session: Session = Depends(_app_reader)):
124
+ listed = _call(_module().items, _tenant(session), session.uname, status=status, action=action,
125
+ cursor=cursor, limit=limit, q=q, st=session.runtime)
126
+ return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
127
+
128
+
129
+ @router.get("/items/{item_id}")
130
+ def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
131
+ detail = _call(_module().item_detail, _tenant(session), item_id, st=session.runtime)
132
+ return {**detail, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
133
+
134
+
135
+ @router.patch("/items/{item_id}")
136
+ def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
137
+ due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
138
+ session: Session = Depends(_app_writer)):
139
+ return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
140
+ action=action, note=note, user=session.uname or "unknown", st=session.runtime)
141
+
142
+
143
+ @router.get("/settings")
144
+ def read_settings(session: Session = Depends(_app_reader)):
145
+ return _call(_module().settings, _tenant(session), st=session.runtime)
146
+
147
+
148
+ @router.patch("/settings")
149
+ def write_settings(adsFromDays: int | None = Body(None), priceDownFromDays: int | None = Body(None),
150
+ removeFromDays: int | None = Body(None), feeHorizonDays: int | None = Body(None),
151
+ dueDays: int | None = Body(None), defaultOwner: str | None = Body(None),
152
+ session: Session = Depends(_app_writer)):
153
+ return _call(_module().set_settings, _tenant(session), ads_from_days=adsFromDays,
154
+ price_down_from_days=priceDownFromDays, remove_from_days=removeFromDays,
155
+ fee_horizon_days=feeHorizonDays, due_days=dueDays, default_owner=defaultOwner,
156
+ st=session.runtime)
157
+
158
+
159
+ @router.post("/refresh")
160
+ def refresh(session: Session = Depends(_app_writer)):
161
+ return _call(_module().refresh, _tenant(session), st=session.runtime)
162
+
163
+
164
+ @router.get("/validate")
165
+ def validate(start: str | None = Query(None), end: str | None = Query(None),
166
+ session: Session = Depends(_app_reader)):
167
+ return _call(_module().validate, _tenant(session), start=start, end=end, st=session.runtime)
api/routes_amazon_operator.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
2
+
3
+ Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`, the
4
+ same path an MCP approval takes. The router never calls the Ads client.
5
+ """
6
+ from fastapi import APIRouter, Body, Depends, Query
7
+
8
+ from deps import Session, err, require_session
9
+
10
+ router = APIRouter(prefix="/api/v1/app-kinds/amazon_operator", tags=["amazon-operator"])
11
+ TEMPLATE_KIND = "amazon_operator"
12
+
13
+
14
+ def _module():
15
+ from modules import amazon_operator
16
+ return amazon_operator
17
+
18
+
19
+ def _reader(session: Session = Depends(require_session)) -> Session:
20
+ """Reads need a signed in session. The App grant is checked per request."""
21
+ return session
22
+
23
+
24
+ def _writer(session: Session = Depends(_reader)) -> Session:
25
+ """Writes also need a role that may edit."""
26
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
27
+ raise err(403, "amazon_operator_read_only", "Your account can view these proposals but not change them.")
28
+ return session
29
+
30
+
31
+ def _tenant(session):
32
+ return str(getattr(session.runtime, "key", "") or session.tenant)
33
+
34
+
35
+ def _call(fn, *args, **kwargs):
36
+ try:
37
+ return fn(*args, **kwargs)
38
+ except PermissionError as exc:
39
+ raise err(403, "amazon_operator_read_only", str(exc)) from exc
40
+ except ValueError as exc:
41
+ raise err(400, "amazon_operator_invalid", str(exc)) from exc
42
+ except KeyError as exc:
43
+ raise err(404, "amazon_operator_not_found", "That proposal does not exist.") from exc
44
+ except Exception as exc: # noqa: BLE001
45
+ raise err(503, "amazon_operator_unavailable", "The Amazon operator could not be read. Try again shortly.") from exc
46
+
47
+
48
+ def _app(session, app_id):
49
+ """The Amazon operator App record this request names, readable by the caller (404 otherwise)."""
50
+ import application_registry
51
+ try:
52
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
53
+ except Exception as exc: # noqa: BLE001
54
+ raise err(503, "amazon_operator_unavailable", "The App could not be read. Try again shortly.") from exc
55
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
56
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
57
+ raise err(404, "amazon_operator_app_not_found", "That Amazon operator App does not exist.")
58
+ return app
59
+
60
+
61
+ def _may_edit(session, app):
62
+ import application_registry
63
+ role = str(session.user.get("role") or "user").strip().lower()
64
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
65
+
66
+
67
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
68
+ """C3: a read names the App, and the caller must be able to read that App."""
69
+ _app(session, app)
70
+ return session
71
+
72
+
73
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
74
+ """C3: a write names the App, and only an editor of that App may change it."""
75
+ if not _may_edit(session, _app(session, app)):
76
+ raise err(403, "amazon_operator_read_only", "You can view this App but not change it.")
77
+ return session
78
+
79
+
80
+ @router.get("/today")
81
+ def read_today(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
82
+ op = _module()
83
+ out = _call(op.today, _tenant(session), st=session.runtime)
84
+ return {**out, "canEdit": _may_edit(session, _app(session, app))}
85
+
86
+
87
+ @router.get("/history")
88
+ def read_history(status: str | None = Query(None), cursor: str | None = Query(None),
89
+ limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)):
90
+ op = _module()
91
+ page = _call(op.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
92
+ st=session.runtime)
93
+ if cursor:
94
+ return page
95
+ # The first page carries the full count for the History tab. Uncapped: every row is counted (rule 1).
96
+ every = _call(op.items, _tenant(session), session.user, status=status, cursor=None, limit=None,
97
+ st=session.runtime)
98
+ return {**page, "total": len(every["items"])}
99
+
100
+
101
+ @router.get("/settings")
102
+ def read_settings(session: Session = Depends(_app_reader)):
103
+ op = _module()
104
+ return _call(op.settings, _tenant(session), st=session.runtime)
105
+
106
+
107
+ @router.put("/settings")
108
+ def write_settings(stop_cover_days: float | None = Body(None), raise_cover_days: float | None = Body(None),
109
+ pause_below_profit: float | None = Body(None), bid_step_pct: float | None = Body(None),
110
+ budget_step_pct: float | None = Body(None), lookback_days: float | None = Body(None),
111
+ negative_min_clicks: float | None = Body(None), negative_min_cost: float | None = Body(None),
112
+ defaultOwner: str | None = Body(None), session: Session = Depends(_app_writer)):
113
+ op = _module()
114
+ return _call(op.set_settings, _tenant(session), st=session.runtime, default_owner=defaultOwner,
115
+ stop_cover_days=stop_cover_days, raise_cover_days=raise_cover_days,
116
+ pause_below_profit=pause_below_profit, bid_step_pct=bid_step_pct,
117
+ budget_step_pct=budget_step_pct, lookback_days=lookback_days,
118
+ negative_min_clicks=negative_min_clicks, negative_min_cost=negative_min_cost)
119
+
120
+
121
+ @router.post("/approve")
122
+ def approve(item_ids: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
123
+ """Approve the ticked proposals as one batch. Returns each row's outcome."""
124
+ op = _module()
125
+ return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime)
126
+
127
+
128
+ @router.post("/dismiss")
129
+ def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length=1),
130
+ session: Session = Depends(_app_writer)):
131
+ op = _module()
132
+ return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
133
+ st=session.runtime)
api/routes_assets.py CHANGED
@@ -36,7 +36,7 @@ from fastapi import Body, Depends
36
  from fastapi import APIRouter
37
  from fastapi.responses import FileResponse, Response
38
 
39
- from deps import Session, err, module_gate, require_session
40
  from core import platform_admin
41
  from routes_admin import admin_gate
42
  import application_registry as applications
@@ -467,6 +467,13 @@ _EXT_MEDIA = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}
467
  #: closed charset — no slashes, no backslashes, no colons, so a code can never be a path.
468
  _CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
469
  _IMMUTABLE = "public, max-age=31536000, immutable"
 
 
 
 
 
 
 
470
 
471
  #: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
472
  _DIR_INDEX = {"path": None, "mtime": None, "map": {}}
@@ -548,8 +555,16 @@ def _assets():
548
  return assets
549
 
550
 
 
 
 
 
 
 
 
 
551
  @router.get("/assets/products")
552
- def asset_manifest(session: Session = Depends(module_gate(MODULE))):
553
  """The codes that HAVE an image — so the catalog designer can mark gaps instead of
554
  rendering broken tiles. `source` names which backend answered."""
555
  d = _local_dir()
@@ -563,7 +578,7 @@ def asset_manifest(session: Session = Depends(module_gate(MODULE))):
563
 
564
  @router.get("/assets/products/{code}")
565
  def product_asset(code: str, q: str = "web",
566
- session: Session = Depends(module_gate(MODULE))):
567
  if q not in _QUALITIES:
568
  raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
569
  norm = _norm(code)
@@ -583,7 +598,7 @@ def product_asset(code: str, q: str = "web",
583
 
584
  media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
585
  return FileResponse(str(path), media_type=media,
586
- headers={"Cache-Control": _IMMUTABLE})
587
 
588
 
589
  # ------------------------------------------------------------------ editorial (DEBT-5)
@@ -626,7 +641,7 @@ def _ed_dir_index(ed):
626
 
627
 
628
  @router.get("/assets/editorial")
629
- def editorial_manifest(session: Session = Depends(module_gate(MODULE))):
630
  """The slugs that HAVE an editorial image — the designer's picker reads this."""
631
  if _local_dir() is not None:
632
  ed = _ed_dir()
@@ -637,7 +652,7 @@ def editorial_manifest(session: Session = Depends(module_gate(MODULE))):
637
 
638
  @router.get("/assets/editorial/{slug}")
639
  def editorial_asset(slug: str, q: str = "web",
640
- session: Session = Depends(module_gate(MODULE))):
641
  if q not in _QUALITIES:
642
  raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
643
  slug = str(slug or "").strip().lower()
@@ -656,7 +671,7 @@ def editorial_asset(slug: str, q: str = "web",
656
  raise err(404, "asset_not_found", f"no editorial image named {slug}")
657
  media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
658
  return FileResponse(str(path), media_type=media,
659
- headers={"Cache-Control": _IMMUTABLE})
660
 
661
 
662
  @router.post("/assets/editorial", status_code=201)
@@ -825,7 +840,7 @@ def record_upload(body: dict = Body(default=None),
825
 
826
  @router.get("/assets/records/{asset_id}")
827
  def record_asset(asset_id: str, q: str = "web",
828
- session: Session = Depends(require_session)):
829
  """Serve one record image for THIS session's tenant. Any signed-in user of the tenant may
830
  read it — a picture on a row is visible to whoever can open the row, and the row walls
831
  already decided that."""
@@ -853,4 +868,4 @@ def record_asset(asset_id: str, q: str = "web",
853
  raise err(404, "asset_not_found", "no image for that reference")
854
  media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
855
  return FileResponse(str(path), media_type=media,
856
- headers={"Cache-Control": _IMMUTABLE})
 
36
  from fastapi import APIRouter
37
  from fastapi.responses import FileResponse, Response
38
 
39
+ from deps import Session, err, is_bearer, module_gate_or_bearer, require_session, require_session_or_bearer
40
  from core import platform_admin
41
  from routes_admin import admin_gate
42
  import application_registry as applications
 
467
  #: closed charset — no slashes, no backslashes, no colons, so a code can never be a path.
468
  _CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
469
  _IMMUTABLE = "public, max-age=31536000, immutable"
470
+ #: A bearer answer stays out of shared caches: a CDN must never serve a key-fetched image to anyone.
471
+ _BEARER_CACHE = "private, max-age=300"
472
+
473
+
474
+ def _cache_control(session):
475
+ """The cookie answer keeps its immutable header. A bearer answer is private."""
476
+ return _BEARER_CACHE if is_bearer(session) else _IMMUTABLE
477
 
478
  #: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
479
  _DIR_INDEX = {"path": None, "mtime": None, "map": {}}
 
555
  return assets
556
 
557
 
558
+ def product_image_codes():
559
+ """The upper-cased codes that have a master image, from the dir or the repo manifest."""
560
+ d = _local_dir()
561
+ if d is not None:
562
+ return set(_dir_index(d))
563
+ return {_norm(code) for code in (_assets().manifest() or {})}
564
+
565
+
566
  @router.get("/assets/products")
567
+ def asset_manifest(session: Session = Depends(module_gate_or_bearer(MODULE))):
568
  """The codes that HAVE an image — so the catalog designer can mark gaps instead of
569
  rendering broken tiles. `source` names which backend answered."""
570
  d = _local_dir()
 
578
 
579
  @router.get("/assets/products/{code}")
580
  def product_asset(code: str, q: str = "web",
581
+ session: Session = Depends(module_gate_or_bearer(MODULE))):
582
  if q not in _QUALITIES:
583
  raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
584
  norm = _norm(code)
 
598
 
599
  media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
600
  return FileResponse(str(path), media_type=media,
601
+ headers={"Cache-Control": _cache_control(session)})
602
 
603
 
604
  # ------------------------------------------------------------------ editorial (DEBT-5)
 
641
 
642
 
643
  @router.get("/assets/editorial")
644
+ def editorial_manifest(session: Session = Depends(module_gate_or_bearer(MODULE))):
645
  """The slugs that HAVE an editorial image — the designer's picker reads this."""
646
  if _local_dir() is not None:
647
  ed = _ed_dir()
 
652
 
653
  @router.get("/assets/editorial/{slug}")
654
  def editorial_asset(slug: str, q: str = "web",
655
+ session: Session = Depends(module_gate_or_bearer(MODULE))):
656
  if q not in _QUALITIES:
657
  raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
658
  slug = str(slug or "").strip().lower()
 
671
  raise err(404, "asset_not_found", f"no editorial image named {slug}")
672
  media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
673
  return FileResponse(str(path), media_type=media,
674
+ headers={"Cache-Control": _cache_control(session)})
675
 
676
 
677
  @router.post("/assets/editorial", status_code=201)
 
840
 
841
  @router.get("/assets/records/{asset_id}")
842
  def record_asset(asset_id: str, q: str = "web",
843
+ session: Session = Depends(require_session_or_bearer)):
844
  """Serve one record image for THIS session's tenant. Any signed-in user of the tenant may
845
  read it — a picture on a row is visible to whoever can open the row, and the row walls
846
  already decided that."""
 
868
  raise err(404, "asset_not_found", "no image for that reference")
869
  media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
870
  return FileResponse(str(path), media_type=media,
871
+ headers={"Cache-Control": _cache_control(session)})
api/routes_automation.py CHANGED
@@ -1350,12 +1350,14 @@ def list_automations(session: Session = Depends(_GATE)):
1350
  retired = []
1351
  tables = None
1352
  if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
1353
- tables = engine.ut_all(session.runtime)
1354
  _BOARD_RETIRED.add(session.tenant)
1355
  # Idempotent Board retirement removes only engine-marked stage fields and legacy Board
1356
  # state. User-created Status/Stage columns remain intact.
1357
- engine.retire_automation_board_state(session.runtime, tables=tables)
1358
- retired = engine.retire_preset_automations(session.runtime)["removed"]
 
 
 
1359
  # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
1360
  # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
1361
  # it present and returns immediately. See `engine.seed_statements_agent` for why this one is
 
1350
  retired = []
1351
  tables = None
1352
  if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
 
1353
  _BOARD_RETIRED.add(session.tenant)
1354
  # Idempotent Board retirement removes only engine-marked stage fields and legacy Board
1355
  # state. User-created Status/Stage columns remain intact.
1356
+ if not (session.runtime.get("automation_migrations") or {}).get("board_retired_v1"):
1357
+ tables = engine.ut_all(session.runtime)
1358
+ engine.retire_automation_board_state(session.runtime, tables=tables)
1359
+ retired = engine.retire_preset_automations(session.runtime)["removed"]
1360
+ session.runtime.update("automation_migrations", lambda d: {**(d or {}), "board_retired_v1": True})
1361
  # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
1362
  # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
1363
  # it present and returns immediately. See `engine.seed_statements_agent` for why this one is
api/routes_catalogs.py CHANGED
@@ -1,10 +1,13 @@
1
- """Authenticated catalog record routes for Catalog Apps."""
 
 
 
2
  from fastapi import APIRouter, Body, Depends, Query
3
  from fastapi.responses import Response
4
 
5
  import app_rules
6
  import catalog_store
7
- from deps import Session, err, require_session
8
 
9
 
10
  router = APIRouter(prefix="/api/v1")
@@ -20,29 +23,30 @@ def _call(operation, *args, **kwargs):
20
  @router.get("/apps/{app_id}/catalogs")
21
  def catalogs(app_id: str, cursor: str | None = None,
22
  limit: int = Query(catalog_store.DEFAULT_LIMIT, ge=1, le=catalog_store.MAX_LIMIT),
23
- session: Session = Depends(require_session)):
24
  return _call(catalog_store.list_catalogs, session, app_id, cursor=cursor, limit=limit)
25
 
26
 
27
  @router.post("/apps/{app_id}/catalogs", status_code=201)
28
  def create_catalog(app_id: str, body: dict = Body(default=None),
29
- session: Session = Depends(require_session)):
 
30
  return {"catalog": _call(catalog_store.create_catalog, session, app_id, body)}
31
 
32
 
33
  # Static paths must precede the catalog id route.
34
  @router.get("/catalogs/templates")
35
- def catalog_templates(session: Session = Depends(require_session)):
36
  return app_rules.catalog_templates()
37
 
38
 
39
  @router.get("/catalogs/{catalog_id}")
40
- def read_catalog(catalog_id: str, session: Session = Depends(require_session)):
41
  return {"catalog": _call(catalog_store.read_catalog, session, catalog_id)}
42
 
43
 
44
  @router.get("/catalogs/{catalog_id}/pdf")
45
- def catalog_pdf(catalog_id: str, session: Session = Depends(require_session)):
46
  import catalog_render
47
 
48
  data, _record, _app = _call(catalog_render.render_catalog, session, catalog_id)
@@ -53,16 +57,19 @@ def catalog_pdf(catalog_id: str, session: Session = Depends(require_session)):
53
 
54
  @router.patch("/catalogs/{catalog_id}")
55
  def update_catalog(catalog_id: str, body: dict = Body(default=None),
56
- session: Session = Depends(require_session)):
 
57
  return {"catalog": _call(catalog_store.update_catalog, session, catalog_id, body)}
58
 
59
 
60
  @router.post("/catalogs/{catalog_id}/duplicate", status_code=201)
61
  def duplicate_catalog(catalog_id: str, body: dict = Body(default=None),
62
- session: Session = Depends(require_session)):
 
63
  return {"catalog": _call(catalog_store.duplicate_catalog, session, catalog_id, body)}
64
 
65
 
66
  @router.delete("/catalogs/{catalog_id}")
67
- def delete_catalog(catalog_id: str, session: Session = Depends(require_session)):
 
68
  return _call(catalog_store.delete_catalog, session, catalog_id)
 
1
+ """Authenticated catalog record routes for Catalog Apps.
2
+
3
+ Every route admits the cookie or an external bearer under its own key's grants (S79-T09).
4
+ """
5
  from fastapi import APIRouter, Body, Depends, Query
6
  from fastapi.responses import Response
7
 
8
  import app_rules
9
  import catalog_store
10
+ from deps import Session, err, require_bearer_write, require_session_or_bearer
11
 
12
 
13
  router = APIRouter(prefix="/api/v1")
 
23
  @router.get("/apps/{app_id}/catalogs")
24
  def catalogs(app_id: str, cursor: str | None = None,
25
  limit: int = Query(catalog_store.DEFAULT_LIMIT, ge=1, le=catalog_store.MAX_LIMIT),
26
+ session: Session = Depends(require_session_or_bearer)):
27
  return _call(catalog_store.list_catalogs, session, app_id, cursor=cursor, limit=limit)
28
 
29
 
30
  @router.post("/apps/{app_id}/catalogs", status_code=201)
31
  def create_catalog(app_id: str, body: dict = Body(default=None),
32
+ session: Session = Depends(require_session_or_bearer)):
33
+ require_bearer_write(session, "catalogs")
34
  return {"catalog": _call(catalog_store.create_catalog, session, app_id, body)}
35
 
36
 
37
  # Static paths must precede the catalog id route.
38
  @router.get("/catalogs/templates")
39
+ def catalog_templates(session: Session = Depends(require_session_or_bearer)):
40
  return app_rules.catalog_templates()
41
 
42
 
43
  @router.get("/catalogs/{catalog_id}")
44
+ def read_catalog(catalog_id: str, session: Session = Depends(require_session_or_bearer)):
45
  return {"catalog": _call(catalog_store.read_catalog, session, catalog_id)}
46
 
47
 
48
  @router.get("/catalogs/{catalog_id}/pdf")
49
+ def catalog_pdf(catalog_id: str, session: Session = Depends(require_session_or_bearer)):
50
  import catalog_render
51
 
52
  data, _record, _app = _call(catalog_render.render_catalog, session, catalog_id)
 
57
 
58
  @router.patch("/catalogs/{catalog_id}")
59
  def update_catalog(catalog_id: str, body: dict = Body(default=None),
60
+ session: Session = Depends(require_session_or_bearer)):
61
+ require_bearer_write(session, "catalogs")
62
  return {"catalog": _call(catalog_store.update_catalog, session, catalog_id, body)}
63
 
64
 
65
  @router.post("/catalogs/{catalog_id}/duplicate", status_code=201)
66
  def duplicate_catalog(catalog_id: str, body: dict = Body(default=None),
67
+ session: Session = Depends(require_session_or_bearer)):
68
+ require_bearer_write(session, "catalogs")
69
  return {"catalog": _call(catalog_store.duplicate_catalog, session, catalog_id, body)}
70
 
71
 
72
  @router.delete("/catalogs/{catalog_id}")
73
+ def delete_catalog(catalog_id: str, session: Session = Depends(require_session_or_bearer)):
74
+ require_bearer_write(session, "catalogs")
75
  return _call(catalog_store.delete_catalog, session, catalog_id)
api/routes_competitor_watch.py CHANGED
@@ -153,7 +153,7 @@ def _not_ready_message(cw, periods):
153
  return cw.NOT_READY
154
 
155
 
156
- def screen_payload(cw, detail, terms, term, can_edit):
157
  """Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
158
  periods = list(detail.get("periods") or ())
159
  top3 = list(detail.get("top3") or ())
@@ -200,18 +200,21 @@ def screen_payload(cw, detail, terms, term, can_edit):
200
  "previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
201
  "comparisonReady": ready,
202
  "comparisonMessage": "" if ready else _not_ready_message(cw, periods),
203
- "top3": rows, "trend": trend, "problems": []}
 
 
204
 
205
 
206
  @router.get("/screen")
207
  def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
 
208
  session: Session = Depends(_app_reader)):
209
  """S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
210
  cw = _module()
211
  tenant = _tenant(session)
212
  record = _app(session, app)
213
- terms = [row["term"] for row in _call(cw.terms_summary, tenant, st=session.runtime)["terms"]]
214
- picked = " ".join(str(term or "").split()).lower()
215
- picked = picked if picked in terms else (terms[0] if terms else None)
216
  detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
217
- return screen_payload(cw, detail, terms, picked, _may_edit(session, record))
 
153
  return cw.NOT_READY
154
 
155
 
156
+ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
157
  """Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
158
  periods = list(detail.get("periods") or ())
159
  top3 = list(detail.get("top3") or ())
 
200
  "previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
201
  "comparisonReady": ready,
202
  "comparisonMessage": "" if ready else _not_ready_message(cw, periods),
203
+ "top3": rows, "trend": trend, "problems": list((page or {}).get("problems") or []),
204
+ **{key: (page or {}).get(key, default) for key, default in
205
+ (("termTotal", len(terms)), ("termOffset", 0), ("termLimit", len(terms)))}}
206
 
207
 
208
  @router.get("/screen")
209
  def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
210
+ q: str = "", offset: int = 0, limit: int = 50,
211
  session: Session = Depends(_app_reader)):
212
  """S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
213
  cw = _module()
214
  tenant = _tenant(session)
215
  record = _app(session, app)
216
+ page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
217
+ limit=limit, selected=term)
218
+ terms, picked = page["terms"], page["term"]
219
  detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
220
+ return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
api/routes_connectors.py CHANGED
@@ -54,6 +54,7 @@ from core import connector_sync
54
  from core.request_timing import raise_if_resource_error
55
  from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
56
  from harness import amazon_client
 
57
  #: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
58
  #: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already
59
  #: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module
@@ -499,6 +500,12 @@ TOKEN_CONNECTORS = [
499
  "marketplace_id.",
500
  # R22: status comes from the stored verification state, without mechanism copy.
501
  "note": ""},
 
 
 
 
 
 
502
  ]
503
 
504
  #: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
@@ -602,7 +609,7 @@ def _register_probers():
602
  store a credential and never the API's ability to boot.
603
  """
604
  ok = False
605
- for mod in ("connectors_meta", "connectors_amazon"):
606
  try:
607
  ok = __import__(mod).register() or ok
608
  except Exception: # noqa: BLE001
@@ -986,7 +993,8 @@ def _entry_types(rt, uname=""):
986
  types, scopes = set(), {}
987
  for e in rows:
988
  t = str(e.get("type") or "")
989
- if t == "amazon_sp":
 
990
  from core import keychain as kc
991
  if kc.entry_verified(rt, e.get("id")):
992
  types.add(t)
@@ -1238,7 +1246,9 @@ def directory(rt, uname, is_admin=False, incomplete=None):
1238
  rows.append({
1239
  "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"],
1240
  "hint": t["hint"],
1241
- "state": ("expired" if t["entry"] == "amazon_sp" and amazon_expired
 
 
1242
  else "connected" if t["entry"] in have else "reconnect"
1243
  if t["entry"] == "amazon_sp" and t["entry"] in scopes
1244
  else "available"),
@@ -1268,8 +1278,10 @@ def directory(rt, uname, is_admin=False, incomplete=None):
1268
  # 400s — the exact failure R6 removed from the Google card. Today that means
1269
  # Stripe and Shopify open for a member and Meta Ads does not
1270
  # ([[one-question-two-normalizers]]).
 
1271
  "manage": ("keychain" if is_admin
1272
  else "" if t["entry"] in TENANT_WIDE_TYPES
 
1273
  else "keychain_personal"),
1274
  # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
1275
  # key is real and useful, and it is not the same as the data being on screen.
@@ -3519,6 +3531,14 @@ def _entry_type_for(connector_key):
3519
  _AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
3520
  "refresh_token": "Refresh token", "seller_id": "Seller ID",
3521
  "marketplace_id": "Marketplace ID"}
 
 
 
 
 
 
 
 
3522
  _CONNECT_FIELDS = {
3523
  "odoo": [{"key": "url", "label": "Server URL", "secret": False,
3524
  "hint": "Your Odoo address, starting with https://."},
@@ -3545,6 +3565,14 @@ _CONNECT_FIELDS = {
3545
  **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
3546
  for key in amazon_client.CREDENTIAL_FIELDS
3547
  ],
 
 
 
 
 
 
 
 
3548
  "stripe": [{"key": "api_key", "label": "API key", "secret": True,
3549
  "hint": "A restricted key with read access."}],
3550
  "shopify": [{"key": "api_key", "label": "Admin API token", "secret": True,
@@ -3669,6 +3697,16 @@ _CONNECT_TUTORIALS = {
3669
  "and check its marketplace access. Correct the saved key in Settings.",
3670
  "testNote": "Checks marketplace access. Saves the key before testing.",
3671
  },
 
 
 
 
 
 
 
 
 
 
3672
  "stripe": {
3673
  "need": "Access to your Stripe dashboard's API keys.",
3674
  "steps": ["Open the Stripe dashboard, then Developers and API keys.",
@@ -3695,12 +3733,12 @@ _CONNECT_TUTORIALS = {
3695
  },
3696
  "webhooks": {
3697
  "need": "An automation and a system that can send an HTTP POST.",
3698
- "steps": ["Go to Automations and create or open an automation.",
3699
- "Choose the Webhooks connector trigger. Saving it creates its token.",
3700
- "Use your Loopable host followed by /api/v1/automations/{automation_id}/hook/{token}. "
3701
- "Use this automation's ID and trigger token; keep the token private.",
3702
- "Paste that URL into your sending system."],
3703
- "failureAction": "Check the automation ID and token in the URL, then confirm the trigger is enabled.",
3704
  "testNote": "Sending to the hook runs the automation. Check its run history after an intentional send.",
3705
  },
3706
  }
@@ -3905,7 +3943,8 @@ def _detail_connect(entry_type, is_admin):
3905
  if not entry_type:
3906
  return None
3907
  fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
3908
- tenant_wide = entry_type in TENANT_WIDE_TYPES
 
3909
  hint = ""
3910
  try:
3911
  hint = next((str(token.get("hint") or "") for token in TOKEN_CONNECTORS
 
54
  from core.request_timing import raise_if_resource_error
55
  from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
56
  from harness import amazon_client
57
+ import connectors_amazon_ads
58
  #: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
59
  #: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already
60
  #: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module
 
500
  "marketplace_id.",
501
  # R22: status comes from the stored verification state, without mechanism copy.
502
  "note": ""},
503
+ # S79-T13 (R7, R9): Amazon Ads reads SP, SB and SD reports. Settings only with a verified
504
+ # credential; an expired or failed token reads Connect again (`connectors_amazon_ads.card_state`).
505
+ {"key": "amazon_ads", "label": "Amazon Ads", "entry": "amazon_ads",
506
+ "desc": "Sponsored ads reports",
507
+ "hint": "Paste the four values from your Amazon Ads app",
508
+ "note": ""},
509
  ]
510
 
511
  #: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
 
609
  store a credential and never the API's ability to boot.
610
  """
611
  ok = False
612
+ for mod in ("connectors_meta", "connectors_amazon", "connectors_amazon_ads"):
613
  try:
614
  ok = __import__(mod).register() or ok
615
  except Exception: # noqa: BLE001
 
993
  types, scopes = set(), {}
994
  for e in rows:
995
  t = str(e.get("type") or "")
996
+ # Amazon SP and Amazon Ads count only with a passing probe proof for the current secret.
997
+ if t in ("amazon_sp", "amazon_ads"):
998
  from core import keychain as kc
999
  if kc.entry_verified(rt, e.get("id")):
1000
  types.add(t)
 
1246
  rows.append({
1247
  "key": t["key"], "label": t["label"], "kind": "token", "desc": t["desc"],
1248
  "hint": t["hint"],
1249
+ "state": (connectors_amazon_ads.card_state(t["entry"] in have)
1250
+ if t["entry"] == connectors_amazon_ads.ENTRY_TYPE
1251
+ else "expired" if t["entry"] == "amazon_sp" and amazon_expired
1252
  else "connected" if t["entry"] in have else "reconnect"
1253
  if t["entry"] == "amazon_sp" and t["entry"] in scopes
1254
  else "available"),
 
1278
  # 400s — the exact failure R6 removed from the Google card. Today that means
1279
  # Stripe and Shopify open for a member and Meta Ads does not
1280
  # ([[one-question-two-normalizers]]).
1281
+ # S79-T13: Amazon Ads is business wide, before and after the keychain learns it.
1282
  "manage": ("keychain" if is_admin
1283
  else "" if t["entry"] in TENANT_WIDE_TYPES
1284
+ or t["entry"] == connectors_amazon_ads.ENTRY_TYPE
1285
  else "keychain_personal"),
1286
  # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
1287
  # key is real and useful, and it is not the same as the data being on screen.
 
3531
  _AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
3532
  "refresh_token": "Refresh token", "seller_id": "Seller ID",
3533
  "marketplace_id": "Marketplace ID"}
3534
+ _AMAZON_ADS_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secret",
3535
+ "refresh_token": "Refresh token", "profile_id": "Profile ID"}
3536
+ _AMAZON_ADS_FIELD_HINTS = {
3537
+ "client_id": "Your Amazon Ads app's Login with Amazon client ID.",
3538
+ "client_secret": "The same app's client secret.",
3539
+ "refresh_token": "The refresh token returned when you authorize the app.",
3540
+ "profile_id": "The advertising profile for your marketplace.",
3541
+ }
3542
  _CONNECT_FIELDS = {
3543
  "odoo": [{"key": "url", "label": "Server URL", "secret": False,
3544
  "hint": "Your Odoo address, starting with https://."},
 
3565
  **({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
3566
  for key in amazon_client.CREDENTIAL_FIELDS
3567
  ],
3568
+ # S79-T13: keys and order derived from `amazon_ads_client.CREDENTIAL_FIELDS`, like Amazon SP.
3569
+ "amazon_ads": [
3570
+ {"key": key,
3571
+ "label": _AMAZON_ADS_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()),
3572
+ "secret": key in ("client_secret", "refresh_token"),
3573
+ "hint": _AMAZON_ADS_FIELD_HINTS.get(key, "")}
3574
+ for key in connectors_amazon_ads.CREDENTIAL_FIELDS
3575
+ ],
3576
  "stripe": [{"key": "api_key", "label": "API key", "secret": True,
3577
  "hint": "A restricted key with read access."}],
3578
  "shopify": [{"key": "api_key", "label": "Admin API token", "secret": True,
 
3697
  "and check its marketplace access. Correct the saved key in Settings.",
3698
  "testNote": "Checks marketplace access. Saves the key before testing.",
3699
  },
3700
+ "amazon_ads": {
3701
+ "need": "An Amazon Ads API app and access to your advertising account.",
3702
+ "steps": ["Open your Amazon Ads API app's Login with Amazon security profile.",
3703
+ "Copy its client ID and client secret.",
3704
+ "Authorize the app for your advertising account and copy the refresh token.",
3705
+ "Copy the profile ID of the marketplace you advertise in."],
3706
+ "failureAction": "Authorize the app again for your advertising account, then check "
3707
+ "that the profile ID belongs to it.",
3708
+ "testNote": "Checks access to your advertising profile. Saves the key before testing.",
3709
+ },
3710
  "stripe": {
3711
  "need": "Access to your Stripe dashboard's API keys.",
3712
  "steps": ["Open the Stripe dashboard, then Developers and API keys.",
 
3733
  },
3734
  "webhooks": {
3735
  "need": "An automation and a system that can send an HTTP POST.",
3736
+ # The client lists each automation's generated URL. No step names a credential or a keychain Test.
3737
+ "steps": ["Open Automations and create or open an automation.",
3738
+ "Choose the Webhooks trigger and save it. Saving creates its own URL.",
3739
+ "Copy its URL below. Keep it private, because anyone with it can run the automation.",
3740
+ "Send an HTTP POST to that URL from your other system."],
3741
+ "failureAction": "Copy the URL again from this guide. Check that the trigger is on.",
3742
  "testNote": "Sending to the hook runs the automation. Check its run history after an intentional send.",
3743
  },
3744
  }
 
3943
  if not entry_type:
3944
  return None
3945
  fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
3946
+ tenant_wide = (entry_type in TENANT_WIDE_TYPES
3947
+ or entry_type == connectors_amazon_ads.ENTRY_TYPE)
3948
  hint = ""
3949
  try:
3950
  hint = next((str(token.get("hint") or "") for token in TOKEN_CONNECTORS
api/routes_customers.py CHANGED
@@ -229,6 +229,22 @@ def warm_default(rt):
229
  _pool_for(rt, None, None)
230
 
231
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
232
  def _team_agent(session: Session):
233
  """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by
234
  `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree.
 
229
  _pool_for(rt, None, None)
230
 
231
 
232
+ def warm_bu_pools(rt):
233
+ """Warm each permitted business-unit pool outside the request path."""
234
+ from core import context, perms
235
+ for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
236
+ if label == "All" or label not in context.BRANDS:
237
+ continue
238
+ team = perms.scope_team_id({"bus": [context.BRANDS[label]], "role": "user"})
239
+ if team is None:
240
+ continue
241
+ try:
242
+ _pool_for(rt, team, None)
243
+ print(f"[aios-api] customer read pool warm: {label}")
244
+ except Exception as exc:
245
+ print(f"[aios-api] customer read pool warm refused: {label} ({type(exc).__name__})")
246
+
247
+
248
  def _team_agent(session: Session):
249
  """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by
250
  `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree.
api/routes_external.py CHANGED
@@ -27,6 +27,9 @@ import aios_session
27
  from deps import Session, err, require_session
28
  from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets
29
  from harness import runtime
 
 
 
30
 
31
 
32
  EXTERNAL_PREFIX = "/api/v1/external"
@@ -228,6 +231,9 @@ class _PreviewRuntime:
228
  key = str(name)
229
  self._load(key)
230
  base = self._base_revisions.get(key)
 
 
 
231
  rev = int((base or {}).get("rev") or 0) + self._local_revisions.get(key, 0)
232
  return {"rev": rev, "token": _json_digest({"key": key, "rev": rev,
233
  "base": base})}
@@ -851,6 +857,12 @@ def _assert_query_filter_bounds(raw) -> None:
851
  stack.extend((child, depth + 1) for child in children)
852
 
853
 
 
 
 
 
 
 
854
  def _assert_tool_query_filter_bounds(name, arguments):
855
  """Name query limits before the finite tool schema reports a generic shape error."""
856
  if name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database":
@@ -1616,6 +1628,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
1616
  "",
1617
  ])
1618
 
 
1619
  if public:
1620
  lines.extend([
1621
  "## Your read scope",
@@ -2381,7 +2394,7 @@ def external_setup(request: Request):
2381
 
2382
 
2383
  @router.get("/skill.md")
2384
- def portable_skill(request: Request, session: Session = Depends(require_session)):
2385
  """Download current governed external read and build instructions."""
2386
  require_external_analyst(session)
2387
  document = _portable_skill_document(session, str(request.base_url))
@@ -2483,11 +2496,13 @@ def query_database(
2483
  database: str,
2484
  body: dict | None = Body(default=None),
2485
  session: Session = Depends(require_external_session),
 
2486
  ):
2487
  """Read one freshly permissioned, bounded and cursor-stable Assistant snapshot."""
2488
  try:
2489
  require_external_analyst(session)
2490
- return _query_database(session, database, body if body is not None else {})
 
2491
  except HTTPException as exc:
2492
  external_activity.record_refused(session, database, exc)
2493
  raise
@@ -2557,6 +2572,11 @@ def _query_database(session: Session, database: str, payload: dict):
2557
  "assistant_store_not_ready", "store_not_ready", "amazon_mirror_not_loaded"}:
2558
  raise err(503, "query_source_unavailable",
2559
  "This database is temporarily unavailable. Try the same query again shortly.") from exc
 
 
 
 
 
2560
  if info.get("code") == "query_cursor_stale":
2561
  fix = _MCP_FIXES["query_cursor_stale"]
2562
  if fix not in info["message"]:
@@ -2612,6 +2632,37 @@ def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggrega
2612
  return result
2613
 
2614
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2615
  def _query_database_page(session: Session, database: str, payload: dict):
2616
  if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
2617
  raise err(400, "query_arguments_invalid",
@@ -2784,6 +2835,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
2784
  records = [{"pid": copy.deepcopy(row.get("pid")),
2785
  **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
2786
  for row in page_rows]
 
2787
  next_offset = offset + limit
2788
  check_publication()
2789
  next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset, publication)
@@ -2849,6 +2901,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
2849
  **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
2850
  for row in page_rows
2851
  ]
 
2852
  next_offset = offset + len(page_rows)
2853
  check_publication()
2854
  next_cursor = (_encode_query_cursor(session, query_digest, snapshot_digest, next_offset, publication)
@@ -3259,6 +3312,11 @@ def _application_upsert(session, args):
3259
  allowed, _ = _application_access(session, current)
3260
  if not current or not allowed:
3261
  raise applications.ApplicationError("application does not exist")
 
 
 
 
 
3262
  proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current
3263
  proposed_allowed, records = _application_access(session, proposed, strict=True)
3264
  return {"application": applications.mutate(
@@ -3616,7 +3674,25 @@ def _execute_database_object(session, args, *, plan=None):
3616
  except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
3617
  raise err(409, "confirmation_stale",
3618
  "The database changed after the preview, " + _STALE_FIX) from exc
3619
- return {"database": database, **copy.deepcopy(plan.get("planned") or {}),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3620
  "affected": copy.deepcopy(plan.get("affected") or {})}
3621
 
3622
 
@@ -4052,6 +4128,11 @@ def _affected_for(name, args, diff, result=None):
4052
  ranges.append({"workbook_id": arguments.get("workbook_id"),
4053
  "tab": arguments.get("tab") or arguments.get("worksheet"), "range": address})
4054
  collect(args)
 
 
 
 
 
4055
  return {"ids": values[:100], "count": max(len(values), len(diff)), "ranges": ranges}
4056
 
4057
 
@@ -4617,7 +4698,7 @@ def _confirmation_claim(session, operation_id):
4617
  if item["status"] != "preview":
4618
  raise _OperationAlreadyClaimed()
4619
  if item["payload"]["expiry"] <= time.time():
4620
- raise err(409, "confirmation_expired", "This confirmation expired. Preview the change again.")
4621
  item["status"] = "claimed"
4622
  item["retain_until"] = time.time() + _OPERATION_RETENTION_SECONDS
4623
  return raw
@@ -4746,6 +4827,24 @@ def _finish_website_note(session, payload, plan, result):
4746
  _STALE_FIX = "so repeat the original call for a new preview and confirm with its new confirmation_token."
4747
 
4748
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4749
  def _confirm_fix(args):
4750
  """Name what a confirm_change call got wrong and the shape that succeeds."""
4751
  allowed = ("confirmation_token", "confirmation_phrase", "intent")
@@ -4778,8 +4877,7 @@ def _confirm_change(session, args):
4778
  if payload.get("environment") != _write_environment():
4779
  raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX)
4780
  if int(payload.get("expiry") or 0) <= int(time.time()):
4781
- raise err(409, "confirmation_expired",
4782
- "This confirmation token expired. Preview the change again.")
4783
  if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE:
4784
  raise err(400, "confirmation_phrase_required",
4785
  "Enter the exact stronger confirmation phrase shown in the preview.")
@@ -4905,7 +5003,7 @@ def _confirm_change(session, args):
4905
  _MCP_CODE_GROUPS = (
4906
  ("The arguments do not match this action. Read its guide and correct them.", """
4907
  tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
4908
- query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
4909
  bad_application bad_entries bad_value bad_values bad_request bad_recurrence invalid_field_contract
4910
  external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid
4911
  """),
@@ -5000,6 +5098,12 @@ MCP_RESULT_CODES.update({
5000
  "confirmation_session_required": "Read this confirmation through its authenticated connection.",
5001
  "link_selection_refused": "Choose a readable linked record that matches the field's selection condition.",
5002
  "app_rule": "Correct the App rule named in the refusal before saving it.",
 
 
 
 
 
 
5003
  "audience_not_allowed": "A person must choose sharing with everyone on screen.",
5004
  "not_owner": "You cannot administer this item's audience.",
5005
  "unknown_people": "Choose active people in this workspace.",
@@ -5148,6 +5252,8 @@ _MCP_RESULT_OBJECTS = frozenset({
5148
  "applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools",
5149
  "annotations", "serverInfo", "environment",
5150
  })
 
 
5151
  _MCP_AUTHORED_DESCRIPTIONS = frozenset({
5152
  "fields", "file", "files", "template", "templates", "application", "applications", "view", "views",
5153
  })
@@ -5174,6 +5280,8 @@ def _mcp_result_copy(value, labels=None, *, field=None, metadata=False):
5174
  result[key] = copy.deepcopy(child)
5175
  elif key == "about":
5176
  result[key] = copy.deepcopy(child)
 
 
5177
  elif key in _MCP_RESULT_OPAQUE and not metadata:
5178
  result[key] = copy.deepcopy(child)
5179
  elif key in {"schema", "inputSchema"}:
@@ -5944,7 +6052,10 @@ def _record_mcp_method(session, req, reply):
5944
  or audit.redact_intent(method) != method):
5945
  method = "[redacted]"
5946
  error = reply.get("error", {}).get("code")
5947
- code = "request_method_unknown" if error == -32601 else "request_invalid" if error else ""
 
 
 
5948
  external_activity.record_protocol(session, method, code=code)
5949
  except Exception:
5950
  pass
@@ -6175,6 +6286,20 @@ def _dispatch_mcp(req, session: Session, base_url: str = ""):
6175
  return reply
6176
 
6177
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6178
  def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
6179
  """One HTTP request object; the stdio precedent's JSON-RPC and content envelopes."""
6180
  protocol_error = _mcp_protocol_error
@@ -6326,6 +6451,9 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
6326
  wiki_log.record_build(session, kind=kind, object_id=arguments["arguments"]["path"],
6327
  event_id="mcp-wiki:" + _json_digest({"path": arguments["arguments"]["path"], "version": data["version"]}),
6328
  occurred_at=data["updated"], purpose=purpose, about=about, build_kind=kind)
 
 
 
6329
  else:
6330
  data = _dispatch_external_tool(_tool_context(session), name, arguments)
6331
  if isinstance(data, dict) and data.get("refused"):
@@ -6353,6 +6481,10 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
6353
  import routes_sheet
6354
  if exc.code == "tool_arguments_invalid" and name == "confirm_change":
6355
  exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments))
 
 
 
 
6356
  translated = routes_sheet._refuse(exc)
6357
  info = translated.detail["error"]
6358
  out = _tool_refusal(name, info["code"], info["message"], translated.status_code,
@@ -6377,6 +6509,8 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
6377
  message = info.get("message", "The app refused this operation.")
6378
  if info.get("code") == "build_about_required" and isinstance(message, str):
6379
  message += _mcp_about_fix(called)
 
 
6380
  out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
6381
  except (TypeError, ValueError, KeyError, AttributeError):
6382
  out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
 
27
  from deps import Session, err, require_session
28
  from core import audit, data_binding, external_access, external_oauth, external_tools, perm_scope, sheets
29
  from harness import runtime
30
+ from tool_groups.data import (QUERY_SYNTAX_CODES, UNKNOWN_DATABASE_FIX, bad_filter_message,
31
+ query_syntax_section, with_query_syntax)
32
+ from tool_groups.sheet import WORKBOOK_MISSING_FIX
33
 
34
 
35
  EXTERNAL_PREFIX = "/api/v1/external"
 
231
  key = str(name)
232
  self._load(key)
233
  base = self._base_revisions.get(key)
234
+ if isinstance(base, dict) and not self._local_revisions.get(key, 0):
235
+ # D-8920: an unchanged copy answers its source revision, so a preview staged inside a preview pins it.
236
+ return copy.deepcopy(base)
237
  rev = int((base or {}).get("rev") or 0) + self._local_revisions.get(key, 0)
238
  return {"rev": rev, "token": _json_digest({"key": key, "rev": rev,
239
  "base": base})}
 
857
  stack.extend((child, depth + 1) for child in children)
858
 
859
 
860
+ def _is_query_call(name, arguments):
861
+ """Tell a query_database call at its bare, table or data address."""
862
+ return name == "query_database" or (
863
+ name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database")
864
+
865
+
866
  def _assert_tool_query_filter_bounds(name, arguments):
867
  """Name query limits before the finite tool schema reports a generic shape error."""
868
  if name in ("table", "data") and isinstance(arguments, dict) and arguments.get("action") == "query_database":
 
1628
  "",
1629
  ])
1630
 
1631
+ lines.extend(query_syntax_section())
1632
  if public:
1633
  lines.extend([
1634
  "## Your read scope",
 
2394
 
2395
 
2396
  @router.get("/skill.md")
2397
+ def portable_skill(request: Request, session: Session = Depends(deps.require_session_or_bearer)):
2398
  """Download current governed external read and build instructions."""
2399
  require_external_analyst(session)
2400
  document = _portable_skill_document(session, str(request.base_url))
 
2496
  database: str,
2497
  body: dict | None = Body(default=None),
2498
  session: Session = Depends(require_external_session),
2499
+ request: Request = None,
2500
  ):
2501
  """Read one freshly permissioned, bounded and cursor-stable Assistant snapshot."""
2502
  try:
2503
  require_external_analyst(session)
2504
+ with _external_image_base(_mcp_public_origin(request) if request is not None else ""):
2505
+ return _query_database(session, database, body if body is not None else {})
2506
  except HTTPException as exc:
2507
  external_activity.record_refused(session, database, exc)
2508
  raise
 
2572
  "assistant_store_not_ready", "store_not_ready", "amazon_mirror_not_loaded"}:
2573
  raise err(503, "query_source_unavailable",
2574
  "This database is temporarily unavailable. Try the same query again shortly.") from exc
2575
+ if info.get("code") == "assistant_bad_filter":
2576
+ # S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
2577
+ info["message"] = bad_filter_message(info.get("message"))
2578
+ if info.get("code") in QUERY_SYNTAX_CODES and isinstance(info.get("message"), str):
2579
+ info["message"] = with_query_syntax(info["message"])
2580
  if info.get("code") == "query_cursor_stale":
2581
  fix = _MCP_FIXES["query_cursor_stale"]
2582
  if fix not in info["message"]:
 
2632
  return result
2633
 
2634
 
2635
+ @contextmanager
2636
+ def _external_image_base(base_url):
2637
+ """Give product rows image URLs on this origin for one external query, and only that query."""
2638
+ from modules import product_data
2639
+ token = product_data.EXTERNAL_IMAGE_BASE.set(str(base_url or "").rstrip("/"))
2640
+ try:
2641
+ yield
2642
+ finally:
2643
+ product_data.EXTERNAL_IMAGE_BASE.reset(token)
2644
+
2645
+
2646
+ def _external_image_urls(session, database, rows, records):
2647
+ """Replace each product row's Image cell with a URL the caller's own credential fetches."""
2648
+ if database != "product_data" or (getattr(session, "claims", None) or {}).get("external") is not True:
2649
+ return records
2650
+ from modules import product_data
2651
+ base = product_data.EXTERNAL_IMAGE_BASE.get()
2652
+ key = product_data.IMAGE_FIELD_KEY
2653
+ if base is None or not records or key not in records[0]:
2654
+ return records
2655
+ import routes_assets
2656
+ try:
2657
+ masters = routes_assets.product_image_codes()
2658
+ except Exception: # noqa: BLE001 - an unreadable manifest leaves empty cells empty, never fails the read
2659
+ masters = set()
2660
+ for row, record in zip(rows, records):
2661
+ path = product_data.image_path(record.get(key), row.get("code"), masters)
2662
+ record[key] = base + path if path.startswith("/") else path
2663
+ return records
2664
+
2665
+
2666
  def _query_database_page(session: Session, database: str, payload: dict):
2667
  if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
2668
  raise err(400, "query_arguments_invalid",
 
2835
  records = [{"pid": copy.deepcopy(row.get("pid")),
2836
  **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
2837
  for row in page_rows]
2838
+ records = _external_image_urls(session, database, page_rows, records)
2839
  next_offset = offset + limit
2840
  check_publication()
2841
  next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset, publication)
 
2901
  **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
2902
  for row in page_rows
2903
  ]
2904
+ records = _external_image_urls(session, database, page_rows, records)
2905
  next_offset = offset + len(page_rows)
2906
  check_publication()
2907
  next_cursor = (_encode_query_cursor(session, query_digest, snapshot_digest, next_offset, publication)
 
3312
  allowed, _ = _application_access(session, current)
3313
  if not current or not allowed:
3314
  raise applications.ApplicationError("application does not exist")
3315
+ stored = applications.load(session.runtime)["applications"].get(app_id)
3316
+ from tool_groups.apps import is_template_app, template_sealed
3317
+ if is_template_app(stored):
3318
+ # R35: a template App's definition is sealed over tools. Admins rename and share it in web.
3319
+ raise template_sealed(stored)
3320
  proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current
3321
  proposed_allowed, records = _application_access(session, proposed, strict=True)
3322
  return {"application": applications.mutate(
 
3674
  except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
3675
  raise err(409, "confirmation_stale",
3676
  "The database changed after the preview, " + _STALE_FIX) from exc
3677
+ planned = copy.deepcopy(plan.get("planned") or {})
3678
+ kind = planned.get("action") or action
3679
+ if kind == "field_create":
3680
+ fields = (replacement.get(planned.get("database") or database) or {}).get("fields") or ()
3681
+ created = next((field for field in fields if isinstance(field, dict)
3682
+ and field.get("key") == planned.get("field_key")), {})
3683
+ refresh = created.get("type") in ("link", "rollup", "if")
3684
+ else:
3685
+ refresh = kind in {"field_update", "field_delete",
3686
+ "record_create", "record_update", "record_delete"}
3687
+ if refresh:
3688
+ import routes_tables
3689
+ # Same relation refresh as the app's field and row routes; never from the preview.
3690
+ # The write is committed: a refresh that cannot start must not report it as failed.
3691
+ try:
3692
+ routes_tables._refresh_relations(session)
3693
+ except Exception as exc: # noqa: BLE001
3694
+ print(f"[external] relation refresh deferred: {type(exc).__name__}: {exc}")
3695
+ return {"database": database, **planned,
3696
  "affected": copy.deepcopy(plan.get("affected") or {})}
3697
 
3698
 
 
4128
  ranges.append({"workbook_id": arguments.get("workbook_id"),
4129
  "tab": arguments.get("tab") or arguments.get("worksheet"), "range": address})
4130
  collect(args)
4131
+ inner = args.get("arguments") if name == "apps" and isinstance(args.get("arguments"), dict) else {}
4132
+ # C8: an App item change names its App and every item id it changes.
4133
+ for value in [inner.get("application"), *(inner.get("item_ids") or [])]:
4134
+ if value is not None and str(value) not in values:
4135
+ values.append(str(value))
4136
  return {"ids": values[:100], "count": max(len(values), len(diff)), "ranges": ranges}
4137
 
4138
 
 
4698
  if item["status"] != "preview":
4699
  raise _OperationAlreadyClaimed()
4700
  if item["payload"]["expiry"] <= time.time():
4701
+ raise err(409, "confirmation_expired", _expired_fix(item["payload"]))
4702
  item["status"] = "claimed"
4703
  item["retain_until"] = time.time() + _OPERATION_RETENTION_SECONDS
4704
  return raw
 
4827
  _STALE_FIX = "so repeat the original call for a new preview and confirm with its new confirmation_token."
4828
 
4829
 
4830
+ def _expired_fix(payload):
4831
+ """Name the call that previews an expired change again and the confirm that follows it."""
4832
+ import tool_groups
4833
+ payload = payload if isinstance(payload, dict) else {}
4834
+ tool, arguments = payload.get("tool"), payload.get("arguments")
4835
+ address = "the original tool"
4836
+ if isinstance(tool, str) and isinstance(arguments, dict):
4837
+ try:
4838
+ group, grouped = tool_groups.legacy_target(tool, arguments)
4839
+ address = group if group == tool else group + " with action " + grouped["action"]
4840
+ except (KeyError, TypeError, ValueError):
4841
+ if tool in tool_groups.groups() and isinstance(arguments.get("action"), str):
4842
+ address = tool + " with action " + arguments["action"]
4843
+ return ("This confirmation token expired. Call " + address + " again with the same arguments for a new "
4844
+ "preview. Then call confirm_change with its new confirmation_token within "
4845
+ + str(_CONFIRMATION_TTL_SECONDS // 60) + " minutes.")
4846
+
4847
+
4848
  def _confirm_fix(args):
4849
  """Name what a confirm_change call got wrong and the shape that succeeds."""
4850
  allowed = ("confirmation_token", "confirmation_phrase", "intent")
 
4877
  if payload.get("environment") != _write_environment():
4878
  raise err(409, "confirmation_stale", "The environment changed after the preview, " + _STALE_FIX)
4879
  if int(payload.get("expiry") or 0) <= int(time.time()):
4880
+ raise err(409, "confirmation_expired", _expired_fix(payload))
 
4881
  if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE:
4882
  raise err(400, "confirmation_phrase_required",
4883
  "Enter the exact stronger confirmation phrase shown in the preview.")
 
5003
  _MCP_CODE_GROUPS = (
5004
  ("The arguments do not match this action. Read its guide and correct them.", """
5005
  tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
5006
+ assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
5007
  bad_application bad_entries bad_value bad_values bad_request bad_recurrence invalid_field_contract
5008
  external_edit_empty external_edit_invalid oauth_edit_empty oauth_edit_invalid
5009
  """),
 
5098
  "confirmation_session_required": "Read this confirmation through its authenticated connection.",
5099
  "link_selection_refused": "Choose a readable linked record that matches the field's selection condition.",
5100
  "app_rule": "Correct the App rule named in the refusal before saving it.",
5101
+ "template_app_sealed": "A Loopable template App cannot be changed over tools. Use its own tools. An Administrator can rename or share it in Loopable.",
5102
+ "app_unavailable": "No App with this id is shared with this account. Call apps.list for the Apps it can open.",
5103
+ "app_items_unavailable": "This App has no items to list or change over tools. Open it in Loopable.",
5104
+ "app_items_forbidden": "This account cannot change this App's items. Ask its owner or an Administrator.",
5105
+ "app_item_status_invalid": "Choose a status this App allows. See allowed.",
5106
+ "app_item_unknown": "Choose item ids that apps.items returned for this App.",
5107
  "audience_not_allowed": "A person must choose sharing with everyone on screen.",
5108
  "not_owner": "You cannot administer this item's audience.",
5109
  "unknown_people": "Choose active people in this workspace.",
 
5252
  "applications", "view", "views", "from", "to", "source", "affected", "aggregates", "tools",
5253
  "annotations", "serverInfo", "environment",
5254
  })
5255
+ # A link descriptor names its target database key. It is an identifier, never prose (D-9307).
5256
+ _MCP_RESULT_DESCRIPTORS = frozenset({"link"})
5257
  _MCP_AUTHORED_DESCRIPTIONS = frozenset({
5258
  "fields", "file", "files", "template", "templates", "application", "applications", "view", "views",
5259
  })
 
5280
  result[key] = copy.deepcopy(child)
5281
  elif key == "about":
5282
  result[key] = copy.deepcopy(child)
5283
+ elif key in _MCP_RESULT_DESCRIPTORS and isinstance(child, dict):
5284
+ result[key] = copy.deepcopy(child)
5285
  elif key in _MCP_RESULT_OPAQUE and not metadata:
5286
  result[key] = copy.deepcopy(child)
5287
  elif key in {"schema", "inputSchema"}:
 
6052
  or audit.redact_intent(method) != method):
6053
  method = "[redacted]"
6054
  error = reply.get("error", {}).get("code")
6055
+ if error == -32601:
6056
+ # D-2667: a probe for a method the server does not offer is no refused query and crowds out history.
6057
+ return
6058
+ code = "request_invalid" if error else ""
6059
  external_activity.record_protocol(session, method, code=code)
6060
  except Exception:
6061
  pass
 
6286
  return reply
6287
 
6288
 
6289
+ #: Raw refusal codes whose MCP answer names the call that succeeds next (S79-T11).
6290
+ _MCP_NEXT_CALL = {"workbook_missing": WORKBOOK_MISSING_FIX, "assistant_unknown_database": UNKNOWN_DATABASE_FIX}
6291
+
6292
+
6293
+ def _mcp_next_call(code, message):
6294
+ """Append the call that succeeds next for a refusal that names a missing item (S79-T11)."""
6295
+ text = str(message or "").rstrip()
6296
+ fix = _MCP_NEXT_CALL.get(code, "")
6297
+ if code == "assistant_unknown_database":
6298
+ # The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
6299
+ return fix
6300
+ return text if not fix or fix in text else (text + " " + fix).strip()
6301
+
6302
+
6303
  def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
6304
  """One HTTP request object; the stdio precedent's JSON-RPC and content envelopes."""
6305
  protocol_error = _mcp_protocol_error
 
6451
  wiki_log.record_build(session, kind=kind, object_id=arguments["arguments"]["path"],
6452
  event_id="mcp-wiki:" + _json_digest({"path": arguments["arguments"]["path"], "version": data["version"]}),
6453
  occurred_at=data["updated"], purpose=purpose, about=about, build_kind=kind)
6454
+ elif name == "query_database":
6455
+ with _external_image_base(base_url):
6456
+ data = _dispatch_external_tool(_tool_context(session), name, arguments)
6457
  else:
6458
  data = _dispatch_external_tool(_tool_context(session), name, arguments)
6459
  if isinstance(data, dict) and data.get("refused"):
 
6481
  import routes_sheet
6482
  if exc.code == "tool_arguments_invalid" and name == "confirm_change":
6483
  exc = sheets.SheetRefusal(exc.code, _confirm_fix(arguments))
6484
+ if exc.code == "tool_arguments_invalid" and _is_query_call(name, arguments):
6485
+ exc = sheets.SheetRefusal(exc.code, with_query_syntax(exc.message))
6486
+ if exc.code in _MCP_NEXT_CALL:
6487
+ exc = sheets.SheetRefusal(exc.code, _mcp_next_call(exc.code, exc.message))
6488
  translated = routes_sheet._refuse(exc)
6489
  info = translated.detail["error"]
6490
  out = _tool_refusal(name, info["code"], info["message"], translated.status_code,
 
6509
  message = info.get("message", "The app refused this operation.")
6510
  if info.get("code") == "build_about_required" and isinstance(message, str):
6511
  message += _mcp_about_fix(called)
6512
+ if info.get("code") in _MCP_NEXT_CALL:
6513
+ message = _mcp_next_call(info["code"], message)
6514
  out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
6515
  except (TypeError, ValueError, KeyError, AttributeError):
6516
  out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
api/routes_keychain.py CHANGED
@@ -122,7 +122,7 @@ SCOPE_KEY = "keychain_scopes"
122
  #: (the resolver-side guard held), but a door that stores a credential the product then ignores,
123
  #: which is its own kind of wrong. `routes_connectors.py::_tokens` imports this tuple rather than
124
  #: keeping its own, so its `manage` field for the Amazon card corrects itself in the same change.
125
- TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp")
126
 
127
 
128
  #: ⭐⭐ W53-T29 / OWNER RULING R6 — WHICH SCOPES ONE SOURCE MAY EVER CARRY, SAID AS DATA.
 
122
  #: (the resolver-side guard held), but a door that stores a credential the product then ignores,
123
  #: which is its own kind of wrong. `routes_connectors.py::_tokens` imports this tuple rather than
124
  #: keeping its own, so its `manage` field for the Amazon card corrects itself in the same change.
125
+ TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp", "amazon_ads")
126
 
127
 
128
  #: ⭐⭐ W53-T29 / OWNER RULING R6 — WHICH SCOPES ONE SOURCE MAY EVER CARRY, SAID AS DATA.
api/routes_nav.py CHANGED
@@ -459,11 +459,39 @@ def _visible_database_keys(session):
459
  return keys, enumerated
460
 
461
 
462
- def _granted_view_sources(session):
463
- """`{view_id: owner}` for grants this caller may read, from one registry read.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
464
 
465
  `owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
466
  source lookup for that case; modern grants point directly at the canonical personal stratum.
 
 
467
  """
468
  import core.shares as shares
469
  try:
@@ -476,9 +504,46 @@ def _granted_view_sources(session):
476
  # add a View to "shared with me"; a valid named or everyone grant did.
477
  if shares.role_from(record, session.uname) is not None:
478
  out[str(raw_id)] = owner
479
- return out
480
  except Exception:
481
  return {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
482
 
483
 
484
  def _granted_view_ids(session):
@@ -531,12 +596,24 @@ def _granted_view_ids(session):
531
  #: of the reserved words below is still a second connection, and its sibling must still be named.
532
  #:
533
  #: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
534
- #: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the
535
- #: "Loopable" group; the legacy "Custom" group is reserved too. A connection someone named
536
  #: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
537
  #: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
538
  _RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
539
 
 
 
 
 
 
 
 
 
 
 
 
 
540
  #: The registry modules that ARE the designated default connection's data rather than a namespaced
541
  #: copy of it: `odoo_relational.namespaced_fields` says so in as many words ("`customer_link`
542
  #: targets the compiled registry module `customer_data`, which holds the DEFAULT connection's
@@ -712,7 +789,7 @@ def nav(session: Session = Depends(require_session)):
712
  continue
713
  pages.append({"key": e["key"], "label": e["label"],
714
  "source": e.get("source") or "Blank",
715
- "sourceGroup": e.get("sourceGroup") or "Loopable", "chrome": "main"})
716
  except Exception:
717
  # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT.
718
  #
@@ -771,8 +848,8 @@ def nav(session: Session = Depends(require_session)):
771
  _folder = _odoo_folder_label(key, _conn_labels, _conn_default)
772
  if _folder:
773
  page["sourceGroup"] = _folder
774
- elif page.get("user_table") or source in {"", "Blank", "Custom"}:
775
- page.setdefault("sourceGroup", "Loopable")
776
  # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
777
  #
778
  # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
@@ -1498,17 +1575,19 @@ def create_nav_application(body: dict = Body(default=None),
1498
  wizard_kinds = {row["kind"] for row in application_registry.template_contracts(session=session)}
1499
  if template_kind not in wizard_kinds:
1500
  raise err(400, "bad_application", "application template kind is not recognized")
1501
- source = str(payload.get("sourceDatabase") or "").strip()
1502
- if not source:
 
 
1503
  raise err(400, "bad_application", "sourceDatabase is required")
1504
- fields = application_registry.chart_field_contracts(session, [source])[source]
1505
  if not session.runtime.available():
1506
  raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.")
1507
  try:
1508
  application = application_registry.create(
1509
  session.runtime, session.tenant, session.uname, payload,
1510
- field_contracts={source: fields if isinstance(fields, list) else []},
1511
- allowed_databases={source})
1512
  except PermissionError as exc:
1513
  raise err(403, "forbidden", str(exc))
1514
  except application_registry.ApplicationExistsError as exc:
 
459
  return keys, enumerated
460
 
461
 
462
+ def _folder_views_in(doc, folder_id, owner=None):
463
+ """`(owner, [view_id])` for the VIEWS folder `folder_id` in one held workspace doc, else None.
464
+
465
+ The same lookup as `TableStore.find_folder`, over a projection the caller already read.
466
+ """
467
+ import core.table_store as table_store
468
+ fid = str(folder_id or "").strip()
469
+ if not fid or not isinstance(doc, dict):
470
+ return None
471
+ for who, ws in doc.items():
472
+ if who in table_store.RESERVED_MEMBERS or not isinstance(ws, dict):
473
+ continue
474
+ if owner and str(who).strip().lower() != owner:
475
+ continue
476
+ rows = (ws.get("folders") or {}).get("views") or []
477
+ if not any(isinstance(f, dict) and str(f.get("id") or "") == fid for f in rows):
478
+ continue
479
+ # `itemFolders` is keyed by surface first: the views map sits under 'views'.
480
+ placed = (ws.get("itemFolders") or {}).get("views") or {}
481
+ placed = placed if isinstance(placed, dict) else {}
482
+ views = ws.get("views") or {}
483
+ return str(who), [str(vid) for vid, view in views.items()
484
+ if isinstance(view, dict) and str(placed.get(str(vid)) or "") == fid]
485
+ return None
486
+
487
+
488
+ def _granted_view_sources(session, docs=None):
489
+ """`{view_id: owner}` for grants this caller may read, from one registry read per kind.
490
 
491
  `owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
492
  source lookup for that case; modern grants point directly at the canonical personal stratum.
493
+ `docs` is `{database: projected workspace doc}` the caller already holds; the folder leg
494
+ reads views from it, else from `TableStore.find_folder` per visible database.
495
  """
496
  import core.shares as shares
497
  try:
 
504
  # add a View to "shared with me"; a valid named or everyone grant did.
505
  if shares.role_from(record, session.uname) is not None:
506
  out[str(raw_id)] = owner
 
507
  except Exception:
508
  return {}
509
+ # Folder leg (the `grid_events._granted_views` D-37 rule): a View filed in a folder shared
510
+ # with the caller is shared with the caller. Direct grants win, so this leg only fills gaps.
511
+ # A failure here drops the folder leg alone and keeps the direct answer: narrower, never wider.
512
+ try:
513
+ folders = {}
514
+ for raw_fid, record in shares.grants_bulk("folder", st=session.runtime).items():
515
+ owner = str(record.get("owner") or "").strip().lower() or None
516
+ if owner == session.uname:
517
+ continue
518
+ if shares.role_from(record, session.uname) in ("view", "edit"):
519
+ folders[str(raw_fid)] = owner
520
+ if not folders:
521
+ return out
522
+ hits = []
523
+ if docs is not None:
524
+ for doc in docs.values():
525
+ for fid, owner in folders.items():
526
+ hits.append(_folder_views_in(doc, fid, owner))
527
+ else:
528
+ import core.table_store as table_store
529
+ import core.view_templates as view_templates
530
+ keys, _ = _visible_database_keys(session)
531
+ for key in sorted(keys):
532
+ ws_key = view_templates.workspace_key(key)
533
+ if not ws_key:
534
+ continue
535
+ tops = table_store.make(ws_key, st=session.runtime)
536
+ for fid in folders:
537
+ found = tops.find_folder(fid)
538
+ hits.append((found[0], list(found[2])) if found else None)
539
+ for hit in hits:
540
+ if not hit or str(hit[0]).strip().lower() == session.uname:
541
+ continue
542
+ for vid in hit[1]:
543
+ out.setdefault(str(vid), hit[0])
544
+ except Exception:
545
+ pass
546
+ return out
547
 
548
 
549
  def _granted_view_ids(session):
 
596
  #: of the reserved words below is still a second connection, and its sibling must still be named.
597
  #:
598
  #: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
599
+ #: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
600
+ #: empty into the one Custom group (S79-T60, key `CUSTOM_SOURCE_GROUP`). A connection someone named
601
  #: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
602
  #: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
603
  _RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
604
 
605
+ #: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "Custom";
606
+ #: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
607
+ CUSTOM_SOURCE_GROUP = "custom"
608
+ #: Native provenance tokens, the legacy "Loopable" from `user_tables` included, fold into the key.
609
+ _NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation"})
610
+
611
+
612
+ def _native_group(group):
613
+ """The `sourceGroup` to send: a native token becomes `CUSTOM_SOURCE_GROUP`, a connector keeps its name."""
614
+ text = str(group or "").strip()
615
+ return CUSTOM_SOURCE_GROUP if text.lower() in _NATIVE_GROUP_TOKENS else text
616
+
617
  #: The registry modules that ARE the designated default connection's data rather than a namespaced
618
  #: copy of it: `odoo_relational.namespaced_fields` says so in as many words ("`customer_link`
619
  #: targets the compiled registry module `customer_data`, which holds the DEFAULT connection's
 
789
  continue
790
  pages.append({"key": e["key"], "label": e["label"],
791
  "source": e.get("source") or "Blank",
792
+ "sourceGroup": _native_group(e.get("sourceGroup")), "chrome": "main"})
793
  except Exception:
794
  # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT.
795
  #
 
848
  _folder = _odoo_folder_label(key, _conn_labels, _conn_default)
849
  if _folder:
850
  page["sourceGroup"] = _folder
851
+ elif page.get("user_table") or source.strip().lower() in _NATIVE_GROUP_TOKENS:
852
+ page.setdefault("sourceGroup", CUSTOM_SOURCE_GROUP)
853
  # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
854
  #
855
  # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
 
1575
  wizard_kinds = {row["kind"] for row in application_registry.template_contracts(session=session)}
1576
  if template_kind not in wizard_kinds:
1577
  raise err(400, "bad_application", "application template kind is not recognized")
1578
+ import app_templates
1579
+ source = str(payload.get("sourceDatabase") or app_templates.source(template_kind) or "").strip()
1580
+ sourceless = template_kind in app_templates.BY_KIND and not app_templates.BY_KIND[template_kind]["sources"]
1581
+ if not source and not sourceless:
1582
  raise err(400, "bad_application", "sourceDatabase is required")
1583
+ fields = application_registry.chart_field_contracts(session, [source])[source] if source else []
1584
  if not session.runtime.available():
1585
  raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.")
1586
  try:
1587
  application = application_registry.create(
1588
  session.runtime, session.tenant, session.uname, payload,
1589
+ field_contracts={source: fields if isinstance(fields, list) else []} if source else {},
1590
+ allowed_databases={source} if source else set())
1591
  except PermissionError as exc:
1592
  raise err(403, "forbidden", str(exc))
1593
  except application_registry.ApplicationExistsError as exc:
api/routes_odoo_attachments.py CHANGED
@@ -1,28 +1,62 @@
1
  """Contract C5 (S78-T14): the delivery photo list and the photo bytes, behind the delivery read wall.
2
 
3
  The work lives in `odoo_attachments.py`. This router decides who may ask and shapes the response.
 
4
  Mounted by `main.py` through `_mount_router("routes_odoo_attachments")`.
5
  """
6
  from fastapi import APIRouter, Depends, Path
7
  from fastapi.responses import Response
8
 
9
  import odoo_attachments
10
- from deps import Session, require_session
11
 
12
  router = APIRouter(prefix="/api/v1")
13
 
14
  _MAX_ID = 2 ** 31 - 1
15
 
16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  @router.get("/odoo/deliveries/{picking_id}/photos")
18
- def delivery_photos(picking_id: int = Path(..., ge=1, le=_MAX_ID), session: Session = Depends(require_session)):
 
19
  """`[{id, name, mimetype, url}]`: every photo attachment of one delivery."""
20
- return odoo_attachments.list_photos(session, picking_id)
 
 
 
 
 
21
 
22
 
23
  @router.get("/odoo/attachments/{attachment_id}")
24
- def delivery_photo(attachment_id: int = Path(..., ge=1, le=_MAX_ID), session: Session = Depends(require_session)):
 
25
  """The photo bytes, streamed through and never stored."""
 
 
 
26
  body, mimetype, _name = odoo_attachments.read_photo(session, attachment_id)
27
  return Response(content=body, media_type=mimetype, headers={
28
  "Cache-Control": "private, max-age=300",
 
1
  """Contract C5 (S78-T14): the delivery photo list and the photo bytes, behind the delivery read wall.
2
 
3
  The work lives in `odoo_attachments.py`. This router decides who may ask and shapes the response.
4
+ Both routes admit the cookie or an external bearer, walled to the key's database scope (D-9926).
5
  Mounted by `main.py` through `_mount_router("routes_odoo_attachments")`.
6
  """
7
  from fastapi import APIRouter, Depends, Path
8
  from fastapi.responses import Response
9
 
10
  import odoo_attachments
11
+ from deps import Session, is_bearer, require_session_or_bearer
12
 
13
  router = APIRouter(prefix="/api/v1")
14
 
15
  _MAX_ID = 2 ** 31 - 1
16
 
17
 
18
+ def _key_allowed(session):
19
+ """The delivery key filter for a bearer: its database scope. None for a cookie session."""
20
+ if not is_bearer(session):
21
+ return None
22
+ from routes_external import _assert_database_scope
23
+
24
+ def allowed(key):
25
+ _assert_database_scope(session, key)
26
+ return allowed
27
+
28
+
29
+ def _bearer_photo_wall(session, attachment_id, key_allowed):
30
+ """Check a bearer's database scope on the photo's delivery before `read_photo` reads bytes."""
31
+ rows = odoo_attachments._search_read(session, odoo_attachments.ATTACHMENT_MODEL,
32
+ [("id", "=", int(attachment_id))], ["res_model", "res_id"])
33
+ row = (rows or [None])[0]
34
+ res_id = row.get("res_id") if row else None
35
+ # A malformed attachment falls through to read_photo, which answers its own 404.
36
+ if (row and row.get("res_model") == odoo_attachments.PICKING_MODEL
37
+ and type(res_id) is int and res_id > 0):
38
+ odoo_attachments.require_delivery_read(session, res_id, key_allowed)
39
+
40
+
41
  @router.get("/odoo/deliveries/{picking_id}/photos")
42
+ def delivery_photos(picking_id: int = Path(..., ge=1, le=_MAX_ID),
43
+ session: Session = Depends(require_session_or_bearer)):
44
  """`[{id, name, mimetype, url}]`: every photo attachment of one delivery."""
45
+ key_allowed = _key_allowed(session)
46
+ if key_allowed is None:
47
+ return odoo_attachments.list_photos(session, picking_id)
48
+ from loopy_links import public_base
49
+ return odoo_attachments.list_photos(session, picking_id, key_allowed=key_allowed,
50
+ base=public_base() or "")
51
 
52
 
53
  @router.get("/odoo/attachments/{attachment_id}")
54
+ def delivery_photo(attachment_id: int = Path(..., ge=1, le=_MAX_ID),
55
+ session: Session = Depends(require_session_or_bearer)):
56
  """The photo bytes, streamed through and never stored."""
57
+ key_allowed = _key_allowed(session)
58
+ if key_allowed is not None:
59
+ _bearer_photo_wall(session, attachment_id, key_allowed)
60
  body, mimetype, _name = odoo_attachments.read_photo(session, attachment_id)
61
  return Response(content=body, media_type=mimetype, headers={
62
  "Cache-Control": "private, max-age=300",
api/routes_opportunity_finder.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Opportunity finder App routes (C3): rising terms, one item, status, a term's trend, settings, validate."""
2
+ from fastapi import APIRouter, Body, Depends, Query
3
+
4
+ from deps import Session, err, require_session
5
+
6
+ router = APIRouter(prefix="/api/v1/app-kinds/opportunity_finder", tags=["opportunity_finder"])
7
+ SOURCE = "ut_amzn_brand_analytics"
8
+ TEMPLATE_KIND = "opportunity_finder"
9
+ #: Weeks in a list row's sparkline. Each week is one store read per /screen call.
10
+ SPARK_WEEKS = 8
11
+ SETTING_NAMES = {"min_rise_pct": "min_rise_pct", "minRisePct": "min_rise_pct", "departments": "departments"}
12
+
13
+
14
+ def _module():
15
+ from modules import opportunity_finder
16
+ return opportunity_finder
17
+
18
+
19
+ def _reader(session: Session = Depends(require_session)) -> Session:
20
+ """Reads need read access to the Amazon brand analytics database."""
21
+ from core import perm_scope
22
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
23
+ raise err(403, "opportunity_finder_forbidden", "Your account cannot read Amazon brand analytics.")
24
+ return session
25
+
26
+
27
+ def _writer(session: Session = Depends(_reader)) -> Session:
28
+ """Writes also need a role that may edit."""
29
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
30
+ raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
31
+ return session
32
+
33
+
34
+ def _tenant(session):
35
+ return str(getattr(session.runtime, "key", "") or session.tenant)
36
+
37
+
38
+ def _user(session):
39
+ return {**session.user, "username": session.uname}
40
+
41
+
42
+ def _call(fn, *args, **kwargs):
43
+ try:
44
+ return fn(*args, **kwargs)
45
+ except PermissionError as exc:
46
+ raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.") from exc
47
+ except KeyError as exc:
48
+ raise err(404, "opportunity_finder_not_found", "That search term is not in the opportunity finder.") from exc
49
+ except ValueError as exc:
50
+ raise err(400, "opportunity_finder_invalid", str(exc)) from exc
51
+ except Exception as exc: # noqa: BLE001
52
+ raise err(503, "opportunity_finder_unavailable",
53
+ "The opportunity finder could not be read. Try again shortly.") from exc
54
+
55
+
56
+ def _app(session, app_id):
57
+ """The Opportunity finder App record this request names, readable by the caller (404 otherwise)."""
58
+ import application_registry
59
+ try:
60
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
61
+ except Exception as exc: # noqa: BLE001
62
+ raise err(503, "opportunity_finder_unavailable", "The App could not be read. Try again shortly.") from exc
63
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
64
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
65
+ raise err(404, "opportunity_finder_app_not_found", "That Opportunity finder App does not exist.")
66
+ return app
67
+
68
+
69
+ def _may_edit(session, app):
70
+ import application_registry
71
+ role = str(session.user.get("role") or "user").strip().lower()
72
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
73
+
74
+
75
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
76
+ """C3: a read names the App, and the caller must be able to read that App."""
77
+ _app(session, app)
78
+ return session
79
+
80
+
81
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
82
+ """C3: a write names the App, and only an editor of that App may change it."""
83
+ if not _may_edit(session, _app(session, app)):
84
+ raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
85
+ return session
86
+
87
+
88
+ def _mirror_opener(session):
89
+ """The typed `ut_amzn_*` door's cursor factory: tenant asserted, serving mirror version. The module closes it."""
90
+ import routes_odoo_tables
91
+ return lambda: routes_odoo_tables._table_cursor(session.runtime, SOURCE)
92
+
93
+
94
+ def _with_spark(of, st, items):
95
+ """Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first."""
96
+ weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
97
+ held = [(week, of.week_point(None, week, st=st).get("terms") or {}) for week in weeks]
98
+ for item in items:
99
+ term = of.norm(item.get("term"))
100
+ item["spark"] = [{"week": week, "rank": terms[term]} for week, terms in held if term in terms]
101
+ return items
102
+
103
+
104
+ @router.get("/screen")
105
+ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
106
+ cursor: str | None = Query(None), session: Session = Depends(_app_reader)):
107
+ """The rising list for the newest week, with counts, departments, settings and any named cause."""
108
+ of = _module()
109
+ record = _app(session, app)
110
+ st = session.runtime
111
+ listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st,
112
+ opener=_mirror_opener(session))
113
+ listed["items"] = _call(_with_spark, of, st, listed["items"])
114
+ return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"],
115
+ "weeks": listed["weeks"], "windows": listed["windows"], "periodType": listed["periodType"],
116
+ "departments": listed["departments"], "counts": listed["counts"],
117
+ "total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
118
+ "problems": listed["problems"], "statuses": list(of.STATUSES),
119
+ "settings": _call(of.settings, _tenant(session), st=st)}
120
+
121
+
122
+ @router.get("/items/{item_id}")
123
+ def item_detail(item_id: str, session: Session = Depends(_app_reader)):
124
+ of = _module()
125
+ return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
126
+ opener=_mirror_opener(session))
127
+
128
+
129
+ @router.patch("/items/{item_id}")
130
+ def update_item(item_id: str, status: str = Body(...), note: str | None = Body(None),
131
+ owner: str | None = Body(None), due: str | None = Body(None),
132
+ session: Session = Depends(_app_writer)):
133
+ of = _module()
134
+ return _call(of.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
135
+ due=due, st=session.runtime, opener=_mirror_opener(session))
136
+
137
+
138
+ @router.get("/terms/{item_id}")
139
+ def term_trend(item_id: str, session: Session = Depends(_app_reader)):
140
+ """A term's weekly rank trend and its newest top 3 clicked ASINs."""
141
+ of = _module()
142
+ return _call(of.term_detail, _tenant(session), item_id, st=session.runtime, opener=_mirror_opener(session))
143
+
144
+
145
+ @router.get("/settings")
146
+ def read_settings(session: Session = Depends(_app_reader)):
147
+ of = _module()
148
+ return _call(of.settings, _tenant(session), st=session.runtime)
149
+
150
+
151
+ @router.put("/settings")
152
+ def write_settings(body: dict = Body(...), session: Session = Depends(_app_writer)):
153
+ of = _module()
154
+ body = dict(body or {})
155
+ owner = body.pop("defaultOwner", None)
156
+ unknown = sorted(k for k in body if k not in SETTING_NAMES)
157
+ if unknown:
158
+ raise err(400, "opportunity_finder_invalid", "Unknown setting: " + ", ".join(unknown) + ".")
159
+ values = {SETTING_NAMES[k]: v for k, v in body.items()}
160
+ return _call(of.set_settings, _tenant(session), _user(session), default_owner=owner, st=session.runtime,
161
+ **values)
162
+
163
+
164
+ @router.get("/validate")
165
+ def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
166
+ session: Session = Depends(_app_reader)):
167
+ of = _module()
168
+ return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
169
+ opener=_mirror_opener(session))
api/routes_payouts.py ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Payouts App routes (C3): the overview, settlements with their waterfall and tie, owned items and settings.
2
+
3
+ Every route names the App (`?app=`) and checks the caller may read it and the settlements database.
4
+ Writes also need edit rights on that App and a role other than viewer. `main.py` does not mount this
5
+ router: `app_routes.mount_app_routers` does, from the App contract.
6
+ """
7
+ from fastapi import APIRouter, Body, Depends, HTTPException, Query
8
+
9
+ from deps import Session, err, require_session
10
+
11
+ router = APIRouter(prefix="/api/v1/app-kinds/payouts", tags=["payouts"])
12
+ SOURCE = "ut_amzn_settlements"
13
+ TEMPLATE_KIND = "payouts"
14
+
15
+
16
+ def _module():
17
+ from modules import payouts
18
+ return payouts
19
+
20
+
21
+ def _reader(session: Session = Depends(require_session)) -> Session:
22
+ """Reads need read access to the Amazon settlements database."""
23
+ from core import perm_scope
24
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
25
+ raise err(403, "payouts_forbidden", "Your account cannot read Amazon settlements.")
26
+ return session
27
+
28
+
29
+ def _writer(session: Session = Depends(_reader)) -> Session:
30
+ """Writes also need a role that may edit."""
31
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
32
+ raise err(403, "payouts_read_only", "Your account can view Payouts but not change it.")
33
+ return session
34
+
35
+
36
+ def _tenant(session):
37
+ return str(getattr(session.runtime, "key", "") or session.tenant)
38
+
39
+
40
+ def _app(session, app_id):
41
+ """The Payouts App record this request names, readable by the caller (404 otherwise)."""
42
+ import application_registry
43
+ try:
44
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
45
+ except Exception as exc: # noqa: BLE001
46
+ raise err(503, "payouts_unavailable", "The App could not be read. Try again shortly.") from exc
47
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
48
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
49
+ raise err(404, "payouts_app_not_found", "That Payouts App does not exist.")
50
+ return app
51
+
52
+
53
+ def _may_edit(session, app):
54
+ import application_registry
55
+ role = str(session.user.get("role") or "user").strip().lower()
56
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
57
+
58
+
59
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
60
+ """A read names the App the caller can read (404 first, so no App is confirmed), then the database."""
61
+ _app(session, app)
62
+ return _reader(session)
63
+
64
+
65
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
66
+ """A write names the App, and only an editor of that App may change it."""
67
+ found = _app(session, app)
68
+ _writer(_reader(session))
69
+ if not _may_edit(session, found):
70
+ raise err(403, "payouts_read_only", "You can view this App but not change it.")
71
+ return session
72
+
73
+
74
+ def _call(fn, *args, missing="payouts_not_found", missing_message="That settlement does not exist.", **kwargs):
75
+ pay = _module()
76
+ try:
77
+ return fn(*args, **kwargs)
78
+ except PermissionError as exc:
79
+ raise err(403, "payouts_read_only", str(exc)) from exc
80
+ except pay.SourceUnavailable as exc:
81
+ raise err(503, "payouts_no_source", str(exc)) from exc
82
+ except ValueError as exc:
83
+ raise err(400, "payouts_invalid", str(exc)) from exc
84
+ except KeyError as exc:
85
+ raise err(404, missing, missing_message) from exc
86
+ except Exception as exc: # noqa: BLE001
87
+ raise err(503, "payouts_unavailable", "Payouts could not be read. Try again shortly.") from exc
88
+
89
+
90
+ @router.get("/overview")
91
+ def read_overview(session: Session = Depends(_app_reader)):
92
+ pay = _module()
93
+ return _call(pay.overview, _tenant(session), st=session.runtime)
94
+
95
+
96
+ @router.get("/settlements")
97
+ def list_settlements(cursor: str | None = Query(None), limit: int = Query(200, ge=1),
98
+ currency: str | None = Query(None), start: str | None = Query(None),
99
+ end: str | None = Query(None), session: Session = Depends(_app_reader)):
100
+ pay = _module()
101
+ return _call(pay.settlements, _tenant(session), st=session.runtime, currency=currency, start=start, end=end,
102
+ cursor=cursor, limit=limit)
103
+
104
+
105
+ @router.get("/settlements/{settlement_id}")
106
+ def read_settlement(settlement_id: str, session: Session = Depends(_app_reader)):
107
+ pay = _module()
108
+ return _call(pay.settlement, _tenant(session), settlement_id, st=session.runtime)
109
+
110
+
111
+ @router.get("/items")
112
+ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
113
+ cursor: str | None = Query(None), limit: int = Query(200, ge=1),
114
+ session: Session = Depends(_app_reader)):
115
+ pay = _module()
116
+ listed = _call(pay.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
117
+ st=session.runtime)
118
+ return {**listed, "statuses": list(pay.STATUSES), "canEdit": _may_edit(session, _app(session, app))}
119
+
120
+
121
+ @router.patch("/items/{item_id}")
122
+ def update_item(item_id: str, status: str = Body(..., embed=True), note: str | None = Body(None, embed=True),
123
+ session: Session = Depends(_app_writer)):
124
+ pay = _module()
125
+ # History records the account name, never a display name.
126
+ actor = {**session.user, "username": session.uname}
127
+ return _call(pay.set_status, _tenant(session), actor, item_id, status, note=note, st=session.runtime,
128
+ missing="payouts_item_not_found", missing_message="That item does not exist.")
129
+
130
+
131
+ # --------------------------------------------------------------------------- screen (S79-T27)
132
+
133
+ #: The module's refusal for a workspace with no stored settlements; the screen reads it as empty.
134
+ EMPTY_SOURCE = "No Amazon settlements are stored"
135
+ SCREEN_PAGE = 50
136
+ LIST_FIELDS = ("id", "title", "currency", "start", "end", "depositDate", "amazonTotal", "linesTotal", "gap",
137
+ "reconciled")
138
+
139
+
140
+ def _empty(exc):
141
+ detail = getattr(exc, "detail", None)
142
+ error = detail.get("error") if isinstance(detail, dict) else None
143
+ return (isinstance(error, dict) and error.get("code") == "payouts_no_source"
144
+ and str(error.get("message") or "").startswith(EMPTY_SOURCE))
145
+
146
+
147
+ def _offset(cursor):
148
+ try:
149
+ offset = int(cursor or 0)
150
+ except (TypeError, ValueError):
151
+ offset = -1
152
+ if offset < 0:
153
+ raise err(400, "payouts_invalid", "The cursor is not valid. Start again from the first page.")
154
+ return offset
155
+
156
+
157
+ def _last_deposits(rows, as_of):
158
+ """The newest deposit on or before `as_of` and its change against the one before in its currency."""
159
+ paid = sorted((r for r in rows if r.get("depositDate") and r["depositDate"] <= as_of
160
+ and r.get("amazonTotal") is not None), key=lambda r: (r["depositDate"], r["id"]), reverse=True)
161
+ if not paid:
162
+ return None
163
+ last = paid[0]
164
+ prev = next((r for r in paid[1:] if r["currency"] == last["currency"]), None)
165
+ return {"settlementId": last["id"], "date": last["depositDate"], "amount": last["amazonTotal"],
166
+ "currency": last["currency"], "previousAmount": None if prev is None else prev["amazonTotal"],
167
+ "previousDate": None if prev is None else prev["depositDate"],
168
+ "change": None if prev is None else round(last["amazonTotal"] - prev["amazonTotal"], 2)}
169
+
170
+
171
+ @router.get("/screen")
172
+ def read_screen(app: str = Query(..., min_length=1), currency: str | None = Query(None),
173
+ view: str = Query("all", pattern="^(all|unreconciled)$"), cursor: str | None = Query(None),
174
+ limit: int = Query(SCREEN_PAGE, ge=1), session: Session = Depends(_app_reader)):
175
+ """The App screen in one read: the KPI strip, tab counts and one page of settlements, newest first. No cap."""
176
+ pay = _module()
177
+ offset = _offset(cursor)
178
+ tenant, st = _tenant(session), session.runtime
179
+ can_edit = _may_edit(session, _app(session, app))
180
+ base = {"canEdit": can_edit, "statuses": list(pay.STATUSES), "view": view, "currency": currency or None}
181
+ try:
182
+ over = _call(pay.overview, tenant, st=st)
183
+ except HTTPException as exc:
184
+ if not _empty(exc):
185
+ raise
186
+ return {**base, "empty": True, "currencies": [], "counts": {"all": 0, "unreconciled": 0},
187
+ "settlements": [], "cursor": None, "kpis": {}}
188
+ listed = _call(pay.settlements, tenant, st=st, currency=currency or None, limit=None)["settlements"]
189
+ counts = {"all": len(listed), "unreconciled": sum(1 for r in listed if not r["reconciled"])}
190
+ rows = listed if view == "all" else [r for r in listed if not r["reconciled"]]
191
+ page = rows[offset:offset + limit]
192
+ opened = over.get("openSettlements") or []
193
+ open_now = next((o for o in opened if currency and o.get("currency") == currency.strip().upper()),
194
+ opened[0] if opened else None)
195
+ return {**base, "empty": not over.get("settlementCount"), "asOf": over.get("asOf"),
196
+ "currencies": over.get("currencies") or [], "economicsCurrency": over.get("economicsCurrency"),
197
+ "counts": counts, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page],
198
+ "cursor": str(offset + limit) if offset + limit < len(rows) else None,
199
+ "kpis": {"nextDeposit": over.get("nextDeposit"), "openSettlement": open_now,
200
+ "lastDeposit": _last_deposits(listed, over.get("asOf") or ""),
201
+ "unreconciled": counts["unreconciled"], "settlements": counts["all"]}}
202
+
203
+
204
+ @router.get("/screen/settlements/{settlement_id}")
205
+ def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
206
+ session: Session = Depends(_app_reader)):
207
+ """One settlement for the drawer: its waterfall, the tie line and, when unreconciled, its owned item."""
208
+ pay = _module()
209
+ tenant, st = _tenant(session), session.runtime
210
+ can_edit = _may_edit(session, _app(session, app))
211
+ detail = _call(pay.settlement, tenant, settlement_id, st=st)
212
+ item = None
213
+ if not detail.get("reconciled"):
214
+ held = _call(pay.items, tenant, session.user, st=st, limit=None)["items"]
215
+ item = next((i for i in held if str(i.get("id")) == str(settlement_id)), None)
216
+ return {"canEdit": can_edit, "statuses": list(pay.STATUSES), "settlement": detail, "item": item}
217
+
218
+
219
+ @router.get("/settings")
220
+ def read_settings(session: Session = Depends(_app_reader)):
221
+ pay = _module()
222
+ return _call(pay.settings, _tenant(session), st=session.runtime)
223
+
224
+
225
+ @router.patch("/settings")
226
+ def write_settings(defaultOwner: str | None = Body(None, embed=True), session: Session = Depends(_app_writer)):
227
+ pay = _module()
228
+ return _call(pay.set_settings, _tenant(session), default_owner=defaultOwner, st=session.runtime)
api/routes_products.py CHANGED
@@ -39,6 +39,8 @@ router = APIRouter(prefix="/api/v1")
39
  MODULE = "product_data"
40
 
41
  _CACHE_TTL = 900
 
 
42
 
43
 
44
  def _pool_for(rt, team_id):
@@ -67,23 +69,10 @@ def _pool_for(rt, team_id):
67
  # the pool a layer below. verify_api's W16 section now exercises the real call.
68
  key = ("product_pool", team_id)
69
 
70
- # ⭐ D-10 (wave 24) — THE PRODUCT POOL HONOURS THE CONNECTOR PAUSE. It did not, and that was
71
- # a shipped defect on a live surface: the CUSTOMER pool got this guard at DEBT-2 and its
72
- # sibling — written from the same template, four files away — never did. So pausing Odoo
73
- # froze the Customer grid and the Product grid kept reading live, with the Settings toggle
74
- # reporting success. Sibling caches diverge exactly this way and nothing greps for
75
- # "the other one".
76
- #
77
- # ⚠ NO SNAPSHOT LEG, and that is the honest difference from the customer path rather than an
78
- # omission: `save_pool_snapshots` persists the `odoo_pool` scopes only — there is no product
79
- # snapshot bucket to serve. So this serves the in-process copy AT ANY AGE, and answers 503
80
- # naming the pause when there is none. Minting a product snapshot here would be a new
81
- # persistence format at a wave tail; refusing with a sentence is the answer the user can act
82
- # on, and it is the same 503 the customer path already gives for an unsnapshotted scope.
83
  # ⭐ WAVE 29 / R12 — THE POOL IS THE CATALOGUE NOW (5,875 active products, not the 2,717 that
84
  # happened to sell in a revenue window). Two consequences that land HERE rather than in
85
  # `product_data`:
86
- # · the 503 below now also covers a TRUNCATED catalogue read. `modules.product_data.
87
  # _catalogue_by_code` RAISES instead of degrading, precisely so a short pull arrives as a
88
  # refusal a user can act on rather than as a plausible smaller grid nobody questions;
89
  # `pool_unavailable` names the cause in its message.
@@ -91,28 +80,37 @@ def _pool_for(rt, team_id):
91
  # which the catalogue read is ~3 s; a BU-scoped build is ~9 s). Only a scope's FIRST-ever
92
  # build blocks — the stale-while-refresh discipline above is what keeps that off the
93
  # request path afterwards.
 
94
  import routes_keychain
95
  if routes_keychain.odoo_paused(rt):
96
  hit = rt.pool_cache.get(key)
97
  if hit:
98
  return hit[1]
99
- raise err(503, "connector_paused",
100
- "this data source is paused and nothing has been read since; "
101
- "an admin can resume it under Settings → Connectors")
102
 
103
  def build():
104
  # Preserve the long-standing pool call shape when no raw field is enabled. Besides
105
  # keeping existing adapters valid, this avoids paying for an unnecessary `extra_fields`
106
  # branch on the overwhelmingly common/default-off path.
107
  from harness import canonical_source, runtime as _runtime
 
108
  # Stock, sales and product reads come from the Odoo mirror; an unsynced mirror raises.
109
  with canonical_source.mirrored(lambda: _runtime.mirror_cursor(rt).cursor()):
110
  rows = (pd.pool(team_id=team_id,
111
  raw_fields=[field.get("rawName") for field in enabled_raw])
112
  if enabled_raw else pd.pool(team_id=team_id))
113
- return _project_enabled_raw_rows(rows, enabled_raw, drop_source=True)
114
-
115
- return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build)
 
 
 
 
 
 
 
 
 
 
116
 
117
 
118
  def scoped_pool(session: Session, *, with_rows: bool = True):
 
39
  MODULE = "product_data"
40
 
41
  _CACHE_TTL = 900
42
+ #: A product pool request slower than this logs its wait, so a cold scope is measurable on staging.
43
+ _SLOW_POOL_S = 5.0
44
 
45
 
46
  def _pool_for(rt, team_id):
 
69
  # the pool a layer below. verify_api's W16 section now exercises the real call.
70
  key = ("product_pool", team_id)
71
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  # ⭐ WAVE 29 / R12 — THE POOL IS THE CATALOGUE NOW (5,875 active products, not the 2,717 that
73
  # happened to sell in a revenue window). Two consequences that land HERE rather than in
74
  # `product_data`:
75
+ # · the route's 503 also covers a TRUNCATED catalogue read. `modules.product_data.
76
  # _catalogue_by_code` RAISES instead of degrading, precisely so a short pull arrives as a
77
  # refusal a user can act on rather than as a plausible smaller grid nobody questions;
78
  # `pool_unavailable` names the cause in its message.
 
80
  # which the catalogue read is ~3 s; a BU-scoped build is ~9 s). Only a scope's FIRST-ever
81
  # build blocks — the stale-while-refresh discipline above is what keeps that off the
82
  # request path afterwards.
83
+ # A pause serves the cached copy at any age, else builds from the mirror. It never reads Odoo.
84
  import routes_keychain
85
  if routes_keychain.odoo_paused(rt):
86
  hit = rt.pool_cache.get(key)
87
  if hit:
88
  return hit[1]
 
 
 
89
 
90
  def build():
91
  # Preserve the long-standing pool call shape when no raw field is enabled. Besides
92
  # keeping existing adapters valid, this avoids paying for an unnecessary `extra_fields`
93
  # branch on the overwhelmingly common/default-off path.
94
  from harness import canonical_source, runtime as _runtime
95
+ started = time.perf_counter()
96
  # Stock, sales and product reads come from the Odoo mirror; an unsynced mirror raises.
97
  with canonical_source.mirrored(lambda: _runtime.mirror_cursor(rt).cursor()):
98
  rows = (pd.pool(team_id=team_id,
99
  raw_fields=[field.get("rawName") for field in enabled_raw])
100
  if enabled_raw else pd.pool(team_id=team_id))
101
+ projected = _project_enabled_raw_rows(rows, enabled_raw, drop_source=True)
102
+ # The cold path's own receipt in the Space log: the build alone, apart from any wait.
103
+ print(f"[aios-api] product pool built team={team_id} rows={len(projected)} "
104
+ f"in {time.perf_counter() - started:.1f}s", flush=True)
105
+ return projected
106
+
107
+ asked = time.perf_counter()
108
+ served = scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build)
109
+ waited = time.perf_counter() - asked
110
+ if waited >= _SLOW_POOL_S:
111
+ # A wait far past the build line above is another build of this scope holding the lock.
112
+ print(f"[aios-api] product pool served team={team_id} after {waited:.1f}s", flush=True)
113
+ return served
114
 
115
 
116
  def scoped_pool(session: Session, *, with_rows: bool = True):
api/routes_query.py CHANGED
@@ -1324,7 +1324,9 @@ def _request_model(messages, tools, requested, session, chosen=None, *, before_a
1324
  turn_audit.requested(turn.get("handle"), active["name"], active["model"])
1325
  try:
1326
  response = provider_stream.post(request, provider=active, post=requests.post)
 
1327
  except Exception:
 
1328
  if priced:
1329
  after_attempt(ticket, None)
1330
  raise
 
1324
  turn_audit.requested(turn.get("handle"), active["name"], active["model"])
1325
  try:
1326
  response = provider_stream.post(request, provider=active, post=requests.post)
1327
+ prov.llm_breaker_record(active["name"], response.status_code)
1328
  except Exception:
1329
+ prov.llm_breaker_record(active["name"], 0)
1330
  if priced:
1331
  after_attempt(ticket, None)
1332
  raise
api/routes_ranking_diagnostic.py ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Ranking diagnostic App routes (C3): drop items, one item's cause card, status, ASIN picker, trend, settings, validate."""
2
+ from fastapi import APIRouter, Body, Depends, Query
3
+
4
+ from deps import Session, err, require_session
5
+
6
+ router = APIRouter(prefix="/api/v1/app-kinds/ranking_diagnostic", tags=["ranking_diagnostic"])
7
+ SOURCE = "ut_amzn_pricing"
8
+ TEMPLATE_KIND = "ranking_diagnostic"
9
+
10
+
11
+ def _module():
12
+ from modules import ranking_diagnostic
13
+ return ranking_diagnostic
14
+
15
+
16
+ def _reader(session: Session = Depends(require_session)) -> Session:
17
+ """Reads need read access to the Amazon Sales and Traffic database."""
18
+ from core import perm_scope
19
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
20
+ raise err(403, "ranking_diagnostic_forbidden", "Your account cannot read Amazon sales and traffic.")
21
+ return session
22
+
23
+
24
+ def _writer(session: Session = Depends(_reader)) -> Session:
25
+ """Writes also need a role that may edit."""
26
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
27
+ raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
28
+ return session
29
+
30
+
31
+ def _tenant(session):
32
+ return str(getattr(session.runtime, "key", "") or session.tenant)
33
+
34
+
35
+ def _user(session):
36
+ return {**session.user, "username": session.uname}
37
+
38
+
39
+ def _call(fn, *args, **kwargs):
40
+ try:
41
+ return fn(*args, **kwargs)
42
+ except PermissionError as exc:
43
+ raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.") from exc
44
+ except ValueError as exc:
45
+ raise err(400, "ranking_diagnostic_invalid", str(exc)) from exc
46
+ except KeyError as exc:
47
+ raise err(404, "ranking_diagnostic_not_found", "That item or ASIN is not in the history.") from exc
48
+ except Exception as exc: # noqa: BLE001
49
+ raise err(503, "ranking_diagnostic_unavailable",
50
+ "The ranking diagnostic could not be read. Try again shortly.") from exc
51
+
52
+
53
+ def _app(session, app_id):
54
+ """The Ranking diagnostic App record this request names, readable by the caller (404 otherwise)."""
55
+ import application_registry
56
+ try:
57
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
58
+ except Exception as exc: # noqa: BLE001
59
+ raise err(503, "ranking_diagnostic_unavailable", "The App could not be read. Try again shortly.") from exc
60
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
61
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
62
+ raise err(404, "ranking_diagnostic_app_not_found", "That Ranking diagnostic App does not exist.")
63
+ return app
64
+
65
+
66
+ def _may_edit(session, app):
67
+ import application_registry
68
+ role = str(session.user.get("role") or "user").strip().lower()
69
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
70
+
71
+
72
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
73
+ """C3: a read names the App, and the caller must be able to read that App."""
74
+ _app(session, app)
75
+ return session
76
+
77
+
78
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
79
+ """C3: a write names the App, and only an editor of that App may change it."""
80
+ if not _may_edit(session, _app(session, app)):
81
+ raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
82
+ return session
83
+
84
+
85
+ @router.get("/screen")
86
+ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
87
+ week: str | None = Query(None), cursor: str | None = Query(None),
88
+ session: Session = Depends(_app_reader)):
89
+ """The week's drop list (the newest week by default), with counts, problems and settings."""
90
+ rd = _module()
91
+ record = _app(session, app)
92
+ st = session.runtime
93
+ held = _call(rd.weeks, _tenant(session), st=st)
94
+ listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor,
95
+ week=week or (held[-1] if held else None), st=st)
96
+ return {"canEdit": _may_edit(session, record), "week": listed["week"], "weeks": listed["weeks"],
97
+ "counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
98
+ "problems": listed["problems"], "statuses": list(rd.STATUSES),
99
+ "settings": _call(rd.settings, _tenant(session), st=st)}
100
+
101
+
102
+ @router.get("/items/{item_id}")
103
+ def item_detail(item_id: str, session: Session = Depends(_app_reader)):
104
+ rd = _module()
105
+ return _call(rd.item_detail, _tenant(session), session.uname, item_id, st=session.runtime)
106
+
107
+
108
+ @router.patch("/items/{item_id}")
109
+ def update_item(item_id: str, status: str = Body(...), note: str | None = Body(None),
110
+ owner: str | None = Body(None), due: str | None = Body(None),
111
+ session: Session = Depends(_app_writer)):
112
+ rd = _module()
113
+ return _call(rd.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
114
+ due=due, st=session.runtime)
115
+
116
+
117
+ @router.get("/asins")
118
+ def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None),
119
+ session: Session = Depends(_app_reader)):
120
+ """The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
121
+ rd = _module()
122
+ page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, st=session.runtime)
123
+ labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
124
+ for row in page["asins"]:
125
+ flagged = bool(row.get("cause") and page["week"])
126
+ row["causeLabel"] = labels.get(row["cause"]) if flagged else None
127
+ row["itemId"] = rd.item_id(page["week"], row["asin"]) if flagged else None
128
+ return page
129
+
130
+
131
+ @router.get("/trend/{asin}")
132
+ def asin_trend(asin: str, session: Session = Depends(_app_reader)):
133
+ rd = _module()
134
+ return _call(rd.trend, _tenant(session), asin, st=session.runtime)
135
+
136
+
137
+ @router.get("/settings")
138
+ def read_settings(session: Session = Depends(_app_reader)):
139
+ rd = _module()
140
+ return _call(rd.settings, _tenant(session), st=session.runtime)
141
+
142
+
143
+ @router.put("/settings")
144
+ def write_settings(body: dict = Body(...), session: Session = Depends(_app_writer)):
145
+ rd = _module()
146
+ body = dict(body or {})
147
+ owner = body.pop("defaultOwner", None)
148
+ unknown = sorted(k for k in body if k not in rd.DEFAULTS)
149
+ if unknown:
150
+ raise err(400, "ranking_diagnostic_invalid", "Unknown setting: " + ", ".join(unknown) + ".")
151
+ return _call(rd.set_settings, _tenant(session), _user(session), default_owner=owner, st=session.runtime, **body)
152
+
153
+
154
+ @router.get("/validate")
155
+ def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
156
+ session: Session = Depends(_app_reader)):
157
+ rd = _module()
158
+ return _call(rd.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime)
api/routes_reimbursements.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Reimbursements App routes (C3): claim items, one item with its case text, status, paid rows, validate."""
2
+ from fastapi import APIRouter, Body, Depends, Query
3
+
4
+ from deps import Session, err, require_session
5
+
6
+ router = APIRouter(prefix="/api/v1/app-kinds/reimbursements", tags=["reimbursements"])
7
+ SOURCE = "ut_amzn_reimbursements"
8
+ TEMPLATE_KIND = "reimbursements"
9
+
10
+
11
+ def _module():
12
+ from modules import reimbursements
13
+ return reimbursements
14
+
15
+
16
+ def _reader(session: Session = Depends(require_session)) -> Session:
17
+ """Reads need read access to the Amazon reimbursements database."""
18
+ from core import perm_scope
19
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
20
+ raise err(403, "reimbursements_forbidden", "Your account cannot read Amazon reimbursements.")
21
+ return session
22
+
23
+
24
+ def _writer(session: Session = Depends(_reader)) -> Session:
25
+ """Writes also need a role that may edit."""
26
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
27
+ raise err(403, "reimbursements_read_only", "Your account can view reimbursements but not change them.")
28
+ return session
29
+
30
+
31
+ def _tenant(session):
32
+ return str(getattr(session.runtime, "key", "") or session.tenant)
33
+
34
+
35
+ def _user(session):
36
+ return {**session.user, "username": session.uname}
37
+
38
+
39
+ def _call(fn, *args, **kwargs):
40
+ try:
41
+ return fn(*args, **kwargs)
42
+ except PermissionError as exc:
43
+ raise err(403, "reimbursements_read_only", "You can view this App but not change it.") from exc
44
+ except ValueError as exc:
45
+ raise err(400, "reimbursements_invalid", str(exc)) from exc
46
+ except KeyError as exc:
47
+ raise err(404, "reimbursements_item_not_found", "That claim does not exist.") from exc
48
+ except Exception as exc: # noqa: BLE001
49
+ raise err(503, "reimbursements_unavailable", "Reimbursements could not be read. Try again shortly.") from exc
50
+
51
+
52
+ def _app(session, app_id):
53
+ """The Reimbursements App record this request names, readable by the caller (404 otherwise)."""
54
+ import application_registry
55
+ try:
56
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
57
+ except Exception as exc: # noqa: BLE001
58
+ raise err(503, "reimbursements_unavailable", "The App could not be read. Try again shortly.") from exc
59
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
60
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
61
+ raise err(404, "reimbursements_app_not_found", "That Reimbursements App does not exist.")
62
+ return app
63
+
64
+
65
+ def _may_edit(session, app):
66
+ import application_registry
67
+ role = str(session.user.get("role") or "user").strip().lower()
68
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
69
+
70
+
71
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
72
+ """C4: a read names the App, and the caller must be able to read that App."""
73
+ _app(session, app)
74
+ return session
75
+
76
+
77
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
78
+ """C4: a write names the App, and only an editor of that App may change it."""
79
+ if not _may_edit(session, _app(session, app)):
80
+ raise err(403, "reimbursements_read_only", "You can view this App but not change it.")
81
+ return session
82
+
83
+
84
+ @router.get("/screen")
85
+ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
86
+ cursor: str | None = Query(None), session: Session = Depends(_app_reader)):
87
+ rb = _module()
88
+ record = _app(session, app)
89
+ listed = _call(rb.items, _tenant(session), session.uname, status=status, cursor=cursor, st=session.runtime)
90
+ return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
91
+ "cursor": listed["cursor"], "problems": listed["problems"], "statuses": list(rb.STATUSES)}
92
+
93
+
94
+ @router.get("/items/{item_id}")
95
+ def item_detail(item_id: str, session: Session = Depends(_app_reader)):
96
+ rb = _module()
97
+ return _call(rb.item_detail, _tenant(session), session.uname, item_id, st=session.runtime)
98
+
99
+
100
+ @router.patch("/items/{item_id}")
101
+ def update_item(item_id: str, status: str = Body(...), caseId: str | None = Body(None),
102
+ note: str | None = Body(None), session: Session = Depends(_app_writer)):
103
+ rb = _module()
104
+ return _call(rb.set_status, _tenant(session), _user(session), item_id, status, note=note, case_id=caseId,
105
+ st=session.runtime)
106
+
107
+
108
+ @router.get("/paid")
109
+ def paid_rows(cursor: str | None = Query(None), session: Session = Depends(_app_reader)):
110
+ rb = _module()
111
+ return _call(rb.paid, _tenant(session), session.uname, cursor=cursor, st=session.runtime)
112
+
113
+
114
+ @router.get("/validate")
115
+ def validate(start: str | None = Query(None), end: str | None = Query(None),
116
+ session: Session = Depends(_app_reader)):
117
+ rb = _module()
118
+ return _call(rb.validate, _tenant(session), start=start, end=end, st=session.runtime)
api/routes_restock_planner.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Restock planner App routes (C3): items, one SKU, settings, refresh and the rule 8 tie."""
2
+ from fastapi import APIRouter, Body, Depends, Query
3
+
4
+ from deps import Session, err, require_session
5
+
6
+ TEMPLATE_KIND = "restock_planner"
7
+ router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["restock-planner"])
8
+ SOURCE = "ut_amzn_inventory_planning"
9
+
10
+
11
+ def _module():
12
+ from modules import restock_planner
13
+ return restock_planner
14
+
15
+
16
+ def _reader(session: Session = Depends(require_session)) -> Session:
17
+ """Reads need read access to the inventory planning database."""
18
+ from core import perm_scope
19
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
20
+ raise err(403, "restock_planner_forbidden", "Your account cannot read Amazon inventory planning.")
21
+ return session
22
+
23
+
24
+ def _writer(session: Session = Depends(_reader)) -> Session:
25
+ """Writes also need a role that may edit."""
26
+ if str(session.user.get("role") or "user").strip().lower() == "viewer":
27
+ raise err(403, "restock_planner_read_only", "You can view this App but not change it.")
28
+ return session
29
+
30
+
31
+ def _tenant(session):
32
+ return str(getattr(session.runtime, "key", "") or session.tenant)
33
+
34
+
35
+ def _call(fn, *args, **kwargs):
36
+ try:
37
+ return fn(*args, **kwargs)
38
+ except PermissionError as exc:
39
+ raise err(403, "restock_planner_read_only", "You can view this App but not change it.") from exc
40
+ except ValueError as exc:
41
+ raise err(400, "restock_planner_invalid", str(exc)) from exc
42
+ except KeyError as exc:
43
+ raise err(404, "restock_planner_item_not_found", "That restock item does not exist.") from exc
44
+ except Exception as exc: # noqa: BLE001
45
+ raise err(503, "restock_planner_unavailable", "The restock plan could not be read. Try again shortly.") from exc
46
+
47
+
48
+ def _app(session, app_id):
49
+ """The Restock planner App record this request names, readable by the caller (404 otherwise)."""
50
+ import application_registry
51
+ try:
52
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
53
+ except Exception as exc: # noqa: BLE001
54
+ raise err(503, "restock_planner_unavailable", "The App could not be read. Try again shortly.") from exc
55
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
56
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
57
+ raise err(404, "restock_planner_app_not_found", "That Restock planner App does not exist.")
58
+ return app
59
+
60
+
61
+ def _may_edit(session, app):
62
+ import application_registry
63
+ role = str(session.user.get("role") or "user").strip().lower()
64
+ return role != "viewer" and application_registry.may_edit(app, session.uname, is_admin=session.admin)
65
+
66
+
67
+ def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(_reader)) -> Session:
68
+ """A read names the App, and the caller must be able to read that App."""
69
+ _app(session, app)
70
+ return session
71
+
72
+
73
+ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(_writer)) -> Session:
74
+ """A write names the App, and only an editor of that App may change it."""
75
+ if not _may_edit(session, _app(session, app)):
76
+ raise err(403, "restock_planner_read_only", "You can view this App but not change it.")
77
+ return session
78
+
79
+
80
+ def owners(session):
81
+ """People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name."""
82
+ rp = _module()
83
+ try:
84
+ users = session.runtime.get("users")
85
+ except Exception: # noqa: BLE001
86
+ users = None
87
+ want = _tenant(session).strip().lower()
88
+ names = set()
89
+ for name, rec in (users.items() if isinstance(users, dict) else ()):
90
+ if (not isinstance(rec, dict) or not rec.get("active", True) or rp._is_service(name, rec)
91
+ or str(rec.get("role") or "").strip().lower() == "viewer"
92
+ or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
93
+ continue
94
+ names.add(str(rec.get("name") or name).strip())
95
+ return sorted((n for n in names if n), key=str.lower)
96
+
97
+
98
+ def thresholds(session):
99
+ """The two band thresholds in days of supply, read from the App settings (R49)."""
100
+ conf = _call(_module().settings, _tenant(session), st=session.runtime)
101
+ return {"sendToFbaDays": conf["sendToFbaDays"], "overstockDays": conf["overstockDays"]}
102
+
103
+
104
+ @router.get("/items")
105
+ def list_items(app: str = Query(..., min_length=1), band: str | None = Query(None),
106
+ status: str | None = Query(None), cursor: str | None = Query(None),
107
+ limit: int = Query(200, ge=1), q: str | None = Query(None),
108
+ session: Session = Depends(_app_reader)):
109
+ rp = _module()
110
+ listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit,
111
+ band=band, q=q, st=session.runtime)
112
+ return {**listed, "owners": owners(session), "thresholds": thresholds(session),
113
+ "canEdit": _may_edit(session, _app(session, app))}
114
+
115
+
116
+ @router.get("/items/{item_id}")
117
+ def read_item(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
118
+ rp = _module()
119
+ detail = _call(rp.item_detail, _tenant(session), item_id, st=session.runtime)
120
+ return {**detail, "thresholds": thresholds(session), "canEdit": _may_edit(session, _app(session, app))}
121
+
122
+
123
+ @router.patch("/items/{item_id}")
124
+ def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
125
+ due: str | None = Body(None), note: str | None = Body(None),
126
+ session: Session = Depends(_app_writer)):
127
+ rp = _module()
128
+ return _call(rp.update_item, _tenant(session), item_id, owner=owner, status=status, due=due, note=note,
129
+ user=session.uname or "unknown", st=session.runtime)
130
+
131
+
132
+ @router.get("/settings")
133
+ def read_settings(session: Session = Depends(_app_reader)):
134
+ rp = _module()
135
+ return _call(rp.settings, _tenant(session), st=session.runtime)
136
+
137
+
138
+ @router.patch("/settings")
139
+ def write_settings(sendToFbaDays: float | None = Body(None), overstockDays: float | None = Body(None),
140
+ dueDays: int | None = Body(None), defaultOwner: str | None = Body(None),
141
+ session: Session = Depends(_app_writer)):
142
+ rp = _module()
143
+ return _call(rp.set_settings, _tenant(session), send_to_fba_days=sendToFbaDays, overstock_days=overstockDays,
144
+ due_days=dueDays, default_owner=defaultOwner, st=session.runtime)
145
+
146
+
147
+ @router.post("/refresh")
148
+ def refresh(session: Session = Depends(_app_writer)):
149
+ rp = _module()
150
+ return _call(rp.refresh, _tenant(session), st=session.runtime)
151
+
152
+
153
+ @router.get("/validate")
154
+ def validate(session: Session = Depends(_app_reader)):
155
+ rp = _module()
156
+ return _call(rp.validate, _tenant(session), st=session.runtime)
api/routes_returns_repeat.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Returns and repeat App routes (C3): summary, SKU segments, SKU detail, owned items and settings.
2
+
3
+ Every endpoint names the App in its path. A caller who cannot read that App gets 404. Item writes
4
+ need a non-viewer account that may read FBA returns. Settings writes need an editor of the App.
5
+ """
6
+ from fastapi import APIRouter, Body, Depends, Path, Query
7
+
8
+ from deps import Session, err, require_session
9
+
10
+ TEMPLATE_KIND = "returns_repeat"
11
+ router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["returns-repeat"])
12
+ SOURCE = "ut_amzn_fba_customer_returns"
13
+
14
+
15
+ def _module():
16
+ from modules import returns_repeat
17
+ return returns_repeat
18
+
19
+
20
+ def _tenant(session):
21
+ return str(getattr(session.runtime, "key", "") or session.tenant)
22
+
23
+
24
+ def _call(fn, *args, **kwargs):
25
+ try:
26
+ return fn(*args, **kwargs)
27
+ except PermissionError as exc:
28
+ raise err(403, "returns_repeat_read_only", str(exc)) from exc
29
+ except ValueError as exc:
30
+ raise err(400, "returns_repeat_invalid", str(exc)) from exc
31
+ except KeyError as exc:
32
+ raise err(404, "returns_repeat_not_found", "That SKU or item does not exist.") from exc
33
+ except Exception as exc: # noqa: BLE001
34
+ raise err(503, "returns_repeat_unavailable", "Returns and repeat could not be read. Try again shortly.") from exc
35
+
36
+
37
+ def _app(session, app_id):
38
+ """The Returns and repeat App this request names, readable by the caller (404 otherwise)."""
39
+ import application_registry
40
+ try:
41
+ app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
42
+ except Exception as exc: # noqa: BLE001
43
+ raise err(503, "returns_repeat_unavailable", "The App could not be read. Try again shortly.") from exc
44
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
45
+ or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
46
+ raise err(404, "returns_repeat_app_not_found", "That Returns and repeat App does not exist.")
47
+ return app
48
+
49
+
50
+ def _viewer(session):
51
+ return str(session.user.get("role") or "user").strip().lower() == "viewer"
52
+
53
+
54
+ def _may_edit_app(session, app):
55
+ import application_registry
56
+ return not _viewer(session) and application_registry.may_edit(app, session.uname, is_admin=session.admin)
57
+
58
+
59
+ def _can_edit_items(session):
60
+ return _module().may_edit(session.runtime, session.user)
61
+
62
+
63
+ def _app_reader(app_id: str = Path(..., min_length=1), session: Session = Depends(require_session)) -> Session:
64
+ """A read names the App, and the caller must read that App and the FBA returns database."""
65
+ _app(session, app_id)
66
+ from core import perm_scope
67
+ if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
68
+ raise err(403, "returns_repeat_forbidden", "Your account cannot read Amazon FBA returns.")
69
+ return session
70
+
71
+
72
+ def _item_writer(session: Session = Depends(_app_reader)) -> Session:
73
+ """An item write also refuses a viewer."""
74
+ if _viewer(session) or not _can_edit_items(session):
75
+ raise err(403, "returns_repeat_read_only", "You can view this App but not change its items.")
76
+ return session
77
+
78
+
79
+ def _settings_writer(app_id: str = Path(..., min_length=1), session: Session = Depends(_app_reader)) -> Session:
80
+ """A settings write needs an editor of the App."""
81
+ if not _may_edit_app(session, _app(session, app_id)):
82
+ raise err(403, "returns_repeat_read_only", "You can view this App but not change its settings.")
83
+ return session
84
+
85
+
86
+ @router.get("/{app_id}/summary")
87
+ def read_summary(start: str | None = Query(None), end: str | None = Query(None),
88
+ session: Session = Depends(_app_reader)):
89
+ rr = _module()
90
+ out = _call(rr.summary, _tenant(session), start=start, end=end, st=session.runtime)
91
+ return {**out, "canEdit": _can_edit_items(session) and not _viewer(session)}
92
+
93
+
94
+ @router.get("/{app_id}/skus")
95
+ def list_skus(segment: str = Query("highest"), start: str | None = Query(None), end: str | None = Query(None),
96
+ cursor: str | None = Query(None), limit: int = Query(200, ge=1), q: str | None = Query(None),
97
+ session: Session = Depends(_app_reader)):
98
+ rr = _module()
99
+ return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
100
+ limit=limit, q=q, st=session.runtime)
101
+
102
+
103
+ @router.get("/{app_id}/sku")
104
+ def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(None),
105
+ end: str | None = Query(None), session: Session = Depends(_app_reader)):
106
+ rr = _module()
107
+ return _call(rr.detail, _tenant(session), key, start=start, end=end, st=session.runtime)
108
+
109
+
110
+ def owners(session):
111
+ """People who may own an item: active, non-service, non-viewer accounts of this tenant, by display name."""
112
+ rr = _module()
113
+ try:
114
+ users = session.runtime.get("users")
115
+ except Exception: # noqa: BLE001
116
+ users = None
117
+ want = _tenant(session).strip().lower()
118
+ names = set()
119
+ for name, rec in (users.items() if isinstance(users, dict) else ()):
120
+ if (not isinstance(rec, dict) or not rec.get("active", True) or rr._is_service(name, rec)
121
+ or str(rec.get("role") or "").strip().lower() == "viewer"
122
+ or (want and str(rec.get("tenant") or "royal-imports").strip().lower() != want)):
123
+ continue
124
+ names.add(str(rec.get("name") or name).strip())
125
+ return sorted((n for n in names if n), key=str.lower)
126
+
127
+
128
+ @router.get("/{app_id}/items")
129
+ def list_items(status: str | None = Query(None), cursor: str | None = Query(None),
130
+ limit: int = Query(200, ge=1), session: Session = Depends(_app_reader)):
131
+ rr = _module()
132
+ listed = _call(rr.items, _tenant(session), session.user, status=status, cursor=cursor, limit=limit,
133
+ st=session.runtime)
134
+ return {**listed, "statuses": list(rr.STATUSES), "owners": owners(session),
135
+ "canEdit": _can_edit_items(session) and not _viewer(session)}
136
+
137
+
138
+ @router.patch("/{app_id}/items/{item_id}")
139
+ def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
140
+ due: str | None = Body(None), note: str | None = Body(None),
141
+ session: Session = Depends(_item_writer)):
142
+ rr = _module()
143
+ return _call(rr.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
144
+ due=due, note=note, st=session.runtime)
145
+
146
+
147
+ @router.get("/{app_id}/settings")
148
+ def read_settings(app_id: str, session: Session = Depends(_app_reader)):
149
+ rr = _module()
150
+ out = _call(rr.settings, _tenant(session), st=session.runtime)
151
+ return {**out, "canEdit": _may_edit_app(session, _app(session, app_id))}
152
+
153
+
154
+ @router.put("/{app_id}/settings")
155
+ def write_settings(risingRatio: float | None = Body(None), risingMinPct: float | None = Body(None),
156
+ defaultOwner: str | None = Body(None), session: Session = Depends(_settings_writer)):
157
+ rr = _module()
158
+ return _call(rr.set_settings, _tenant(session), rising_ratio=risingRatio, rising_min_pct=risingMinPct,
159
+ default_owner=defaultOwner, st=session.runtime)
api/routes_shares.py CHANGED
@@ -1090,6 +1090,97 @@ def my_shares(session: Session = Depends(require_session)):
1090
  return shares.shared_with(session.uname, st=session.runtime)
1091
 
1092
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1093
  @router.get("/share/diagnostics/{reference}")
1094
  def get_view_share_diagnostic(reference: str, session: Session = Depends(require_session)):
1095
  """Read one support reference from this tenant's ring, after administrator admission."""
@@ -1240,6 +1331,18 @@ def _people(tenant):
1240
  return sorted(out, key=lambda p: p["name"].lower())
1241
 
1242
 
 
 
 
 
 
 
 
 
 
 
 
 
1243
  @router.put("/share/{kind}/{oid}")
1244
  def put_share(kind: str, oid: str, body: dict = Body(default=None),
1245
  session: Session = Depends(require_session)):
@@ -1277,6 +1380,9 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
1277
  # both answerable (*"every editable field has an owner"*), so the `field` arm prints
1278
  # them. ⛔ THE CODE WORD `not_owner` IS UNCHANGED — clients match on it — and the
1279
  # OTHER THREE KINDS KEEP THIS SENTENCE BYTE FOR BYTE, because their gates assert it.
 
 
 
1280
  if kind == "field":
1281
  table_key, field_key = shares.split_field_oid(oid)
1282
  raise err(403, "not_owner", _field_refusal(session, table_key, field_key))
 
1090
  return shares.shared_with(session.uname, st=session.runtime)
1091
 
1092
 
1093
+ def _name_resolver():
1094
+ """`username -> display name` from ONE registry read; falls back to the username."""
1095
+ try:
1096
+ registry = users.registry() or {}
1097
+ except Exception: # noqa: BLE001
1098
+ registry = {}
1099
+
1100
+ def name(username):
1101
+ who = str(username or "").strip()
1102
+ row = registry.get(who) if isinstance(registry, dict) else None
1103
+ return (str(row.get("name") or "").strip() if isinstance(row, dict) else "") or who
1104
+ return name
1105
+
1106
+
1107
+ def _audience_mark(owner, audience, everyone, viewer):
1108
+ """The mark an owner-plus-audience object gives `viewer`, or None. Same rule as `shares.mark_from`."""
1109
+ owner = str(owner or "").strip()
1110
+ named = [u for u in dict.fromkeys(str(u or "").strip() for u in audience or ()) if u]
1111
+ if owner and owner == viewer:
1112
+ others = [u for u in named if u != viewer]
1113
+ if not others and not everyone:
1114
+ return None
1115
+ return {"direction": "by_me", "count": len(others), "everyone": bool(everyone)}
1116
+ if everyone or viewer in named:
1117
+ return {"direction": "with_me", "by": owner}
1118
+ return None
1119
+
1120
+
1121
+ def _app_marks(session):
1122
+ """App marks from the App registry's own record: `createdBy` plus its `share`."""
1123
+ try:
1124
+ import application_registry as _apps
1125
+ rows = (_apps.load(session.runtime).get("applications") or {}).values()
1126
+ except Exception: # noqa: BLE001
1127
+ return {}
1128
+ out = {}
1129
+ for app in rows:
1130
+ if not isinstance(app, dict) or app.get("retired") is True or not app.get("id"):
1131
+ continue
1132
+ share = app.get("share") if isinstance(app.get("share"), dict) else {}
1133
+ mark = _audience_mark(app.get("createdBy"), share.get("users"),
1134
+ share.get("mode") == "tenant", session.uname)
1135
+ if mark:
1136
+ out[f"app:{app['id']}"] = mark
1137
+ return out
1138
+
1139
+
1140
+ def _agent_marks(session):
1141
+ """Agent marks from each Agent contract's `sharing`, read once."""
1142
+ try:
1143
+ import automation_engine as _engine
1144
+ contracts = _engine._agent_contracts(session.runtime)
1145
+ except Exception: # noqa: BLE001
1146
+ return {}
1147
+ out = {}
1148
+ for aid, rec in (contracts or {}).items():
1149
+ if not isinstance(rec, dict):
1150
+ continue
1151
+ sharing = rec.get("sharing") if isinstance(rec.get("sharing"), dict) else {}
1152
+ audience = sharing.get("sharedWith") if sharing.get("visibility") == "shared" else ()
1153
+ mark = _audience_mark(rec.get("createdBy"), audience, False, session.uname)
1154
+ if mark:
1155
+ out[f"agent:{aid}"] = mark
1156
+ return out
1157
+
1158
+
1159
+ @router.get("/share/marks")
1160
+ def get_share_marks(session: Session = Depends(require_session)):
1161
+ """`{marks: {"<kind>:<id>": mark}}`: every shared icon a module home draws, in ONE request.
1162
+
1163
+ A mark exists only for the owner (`by_me`, with `count`) or a grantee (`with_me`, with `by`),
1164
+ so no viewer learns of a share they are not part of. View keys are bare view ids, because the
1165
+ grant registry stores them bare. An unreadable grant bucket answers 503 and the client draws
1166
+ no icon. App and Agent marks drop out alone when their own record cannot be read.
1167
+ """
1168
+ held = shares.share_marks(session.uname, st=session.runtime)
1169
+ if held is None:
1170
+ raise err(503, "marks_unavailable", "Sharing could not be read. Reload to try again.")
1171
+ name = _name_resolver()
1172
+ marks = {}
1173
+ for kind, rows in held.items():
1174
+ for oid, mark in rows.items():
1175
+ marks[f"{kind}:{oid}"] = mark
1176
+ marks.update(_app_marks(session))
1177
+ marks.update(_agent_marks(session))
1178
+ for mark in marks.values():
1179
+ if mark["direction"] == "with_me":
1180
+ mark["by"] = name(mark.get("by")) if mark.get("by") else ""
1181
+ return {"marks": marks}
1182
+
1183
+
1184
  @router.get("/share/diagnostics/{reference}")
1185
  def get_view_share_diagnostic(reference: str, session: Session = Depends(require_session)):
1186
  """Read one support reference from this tenant's ring, after administrator admission."""
 
1331
  return sorted(out, key=lambda p: p["name"].lower())
1332
 
1333
 
1334
+ def _stranger(session, kind, oid):
1335
+ """True when the caller holds no role and cannot reach the object: `get_share`'s 404 test."""
1336
+ admin = bool(session.admin) and kind != "cohort"
1337
+ if kind == "field":
1338
+ table_key, field_key = shares.split_field_oid(oid)
1339
+ role = _field_permissions.field_role_sources(
1340
+ table_key, field_key, session.uname, is_admin=admin, st=session.runtime)["direct"]
1341
+ else:
1342
+ role = shares.role_for(kind, oid, session.uname, is_admin=admin, st=session.runtime)
1343
+ return role is None and not _can_see_object(session, kind, oid)
1344
+
1345
+
1346
  @router.put("/share/{kind}/{oid}")
1347
  def put_share(kind: str, oid: str, body: dict = Body(default=None),
1348
  session: Session = Depends(require_session)):
 
1380
  # both answerable (*"every editable field has an owner"*), so the `field` arm prints
1381
  # them. ⛔ THE CODE WORD `not_owner` IS UNCHANGED — clients match on it — and the
1382
  # OTHER THREE KINDS KEEP THIS SENTENCE BYTE FOR BYTE, because their gates assert it.
1383
+ # D-476: a caller with no relationship gets GET's 404, so this door confirms no id.
1384
+ if _stranger(session, kind, oid):
1385
+ raise err(404, "no_object", "no such item, or it is not shared with this account")
1386
  if kind == "field":
1387
  table_key, field_key = shares.split_field_oid(oid)
1388
  raise err(403, "not_owner", _field_refusal(session, table_key, field_key))
api/routes_sheet.py CHANGED
@@ -207,6 +207,142 @@ def _binding_catching_up(session, binding):
207
  return any(_database_catching_up(session, target) for target in targets)
208
 
209
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
210
  def _mirror(session: Session):
211
  """`(cursor, spec, columns)` for one read-through database, or the refusal that names why.
212
 
@@ -278,7 +414,7 @@ def _mirror(session: Session):
278
  session._sheet_amzn_cursor = (cur, None)
279
  else:
280
  cur, mismatch = (getattr(session, "_sheet_drill_cursor", None)
281
- or rot._mirror_cur(session))
282
  except RuntimeError as exc:
283
  raise err(503, "store_not_ready", str(exc))
284
  if mismatch:
@@ -350,6 +486,10 @@ def _mirror(session: Session):
350
  session._sheet_mirrors = memo
351
  slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str))
352
  want = None if needed is None else {str(k) for k in needed}
 
 
 
 
353
  entry = memo.get(slot)
354
  if entry is not None:
355
  built, made = entry
@@ -357,10 +497,10 @@ def _mirror(session: Session):
357
  return made
358
  if want is not None:
359
  want = set(want) | built
360
- made = _registry_mirror(session, cur, table_key, metric_window, needed=want)
361
  memo[slot] = (None if want is None else set(want), made)
362
  return made
363
- spec = rot._source_for(cur, table_key)
364
  if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
365
  spec = _amazon_stored_spec(cur, table_key)
366
  if spec is None:
@@ -3774,6 +3914,26 @@ REFRESH_KEEP_SECONDS = 3600
3774
  _REFRESH_FAILED = {"code": "sheet_unresolved", "message": "This workbook could not be refreshed."}
3775
  _REFRESH_TIMEOUT = {"code": "sheet_refresh_timeout",
3776
  "message": "This refresh took too long. Try again in a few minutes."}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3777
 
3778
 
3779
  def _refresh_desk_queues(session):
@@ -3857,36 +4017,78 @@ def _refresh_desk_enqueue(store, tenant, workbook, reader, args, now=None, path=
3857
  return made["row"]
3858
 
3859
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3860
  def _refresh_desk_claim(store, now=None):
3861
- """Mark the oldest queued request running and return it. None while one is already running."""
 
 
 
 
3862
  now = _desk_time.time() if now is None else now
3863
  claimed = {}
3864
 
3865
- def stale(row):
3866
- return row.get("state") == "running" and now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS
3867
-
3868
  # Read first: an idle poll must never write the store.
3869
  rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now).values()
3870
- busy = any(row.get("state") == "running" and not stale(row) for row in rows)
3871
- if not any(stale(row) for row in rows) and (busy or not any(row.get("state") == "queued" for row in rows)):
 
3872
  return None
3873
 
3874
  def apply(current):
3875
  rows = _refresh_desk_rows(current, now)
3876
- for row in rows.values():
3877
- if stale(row):
3878
- if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS:
3879
- row["state"] = "queued"
3880
- else:
3881
- row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now),
3882
- error=dict(_REFRESH_TIMEOUT))
3883
  if not any(row.get("state") == "running" for row in rows.values()):
3884
  queued = sorted((row for row in rows.values() if row.get("state") == "queued"),
3885
  key=_refresh_desk_order)
3886
  if queued:
3887
  row = queued[0]
3888
  row.update(state="running", startedTs=now, startedAt=_desk_iso(now),
3889
- attempts=int(row.get("attempts") or 0) + 1)
 
3890
  claimed["row"] = dict(row)
3891
  return {"requests": rows}
3892
 
@@ -3894,19 +4096,38 @@ def _refresh_desk_claim(store, now=None):
3894
  return claimed.get("row")
3895
 
3896
 
3897
- def _refresh_desk_finish(store, request_id, error=None, now=None):
3898
- """Mark one running request done, or failed with a public `{code, message}`."""
 
 
 
 
 
 
 
 
3899
  now = _desk_time.time() if now is None else now
 
 
 
 
 
 
 
 
3900
 
3901
  def apply(current):
3902
  rows = _refresh_desk_rows(current, now)
3903
- row = rows.get(request_id)
3904
- if row is not None and row.get("state") == "running":
3905
  row.update(state="failed" if error else "done", finishedTs=now,
3906
- finishedAt=_desk_iso(now), error=dict(error) if error else None)
 
 
3907
  return {"requests": rows}
3908
 
3909
  store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
 
3910
 
3911
 
3912
  def _refresh_desk_entry(store, workbook_id, reader, request_id):
@@ -3970,9 +4191,19 @@ def _refresh_desk_run(session, row):
3970
  return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
3971
  with _sheet_request_scope(session):
3972
  if args.get("kind") == "retry":
3973
- return _refresh_sheet_scoped(workbook, session, retry_id=args.get("retryId"))
3974
- only = set(args.get("cells") or ()) if args.get("kind") == "cells" else None
3975
- return _refresh_sheet_scoped(workbook, session, only=only)
 
 
 
 
 
 
 
 
 
 
3976
 
3977
 
3978
  # ── the opening cache: a repeat open of the same tab at the same state is served from memory ─────
@@ -4368,6 +4599,7 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
4368
  return hit["cell"]
4369
  return None
4370
 
 
4371
  started = perf_counter()
4372
  resolved_count = 0
4373
  held_refusal = None
@@ -5483,6 +5715,7 @@ _BINDING_REFUSAL_DISPLAY = {
5483
  "store_not_ready": "#UPDATING",
5484
  "mirror_not_ready": "#UPDATING",
5485
  "amazon_mirror_not_loaded": "#UPDATING",
 
5486
  }
5487
 
5488
 
@@ -5491,6 +5724,7 @@ _TRANSIENT_REFUSAL_CODES = frozenset((
5491
  "store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
5492
  "mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
5493
  "sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
 
5494
  ))
5495
 
5496
  #: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
@@ -5935,7 +6169,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
5935
  # and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below.
5936
  _refuse_unapplied_audience(session, database)
5937
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
5938
- if perm_scope.row_scope_applies(session.user, database) else None)
5939
  wall_nodes, wall_conj = _fe_parts(wall_tree)
5940
  scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
5941
  scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
@@ -6376,7 +6610,9 @@ def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
6376
  def view_rows(workbook_id: str, body: object = Body(default=None),
6377
  session: Session = Depends(require_session)):
6378
  """W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
 
6379
  from fastapi.responses import JSONResponse
 
6380
  from core import perm_scope, sheets, user_tables
6381
  from modules import sheet as sheet_mod
6382
 
@@ -6402,74 +6638,74 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
6402
 
6403
  # W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
6404
  _refuse_unapplied_audience(session, database)
6405
- wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
6406
- if perm_scope.row_scope_applies(session.user, database) else None)
6407
- wall_nodes, wall_conj = _fe_parts(wall_tree)
6408
- scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
6409
- scope_nodes = _rewrite_related_filters(session, database, scope_nodes)
6410
  # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
6411
  for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
6412
  if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
6413
  return JSONResponse(status_code=400, content={
6414
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6415
  today = _sheet_today(session)
 
 
6416
  try:
6417
- scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today)
6418
- window = _ending_window(session, binding, book=book, today=today)
6419
- needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
6420
- | sheet_mod.filter_columns_named(wall_nodes))
6421
- related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
6422
- cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
6423
- cols = dict(cols)
6424
- if spec.get("scope_applied"):
6425
- wall_nodes, wall_conj = [], "and"
6426
- related = related_bags(cur, spec, cols, window=window)
6427
- for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
6428
- if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
6429
- continue
6430
- if key in hidden or not (cols.get(key) or {}).get("sql"):
6431
- return JSONResponse(status_code=400, content={
6432
- "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6433
- # W62 C2: an unanswerable row wall must refuse before either SQL read can widen it.
6434
- wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
6435
- if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled):
6436
- raise ValueError("unapplied row wall")
6437
- count_sql, params, limits = sheet_mod.build_aggregate_query(
6438
- spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
6439
- scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
6440
- related=related, summary_default=False)
6441
- if limits:
6442
- raise ValueError("unapplied view filter")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6443
  except sheet_mod.SheetRefusal as exc:
6444
  return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
6445
  except ValueError:
6446
  raise err(409, "scope_not_applicable",
6447
  "This view uses a restriction that cannot be applied to this database's live connection.")
6448
-
6449
- # W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
6450
- total = (cur.execute(count_sql, params).fetchone() or [0])[0]
6451
- # ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
6452
- # A placed Grid over one of our own tables writes a record back, and the write door is Data's
6453
- # own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
6454
- # from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
6455
- # refreshes. So the id rides beside the values under the key `pid` while `fields` still names
6456
- # only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
6457
- # uses, so the identity here and the identity the drill lists can never be two different things.
6458
- id_expr = str(spec.get("id") or "").strip()
6459
- projection = ", ".join([*(cols[key]["sql"] for key in requested),
6460
- *([id_expr] if id_expr else [])])
6461
- rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
6462
- if binding["sort"]:
6463
- # W62 C2 / R30: expressions come only from resolved columns; directions are validated.
6464
- order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
6465
- for item in binding["sort"])
6466
- rows_sql += " ORDER BY " + order
6467
- rows_sql += f" LIMIT {binding['limit']}"
6468
- keys = [*requested, *(["pid"] if id_expr else [])]
6469
- rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
6470
  # W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
6471
  # W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
6472
  editable = user_tables.records_mutable(database, st=session.runtime)
 
 
 
 
6473
  return {"fields": [{"key": key, "label": visible[key].get("label") or key,
6474
  "kind": visible[key].get("type") or visible[key].get("kind") or "text"}
6475
  for key in requested],
@@ -6477,14 +6713,105 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
6477
  "viewType": binding["viewType"], "editable": bool(editable)}
6478
 
6479
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6480
  _DRILL_MIRRORS = OrderedDict()
 
6481
  _DRILL_MIRROR_LOCK = threading.RLock()
6482
  _DRILL_MIRROR_MAX = 6
 
 
6483
 
6484
 
6485
  @contextlib.contextmanager
6486
  def _drill_mirror_lock():
6487
- """Include shared-cursor contention in the caller's absolute request budget."""
6488
  from core import request_timing
6489
  timeout = request_timing.remaining_budget()
6490
  acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None
@@ -6501,6 +6828,38 @@ def _drill_mirror_lock():
6501
  _DRILL_MIRROR_LOCK.release()
6502
 
6503
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6504
  def _drill_tokens(session, databases):
6505
  """Unknown freshness disables reuse. No workspace documents are retained here."""
6506
  from core import sheet_binding
@@ -6519,7 +6878,7 @@ def _drill_tokens(session, databases):
6519
 
6520
  @contextlib.contextmanager
6521
  def _drill_mirror(session, database, window, needed, databases):
6522
- """A small reader-scoped LRU of query mirrors, with serialized cursor use and eviction."""
6523
  from core import sheet_binding, perm_scope
6524
  from core.request_timing import raise_if_resource_error
6525
  import datetime
@@ -6552,8 +6911,10 @@ def _drill_mirror(session, database, window, needed, databases):
6552
  "_sheet_drill_entries")
6553
  absent = object()
6554
  previous = {name: getattr(session, name, absent) for name in attrs}
6555
- with _drill_mirror_lock():
6556
- hit = _DRILL_MIRRORS.get(key) if token else None
 
 
6557
  session._sheet_drill_isolated = True
6558
  session._sheet_drill_cursor = hit["owned"] if hit else None
6559
  session._sheet_drill_entries = hit["mirrors"] if hit else {}
@@ -6565,21 +6926,30 @@ def _drill_mirror(session, database, window, needed, databases):
6565
  want = set(needed) | (hit["needed"] if hit else set())
6566
  if hit and want <= hit["needed"]:
6567
  made = hit["made"]
6568
- _DRILL_MIRRORS.move_to_end(key)
 
 
6569
  else:
6570
- _DRILL_MIRRORS.pop(key, None)
 
 
 
6571
  made = _mirror(session)(database, metric_window=window, needed=want)
6572
  owned = getattr(session, "_sheet_drill_cursor", None)
6573
  hit = {"made": made, "needed": want, "owned": owned,
6574
  "mirrors": session._sheet_drill_entries}
6575
  # A build racing a source change is usable for this request, never reusable.
6576
  if token and _drill_tokens(session, databases) == sources:
6577
- _DRILL_MIRRORS[key] = hit
6578
- _DRILL_MIRRORS.move_to_end(key)
6579
- while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
6580
- _, evicted = _DRILL_MIRRORS.popitem(last=False)
6581
- if evicted["owned"]:
6582
- evicted["owned"][0].close()
 
 
 
 
6583
  else:
6584
  token = None
6585
  # Related leaves may revisit the primary registry. Keep its table intact and widen
@@ -6589,8 +6959,16 @@ def _drill_mirror(session, database, window, needed, databases):
6589
  yield made, token
6590
  finally:
6591
  owned = getattr(session, "_sheet_drill_cursor", None)
6592
- if owned and not any(item["owned"] is owned for item in _DRILL_MIRRORS.values()):
6593
- owned[0].close()
 
 
 
 
 
 
 
 
6594
  for name, value in previous.items():
6595
  if value is absent:
6596
  session.__dict__.pop(name, None)
 
207
  return any(_database_catching_up(session, target) for target in targets)
208
 
209
 
210
+ #: Builds of one registry mirror per cell. A source write during a build costs a rebuild.
211
+ _MIRROR_BUILD_TRIES = 3
212
+
213
+
214
+ def _request_memo(session):
215
+ """The Sheet request memo, or None outside a request scope or inside an isolated drill."""
216
+ memo = getattr(session, "_sheet_read_memo", None)
217
+ if not isinstance(memo, dict) or getattr(session, "_sheet_drill_isolated", False):
218
+ return None
219
+ return memo
220
+
221
+
222
+ def _request_mirror_cur(session):
223
+ """One gated mirror cursor per Sheet request and mirror install generation."""
224
+ import routes_odoo_tables as rot
225
+ memo = _request_memo(session)
226
+ generation = _install_generation() if memo is not None else None
227
+ if generation is None:
228
+ return rot._mirror_cur(session)
229
+ hit = memo.get("mirror_cur")
230
+ if hit is not None and hit[0] == generation:
231
+ return hit[1]
232
+ got = rot._mirror_cur(session)
233
+ if got[1] is None:
234
+ memo["mirror_cur"] = (generation, got)
235
+ return got
236
+
237
+
238
+ def _request_source_for(session, cur, table_key):
239
+ """`rot._source_for` once per Sheet request, cursor and database. Its catalog reads cost per call."""
240
+ import routes_odoo_tables as rot
241
+ memo = _request_memo(session)
242
+ if memo is None:
243
+ return rot._source_for(cur, table_key)
244
+ slot = ("source_for", table_key)
245
+ hit = memo.get(slot)
246
+ if hit is not None and hit[0] is cur:
247
+ return hit[1]
248
+ spec = rot._source_for(cur, table_key)
249
+ if spec is not None:
250
+ memo[slot] = (cur, spec)
251
+ return spec
252
+
253
+
254
+ def _row_scope_applies(session, database):
255
+ """`perm_scope.row_scope_applies` once per Sheet request: its storeless form reads a revision."""
256
+ import core.perm_scope as perm_scope
257
+ memo = _request_memo(session)
258
+ slot = ("row_scope_applies", database)
259
+ if memo is not None and slot in memo:
260
+ return memo[slot]
261
+ applies = perm_scope.row_scope_applies(session.user, database)
262
+ if memo is not None:
263
+ memo[slot] = applies
264
+ return applies
265
+
266
+
267
+ def _registry_mirror_settled(session, cur, table_key, metric_window, needed):
268
+ """Rebuild when the source changed during the build. Only the last change refuses."""
269
+ from copy import deepcopy
270
+ from time import perf_counter
271
+ from core.request_timing import remaining_budget
272
+ slot = ("mirror_refused", table_key, json.dumps(metric_window, sort_keys=True, default=str))
273
+ memo = getattr(session, "_sheet_read_memo", None)
274
+ refused = memo if isinstance(memo, dict) else {}
275
+ if slot in refused:
276
+ # ⛔ One refused build per database and window per request: every further cell would rebuild.
277
+ status, detail = refused[slot]
278
+ raise HTTPException(status_code=status, detail=deepcopy(detail))
279
+ for attempt in range(_MIRROR_BUILD_TRIES):
280
+ started = perf_counter()
281
+ session._sheet_mirror_builds = getattr(session, "_sheet_mirror_builds", 0) + 1
282
+ try:
283
+ return _registry_mirror(session, cur, table_key, metric_window, needed=needed)
284
+ except HTTPException as exc:
285
+ detail = exc.detail if isinstance(exc.detail, dict) else {}
286
+ if (detail.get("error") or {}).get("code") != "drill_population_changed":
287
+ raise
288
+ spent = perf_counter() - started
289
+ moved = ",".join(getattr(exc, "moved", None) or ["unknown"])
290
+ left = remaining_budget()
291
+ # A rebuild that cannot finish inside the budget only turns a held cell into a timeout.
292
+ short = left is not None and left < spent * 1.2
293
+ if attempt + 1 >= _MIRROR_BUILD_TRIES or short:
294
+ print(f"[sheet] mirror {table_key} changed during its build, refused "
295
+ f"moved={moved} build_ms={int(spent * 1000)}"
296
+ + (f" budget_left_ms={int(left * 1000)}" if short else ""), flush=True)
297
+ refused[slot] = (exc.status_code, deepcopy(exc.detail))
298
+ raise
299
+ print(f"[sheet] mirror {table_key} changed during its build, rebuild "
300
+ f"{attempt + 2}/{_MIRROR_BUILD_TRIES} moved={moved} build_ms={int(spent * 1000)}",
301
+ flush=True)
302
+
303
+
304
+ def _hint_mirror_columns(session, book, only=None):
305
+ """Name every column one refresh reads per registry database, so its mirror builds once."""
306
+ import odoo_relational as rel
307
+ from modules import sheet as sheet_mod
308
+ from core.request_timing import raise_if_resource_error
309
+ context = getattr(session, "_sheet_scope_context", None)
310
+ if not isinstance(context, dict):
311
+ return
312
+ hint = {}
313
+ for tab in book.get("sheets") or []:
314
+ for addr, cell in (tab.get("cells") or {}).items():
315
+ if (not isinstance(cell, dict) or cell.get("kind") != "bound" or _is_view_binding(cell)
316
+ or (only is not None and f"{tab.get('id')}!{addr}" not in only)):
317
+ continue
318
+ try:
319
+ binding = cell.get("binding") if isinstance(cell.get("binding"), dict) else {}
320
+ database = str(binding.get("database") or "")
321
+ scope = binding.get("scope") if isinstance(binding.get("scope"), dict) else {}
322
+ nodes, _conj = _fe_parts(scope.get("filter"))
323
+ related = sheet_mod.related_filter_ids(nodes)
324
+ if database not in rel._REGISTRY_GRID_FIELDS and not related:
325
+ continue
326
+ columns = hint.setdefault(database, set())
327
+ columns.update(name for name in sheet_mod.filter_columns_named(nodes) - related
328
+ if ":" not in str(name))
329
+ field = (binding.get("aggregate") or {}).get("field")
330
+ if field:
331
+ columns.add(str(field))
332
+ for key in sorted(related):
333
+ via, field = sheet_mod.related_field_parts(key)
334
+ entry = _binding_related_entry(session, database, via)
335
+ if entry["via"].get("from"):
336
+ columns.add(str(entry["via"]["from"]))
337
+ hint.setdefault(entry["target"], set()).update(
338
+ {str(field), str(entry["via"].get("on") or field)})
339
+ except Exception as exc: # noqa: BLE001
340
+ # The cell's own resolution reports its refusal. A hint only widens a build.
341
+ raise_if_resource_error(exc)
342
+ continue
343
+ context[("mirror_columns",)] = hint
344
+
345
+
346
  def _mirror(session: Session):
347
  """`(cursor, spec, columns)` for one read-through database, or the refusal that names why.
348
 
 
414
  session._sheet_amzn_cursor = (cur, None)
415
  else:
416
  cur, mismatch = (getattr(session, "_sheet_drill_cursor", None)
417
+ or _request_mirror_cur(session))
418
  except RuntimeError as exc:
419
  raise err(503, "store_not_ready", str(exc))
420
  if mismatch:
 
486
  session._sheet_mirrors = memo
487
  slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str))
488
  want = None if needed is None else {str(k) for k in needed}
489
+ context = getattr(session, "_sheet_scope_context", None)
490
+ hint = (context.get(("mirror_columns",)) or {}).get(table_key) if isinstance(context, dict) else None
491
+ if want is not None and hint:
492
+ want |= hint
493
  entry = memo.get(slot)
494
  if entry is not None:
495
  built, made = entry
 
497
  return made
498
  if want is not None:
499
  want = set(want) | built
500
+ made = _registry_mirror_settled(session, cur, table_key, metric_window, want)
501
  memo[slot] = (None if want is None else set(want), made)
502
  return made
503
+ spec = _request_source_for(session, cur, table_key)
504
  if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
505
  spec = _amazon_stored_spec(cur, table_key)
506
  if spec is None:
 
3914
  _REFRESH_FAILED = {"code": "sheet_unresolved", "message": "This workbook could not be refreshed."}
3915
  _REFRESH_TIMEOUT = {"code": "sheet_refresh_timeout",
3916
  "message": "This refresh took too long. Try again in a few minutes."}
3917
+ #: Seconds the supervisor waits past a run's own deadline before it fails the row for the desk.
3918
+ REFRESH_GRACE_SECONDS = 5.0
3919
+
3920
+
3921
+ def _refresh_budget_env():
3922
+ """`AIOS_SHEET_REFRESH_BUDGET_SECONDS`, else 300. Held below half of REFRESH_STALE_SECONDS."""
3923
+ try:
3924
+ value = float(os.environ.get("AIOS_SHEET_REFRESH_BUDGET_SECONDS") or 300)
3925
+ except ValueError:
3926
+ value = 300.0
3927
+ return min(value if value > 0 else 300.0, REFRESH_STALE_SECONDS / 2)
3928
+
3929
+
3930
+ #: The total budget of one desk run, resolver included. It stays below REFRESH_STALE_SECONDS, so
3931
+ #: a requeue of another process's row never meets an executor that is still inside its budget.
3932
+ REFRESH_BUDGET_SECONDS = _refresh_budget_env()
3933
+ #: This process. A claimed row names its runner, so a reader can tell whose desk holds it.
3934
+ _REFRESH_RUNNER = uuid.uuid4().hex
3935
+ #: Claim token -> {"tenant", "store", "id", "deadline"} for every run this process executes now.
3936
+ _REFRESH_LIVE = {}
3937
 
3938
 
3939
  def _refresh_desk_queues(session):
 
4017
  return made["row"]
4018
 
4019
 
4020
+ def _refresh_desk_overdue(row, now):
4021
+ """"fail", "requeue" or None for one row. A row this process still executes is never requeued."""
4022
+ if row.get("state") != "running":
4023
+ return None
4024
+ live = _REFRESH_LIVE.get(row.get("claim")) if row.get("claim") else None
4025
+ if live is not None:
4026
+ return "fail" if now >= float(live["deadline"]) + REFRESH_GRACE_SECONDS else None
4027
+ if now - float(row.get("startedTs") or 0) > REFRESH_STALE_SECONDS:
4028
+ return "requeue" if int(row.get("attempts") or 0) < REFRESH_MAX_ATTEMPTS else "fail"
4029
+ return None
4030
+
4031
+
4032
+ def _refresh_desk_settle(rows, now, by):
4033
+ """Requeue or fail every overdue row in place. Returns [(request id, new state)]."""
4034
+ changed = []
4035
+ for row in rows.values():
4036
+ verdict = _refresh_desk_overdue(row, now)
4037
+ if verdict == "requeue":
4038
+ # A dropped claim token stops the old executor's late finish from landing on a new run.
4039
+ row.update(state="queued", claim=None, requeuedBy=by)
4040
+ elif verdict == "fail":
4041
+ row.update(state="failed", finishedTs=now, finishedAt=_desk_iso(now),
4042
+ error=dict(_REFRESH_TIMEOUT), finishedBy=by)
4043
+ if verdict:
4044
+ changed.append((row["id"], row["state"]))
4045
+ return changed
4046
+
4047
+
4048
+ def _refresh_desk_reap(store, now=None, by="supervisor"):
4049
+ """The stale step on its own: settle every overdue running row. An idle call writes nothing."""
4050
+ now = _desk_time.time() if now is None else now
4051
+ rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now)
4052
+ if not any(_refresh_desk_overdue(row, now) for row in rows.values()):
4053
+ return []
4054
+ changed = []
4055
+
4056
+ def apply(current):
4057
+ rows = _refresh_desk_rows(current, now)
4058
+ changed[:] = _refresh_desk_settle(rows, now, by)
4059
+ return {"requests": rows}
4060
+
4061
+ store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
4062
+ return changed
4063
+
4064
+
4065
  def _refresh_desk_claim(store, now=None):
4066
+ """Settle overdue rows, then mark the oldest queued request running and return it.
4067
+
4068
+ None while one is already running. The claimed row carries a fresh `claim` token and this
4069
+ process's `runner` id. Only that token can finish it.
4070
+ """
4071
  now = _desk_time.time() if now is None else now
4072
  claimed = {}
4073
 
 
 
 
4074
  # Read first: an idle poll must never write the store.
4075
  rows = _refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now).values()
4076
+ overdue = {row.get("id") for row in rows if _refresh_desk_overdue(row, now)}
4077
+ busy = any(row.get("state") == "running" and row.get("id") not in overdue for row in rows)
4078
+ if not overdue and (busy or not any(row.get("state") == "queued" for row in rows)):
4079
  return None
4080
 
4081
  def apply(current):
4082
  rows = _refresh_desk_rows(current, now)
4083
+ _refresh_desk_settle(rows, now, "claim")
 
 
 
 
 
 
4084
  if not any(row.get("state") == "running" for row in rows.values()):
4085
  queued = sorted((row for row in rows.values() if row.get("state") == "queued"),
4086
  key=_refresh_desk_order)
4087
  if queued:
4088
  row = queued[0]
4089
  row.update(state="running", startedTs=now, startedAt=_desk_iso(now),
4090
+ attempts=int(row.get("attempts") or 0) + 1,
4091
+ claim=uuid.uuid4().hex, runner=_REFRESH_RUNNER)
4092
  claimed["row"] = dict(row)
4093
  return {"requests": rows}
4094
 
 
4096
  return claimed.get("row")
4097
 
4098
 
4099
+ def _refresh_desk_owns(row, claim):
4100
+ """True when `claim` is the row's current token. A caller without a token skips the check."""
4101
+ return claim is None or row.get("claim") == claim
4102
+
4103
+
4104
+ def _refresh_desk_finish(store, request_id, error=None, now=None, claim=None):
4105
+ """Mark one running request done, or failed with a public `{code, message}`.
4106
+
4107
+ Returns False and writes nothing when the row is no longer running under `claim`.
4108
+ """
4109
  now = _desk_time.time() if now is None else now
4110
+ done = {"ok": False}
4111
+
4112
+ def mine(rows):
4113
+ row = rows.get(request_id)
4114
+ return row if row is not None and row.get("state") == "running" and _refresh_desk_owns(row, claim) else None
4115
+
4116
+ if mine(_refresh_desk_rows(store.get(REFRESH_REQUESTS_KEY, fresh=True), now)) is None:
4117
+ return False
4118
 
4119
  def apply(current):
4120
  rows = _refresh_desk_rows(current, now)
4121
+ row = mine(rows)
4122
+ if row is not None:
4123
  row.update(state="failed" if error else "done", finishedTs=now,
4124
+ finishedAt=_desk_iso(now), error=dict(error) if error else None,
4125
+ finishedBy="desk")
4126
+ done["ok"] = True
4127
  return {"requests": rows}
4128
 
4129
  store.update(REFRESH_REQUESTS_KEY, apply, flush="sync")
4130
+ return done["ok"]
4131
 
4132
 
4133
  def _refresh_desk_entry(store, workbook_id, reader, request_id):
 
4191
  return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
4192
  with _sheet_request_scope(session):
4193
  if args.get("kind") == "retry":
4194
+ result = _refresh_sheet_scoped(workbook, session, retry_id=args.get("retryId"))
4195
+ else:
4196
+ only = set(args.get("cells") or ()) if args.get("kind") == "cells" else None
4197
+ result = _refresh_sheet_scoped(workbook, session, only=only)
4198
+ if isinstance(result, dict) and isinstance(result.get("workbook"), dict):
4199
+ # Placed views read their rows after the run ends. Their mirror builds here, in a fresh
4200
+ # scope, so the view-rows call reuses it. The run's own result never depends on it.
4201
+ try:
4202
+ with _sheet_request_scope(session):
4203
+ _prewarm_view_mirrors(session, result["workbook"])
4204
+ except Exception as exc: # noqa: BLE001
4205
+ print(f"[sheet] view prewarm wb={workbook} failed {type(exc).__name__}", flush=True)
4206
+ return result
4207
 
4208
 
4209
  # ── the opening cache: a repeat open of the same tab at the same state is served from memory ─────
 
4599
  return hit["cell"]
4600
  return None
4601
 
4602
+ _hint_mirror_columns(session, book, only)
4603
  started = perf_counter()
4604
  resolved_count = 0
4605
  held_refusal = None
 
5715
  "store_not_ready": "#UPDATING",
5716
  "mirror_not_ready": "#UPDATING",
5717
  "amazon_mirror_not_loaded": "#UPDATING",
5718
+ "drill_population_changed": "#UPDATING",
5719
  }
5720
 
5721
 
 
5724
  "store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
5725
  "mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
5726
  "sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
5727
+ "drill_population_changed",
5728
  ))
5729
 
5730
  #: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
 
6169
  # and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below.
6170
  _refuse_unapplied_audience(session, database)
6171
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
6172
+ if _row_scope_applies(session, database) else None)
6173
  wall_nodes, wall_conj = _fe_parts(wall_tree)
6174
  scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
6175
  scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
 
6610
  def view_rows(workbook_id: str, body: object = Body(default=None),
6611
  session: Session = Depends(require_session)):
6612
  """W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
6613
+ from time import perf_counter
6614
  from fastapi.responses import JSONResponse
6615
+ import odoo_relational as rel
6616
  from core import perm_scope, sheets, user_tables
6617
  from modules import sheet as sheet_mod
6618
 
 
6638
 
6639
  # W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
6640
  _refuse_unapplied_audience(session, database)
6641
+ wall_nodes, wall_conj, scope_nodes, scope_conj = _view_scope(session, database, binding)
 
 
 
 
6642
  # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
6643
  for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
6644
  if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
6645
  return JSONResponse(status_code=400, content={
6646
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6647
  today = _sheet_today(session)
6648
+ started = perf_counter()
6649
+ builds = getattr(session, "_sheet_mirror_builds", 0)
6650
  try:
6651
+ scope_nodes, window, related_bags, door = _view_mirror_door(
6652
+ session, book, binding, database, scope_nodes, wall_nodes, today)
6653
+ with door as (made, _token):
6654
+ cur, spec, cols = made
6655
+ cols = dict(cols)
6656
+ if spec.get("scope_applied"):
6657
+ wall_nodes, wall_conj = [], "and"
6658
+ related = related_bags(cur, spec, cols, window=window)
6659
+ for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
6660
+ if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
6661
+ continue
6662
+ if key in hidden or not (cols.get(key) or {}).get("sql"):
6663
+ return JSONResponse(status_code=400, content={
6664
+ "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6665
+ # W62 C2: an unanswerable row wall must refuse before either SQL read can widen it.
6666
+ wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
6667
+ if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled):
6668
+ raise ValueError("unapplied row wall")
6669
+ count_sql, params, limits = sheet_mod.build_aggregate_query(
6670
+ spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
6671
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
6672
+ related=related, summary_default=False)
6673
+ if limits:
6674
+ raise ValueError("unapplied view filter")
6675
+
6676
+ # W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
6677
+ total = (cur.execute(count_sql, params).fetchone() or [0])[0]
6678
+ # ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
6679
+ # A placed Grid over one of our own tables writes a record back, and the write door is Data's
6680
+ # own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
6681
+ # from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
6682
+ # refreshes. So the id rides beside the values under the key `pid` while `fields` still names
6683
+ # only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
6684
+ # uses, so the identity here and the identity the drill lists can never be two different things.
6685
+ id_expr = str(spec.get("id") or "").strip()
6686
+ projection = ", ".join([*(cols[key]["sql"] for key in requested),
6687
+ *([id_expr] if id_expr else [])])
6688
+ rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
6689
+ if binding["sort"]:
6690
+ # W62 C2 / R30: expressions come only from resolved columns; directions are validated.
6691
+ order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
6692
+ for item in binding["sort"])
6693
+ rows_sql += " ORDER BY " + order
6694
+ rows_sql += f" LIMIT {binding['limit']}"
6695
+ keys = [*requested, *(["pid"] if id_expr else [])]
6696
+ rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
6697
  except sheet_mod.SheetRefusal as exc:
6698
  return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
6699
  except ValueError:
6700
  raise err(409, "scope_not_applicable",
6701
  "This view uses a restriction that cannot be applied to this database's live connection.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6702
  # W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
6703
  # W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
6704
  editable = user_tables.records_mutable(database, st=session.runtime)
6705
+ reused = (int(getattr(session, "_sheet_mirror_builds", 0) == builds)
6706
+ if database in rel._REGISTRY_GRID_FIELDS else "na")
6707
+ print(f"[sheet] view-rows wb={workbook_id} db={database} ms={int((perf_counter() - started) * 1000)} "
6708
+ f"reused={reused} rows={len(rows)} total={int(total or 0)}", flush=True)
6709
  return {"fields": [{"key": key, "label": visible[key].get("label") or key,
6710
  "kind": visible[key].get("type") or visible[key].get("kind") or "text"}
6711
  for key in requested],
 
6713
  "viewType": binding["viewType"], "editable": bool(editable)}
6714
 
6715
 
6716
+ def _view_scope(session, database, binding):
6717
+ """`(wall_nodes, wall_conj, scope_nodes, scope_conj)` of one placed view for this reader."""
6718
+ import core.perm_scope as perm_scope
6719
+ _refuse_unapplied_audience(session, database)
6720
+ wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
6721
+ if perm_scope.row_scope_applies(session.user, database) else None)
6722
+ wall_nodes, wall_conj = _fe_parts(wall_tree)
6723
+ scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
6724
+ return wall_nodes, wall_conj, _rewrite_related_filters(session, database, scope_nodes), scope_conj
6725
+
6726
+
6727
+ def _view_mirror_door(session, book, binding, database, scope_nodes, wall_nodes, today):
6728
+ """`(scope_nodes, window, related_bags, door)`; `door` yields `((cur, spec, cols), token)`.
6729
+
6730
+ A compiled registry grid reads through the reader-keyed drill mirror cache, so a repeat view
6731
+ and a view the desk prewarmed reuse one build. Every other database builds per request.
6732
+ """
6733
+ import odoo_relational as rel
6734
+ from modules import sheet as sheet_mod
6735
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today)
6736
+ window = _ending_window(session, binding, book=book, today=today)
6737
+ needed = (set(binding["fields"]) | {item["field"] for item in binding["sort"]}
6738
+ | sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes))
6739
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
6740
+ if database in rel._REGISTRY_GRID_FIELDS:
6741
+ databases = {database}
6742
+ databases.update(_binding_related_entry(session, database, via)["target"]
6743
+ for via in sheet_mod.related_vias(scope_nodes))
6744
+ return scope_nodes, window, related_bags, _drill_mirror(session, database, window, needed, databases)
6745
+ made = _mirror(session)(database, metric_window=window, needed=needed)
6746
+ return scope_nodes, window, related_bags, contextlib.nullcontext((made, None))
6747
+
6748
+
6749
+ #: A desk prewarm of placed views starts only with this much of the run's budget left.
6750
+ VIEW_PREWARM_MIN_SECONDS = 90.0
6751
+ #: And it stops this long before the run's own deadline.
6752
+ VIEW_PREWARM_MARGIN_SECONDS = 20.0
6753
+
6754
+
6755
+ def _prewarm_view_mirrors(session, book):
6756
+ """Build the drill mirror each registry-grid placed view of `book` reads, as this reader.
6757
+
6758
+ The client reads view rows after a Refresh within a 45 s request budget. A customer grid
6759
+ build costs more than that, so the desk builds it first and the view-rows call reuses it.
6760
+ A failed prewarm leaves the view to build on its own request.
6761
+ """
6762
+ import time
6763
+ from copy import deepcopy
6764
+ from time import perf_counter
6765
+ import odoo_relational as rel
6766
+ from core import request_timing, sheets
6767
+ left = request_timing.remaining_budget()
6768
+ warmed = 0
6769
+ for tab in book.get("sheets") or []:
6770
+ for addr, cell in (tab.get("cells") or {}).items():
6771
+ if not (isinstance(cell, dict) and cell.get("kind") == "bound" and _is_view_binding(cell)
6772
+ and str((cell.get("binding") or {}).get("database") or "") in rel._REGISTRY_GRID_FIELDS):
6773
+ continue
6774
+ if left is not None and left < VIEW_PREWARM_MIN_SECONDS:
6775
+ print(f"[sheet] view prewarm wb={book.get('id')} skipped budget_left_ms={int(left * 1000)}",
6776
+ flush=True)
6777
+ return warmed
6778
+ started = perf_counter()
6779
+ builds = getattr(session, "_sheet_mirror_builds", 0)
6780
+ try:
6781
+ binding = sheets.clean_view_binding(deepcopy(cell["binding"]))
6782
+ database = _require_database(session, binding["database"])
6783
+ _refuse_unapplied_audience(session, database)
6784
+ wall_nodes, _wall_conj, scope_nodes, _conj = _view_scope(session, database, binding)
6785
+ deadline = (None if left is None else
6786
+ time.monotonic() + left - VIEW_PREWARM_MARGIN_SECONDS)
6787
+ with (request_timing.budget_scope(deadline) if deadline is not None
6788
+ else contextlib.nullcontext()):
6789
+ _nodes, window, _bags, door = _view_mirror_door(
6790
+ session, book, binding, database, scope_nodes, wall_nodes, _sheet_today(session))
6791
+ # A second view of the same reader and window reuses the first build.
6792
+ with door as (_made, token):
6793
+ outcome = "cached" if token else "uncached"
6794
+ warmed += 1 if token else 0
6795
+ except Exception as exc: # noqa: BLE001
6796
+ outcome = "failed " + type(exc).__name__
6797
+ print(f"[sheet] view prewarm wb={book.get('id')} cell={tab.get('id')}!{addr} {outcome} "
6798
+ f"built={int(getattr(session, '_sheet_mirror_builds', 0) != builds)} "
6799
+ f"ms={int((perf_counter() - started) * 1000)}", flush=True)
6800
+ left = request_timing.remaining_budget()
6801
+ return warmed
6802
+
6803
+
6804
  _DRILL_MIRRORS = OrderedDict()
6805
+ # The registry guard: held only for lookups and bookkeeping, never across a build or a yield.
6806
  _DRILL_MIRROR_LOCK = threading.RLock()
6807
  _DRILL_MIRROR_MAX = 6
6808
+ # One lease per cache key: a mirror cursor holds connection-scoped temp tables and serves one thread.
6809
+ _DRILL_LEASES = {}
6810
 
6811
 
6812
  @contextlib.contextmanager
6813
  def _drill_mirror_lock():
6814
+ """Include registry-guard contention in the caller's absolute request budget."""
6815
  from core import request_timing
6816
  timeout = request_timing.remaining_budget()
6817
  acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None
 
6828
  _DRILL_MIRROR_LOCK.release()
6829
 
6830
 
6831
+ @contextlib.contextmanager
6832
+ def _drill_lease(key):
6833
+ """Serialize one cache key's cursor. Other keys never wait, and a waiter spends its budget."""
6834
+ from core import request_timing
6835
+ if key is None:
6836
+ # An uncacheable drill owns a private cursor, so it shares nothing to serialize.
6837
+ yield {"entry": None}
6838
+ return
6839
+ with _drill_mirror_lock():
6840
+ lease = _DRILL_LEASES.get(key)
6841
+ if lease is None:
6842
+ lease = _DRILL_LEASES[key] = {"cond": threading.Condition(threading.Lock()),
6843
+ "busy": False, "refs": 0, "entry": None}
6844
+ lease["refs"] += 1
6845
+ try:
6846
+ with lease["cond"]:
6847
+ while lease["busy"]:
6848
+ lease["cond"].wait(request_timing.remaining_budget())
6849
+ lease["busy"] = True
6850
+ try:
6851
+ yield lease
6852
+ finally:
6853
+ with lease["cond"]:
6854
+ lease["busy"] = False
6855
+ lease["cond"].notify_all()
6856
+ finally:
6857
+ with _DRILL_MIRROR_LOCK:
6858
+ lease["refs"] -= 1
6859
+ if not lease["refs"] and _DRILL_LEASES.get(key) is lease:
6860
+ _DRILL_LEASES.pop(key)
6861
+
6862
+
6863
  def _drill_tokens(session, databases):
6864
  """Unknown freshness disables reuse. No workspace documents are retained here."""
6865
  from core import sheet_binding
 
6878
 
6879
  @contextlib.contextmanager
6880
  def _drill_mirror(session, database, window, needed, databases):
6881
+ """A small reader-scoped LRU of query mirrors: one lease per key, short registry guard."""
6882
  from core import sheet_binding, perm_scope
6883
  from core.request_timing import raise_if_resource_error
6884
  import datetime
 
6911
  "_sheet_drill_entries")
6912
  absent = object()
6913
  previous = {name: getattr(session, name, absent) for name in attrs}
6914
+ with _drill_lease(key if token else None) as lease:
6915
+ with _drill_mirror_lock():
6916
+ hit = _DRILL_MIRRORS.get(key) if token else None
6917
+ lease["entry"] = hit
6918
  session._sheet_drill_isolated = True
6919
  session._sheet_drill_cursor = hit["owned"] if hit else None
6920
  session._sheet_drill_entries = hit["mirrors"] if hit else {}
 
6926
  want = set(needed) | (hit["needed"] if hit else set())
6927
  if hit and want <= hit["needed"]:
6928
  made = hit["made"]
6929
+ with _DRILL_MIRROR_LOCK:
6930
+ if _DRILL_MIRRORS.get(key) is hit:
6931
+ _DRILL_MIRRORS.move_to_end(key)
6932
  else:
6933
+ with _DRILL_MIRROR_LOCK:
6934
+ if hit is not None and _DRILL_MIRRORS.get(key) is hit:
6935
+ _DRILL_MIRRORS.pop(key)
6936
+ # The build runs under this key's lease only, so other keys build at once.
6937
  made = _mirror(session)(database, metric_window=window, needed=want)
6938
  owned = getattr(session, "_sheet_drill_cursor", None)
6939
  hit = {"made": made, "needed": want, "owned": owned,
6940
  "mirrors": session._sheet_drill_entries}
6941
  # A build racing a source change is usable for this request, never reusable.
6942
  if token and _drill_tokens(session, databases) == sources:
6943
+ with _DRILL_MIRROR_LOCK:
6944
+ _DRILL_MIRRORS[key] = hit
6945
+ _DRILL_MIRRORS.move_to_end(key)
6946
+ lease["entry"] = hit
6947
+ while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
6948
+ gone, evicted = _DRILL_MIRRORS.popitem(last=False)
6949
+ # A leased cursor closes when its holder leaves, never under it.
6950
+ held = _DRILL_LEASES.get(gone)
6951
+ if evicted["owned"] and not (held and held["entry"] is evicted):
6952
+ evicted["owned"][0].close()
6953
  else:
6954
  token = None
6955
  # Related leaves may revisit the primary registry. Keep its table intact and widen
 
6959
  yield made, token
6960
  finally:
6961
  owned = getattr(session, "_sheet_drill_cursor", None)
6962
+ entry = lease["entry"]
6963
+ with _DRILL_MIRROR_LOCK:
6964
+ lease["entry"] = None
6965
+ kept = [item["owned"] for item in _DRILL_MIRRORS.values()]
6966
+ # The last lease on an uncached or evicted cursor closes it.
6967
+ spent = [owned, entry["owned"] if entry else None]
6968
+ for index, cursor in enumerate(spent):
6969
+ if (cursor and not any(seen is cursor for seen in spent[:index])
6970
+ and not any(item is cursor for item in kept)):
6971
+ cursor[0].close()
6972
  for name, value in previous.items():
6973
  if value is absent:
6974
  session.__dict__.pop(name, None)
api/routes_starred.py CHANGED
@@ -329,7 +329,8 @@ def _scan_starred_views(session, keys=None, cache=None, include=None, with_right
329
  import core.table_store as table_store
330
  import core.view_templates as view_templates
331
 
332
- granted = routes_nav._granted_view_sources(session)
 
333
  # ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
334
  # where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
335
  # (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
@@ -402,6 +403,13 @@ def _scan_starred_views(session, keys=None, cache=None, include=None, with_right
402
  if with_rights:
403
  row.update(_home_view_row(
404
  session, (key, ws_key, owners[vid], {**view, "id": vid}, None)))
 
 
 
 
 
 
 
405
  out.append(row)
406
  return out, unread
407
 
@@ -522,9 +530,10 @@ def _payload(session, keys=None, cache=None):
522
 
523
  @router.get("/nav/views")
524
  def nav_views(session: Session = Depends(require_session)):
525
- """Project readable saved View names and their current item rights."""
526
  rows, unread = _scan_starred_views(session, include=lambda view: True, with_rights=True)
527
- return {"views": [{key: row[key] for key in ("database", "id", "name", "capabilities")}
 
528
  for row in rows], "unread": unread}
529
 
530
 
 
329
  import core.table_store as table_store
330
  import core.view_templates as view_templates
331
 
332
+ # The docs this scan holds resolve folder-shared Views without a second read per database.
333
+ granted = routes_nav._granted_view_sources(session, docs=docs)
334
  # ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
335
  # where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
336
  # (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
 
403
  if with_rights:
404
  row.update(_home_view_row(
405
  session, (key, ws_key, owners[vid], {**view, "id": vid}, None)))
406
+ # The owner's `itemFolders` is keyed by surface first. An empty id must never match.
407
+ owner_ws = doc.get(owners[vid])
408
+ placed = (((owner_ws.get("itemFolders") or {}).get("views") or {})
409
+ if isinstance(owner_ws, dict) else {})
410
+ fid = str(placed.get(vid) or "") if isinstance(placed, dict) else ""
411
+ if fid:
412
+ row["folderId"] = fid
413
  out.append(row)
414
  return out, unread
415
 
 
530
 
531
  @router.get("/nav/views")
532
  def nav_views(session: Session = Depends(require_session)):
533
+ """Project readable saved View names, current item rights and the owner's folder id."""
534
  rows, unread = _scan_starred_views(session, include=lambda view: True, with_rights=True)
535
+ return {"views": [{key: row[key] for key in ("database", "id", "name", "capabilities", "folderId")
536
+ if key in row}
537
  for row in rows], "unread": unread}
538
 
539