Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- Dockerfile +1 -0
- RELEASES.json +13 -463
- VERSION +1 -1
- api/ads_intent_groups.py +23 -5
- api/agent_directory.py +20 -3
- api/amazon_quotes_loop.py +128 -0
- api/amazon_relational.py +15 -2
- api/app_templates.py +22 -8
- api/application_registry.py +402 -23
- api/assistant_feed.py +2 -4
- api/assistant_memory.py +196 -1
- api/assistant_mirror.py +19 -12
- api/automation_engine.py +737 -20
- api/case_drafts.py +45 -30
- api/connector_onboarding.py +19 -5
- api/deps.py +288 -62
- api/diag_aging.py +8 -13
- api/external_freshness.py +22 -9
- api/loopy_catalog_tools.py +10 -1
- api/loopy_file_tools.py +3 -1
- api/loopy_tools.py +1 -1
- api/main.py +795 -107
- api/notify_destinations.py +111 -3
- api/odoo_relational.py +478 -156
- api/pages.py +6 -4
- api/process_role.py +398 -162
- api/registry_source.py +193 -1
- api/rollup_sql.py +19 -7
- api/routes_account_health.py +168 -20
- api/routes_admin.py +118 -189
- api/routes_aged_stock.py +8 -5
- api/routes_amazon_dashboard.py +18 -43
- api/routes_amazon_operator.py +104 -16
- api/routes_amazon_profit.py +163 -19
- api/routes_amazon_projects.py +703 -0
- api/routes_app_dashboard.py +72 -26
- api/routes_applications.py +1 -1
- api/routes_assets.py +36 -5
- api/routes_auth.py +7 -2
- api/routes_automation.py +118 -0
- api/routes_competitor_watch.py +80 -16
- api/routes_competitors.py +203 -0
- api/routes_connectors.py +236 -36
- api/routes_customers.py +100 -21
- api/routes_external.py +688 -67
- api/routes_feedback.py +45 -4
- api/routes_forms.py +5 -0
- api/routes_grid.py +45 -3
- api/routes_keychain.py +283 -4
- api/routes_nav.py +12 -8
Dockerfile
CHANGED
|
@@ -61,6 +61,7 @@ COPY requirements.txt ./
|
|
| 61 |
RUN python -m pip install --no-cache-dir --disable-pip-version-check --upgrade "pip==25.0.1" \
|
| 62 |
&& python -m pip install --no-cache-dir --disable-pip-version-check -r requirements.txt
|
| 63 |
COPY api/ ./api/
|
|
|
|
| 64 |
# the reused data layer (modules/core/harness)
|
| 65 |
# ⛔ COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
|
| 66 |
# it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
|
|
|
|
| 61 |
RUN python -m pip install --no-cache-dir --disable-pip-version-check --upgrade "pip==25.0.1" \
|
| 62 |
&& python -m pip install --no-cache-dir --disable-pip-version-check -r requirements.txt
|
| 63 |
COPY api/ ./api/
|
| 64 |
+
COPY VERSION ./VERSION
|
| 65 |
# the reused data layer (modules/core/harness)
|
| 66 |
# ⛔ COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
|
| 67 |
# it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
|
RELEASES.json
CHANGED
|
@@ -1,515 +1,65 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
| 6 |
-
"sha": "
|
| 7 |
-
"date": "2026-
|
| 8 |
-
"subject": "release v86
|
| 9 |
},
|
| 10 |
{
|
| 11 |
"version": "v85",
|
| 12 |
-
"sha": "
|
| 13 |
"date": "2026-10-25",
|
| 14 |
"subject": "Merge remote-tracking branch 'origin/master' into hfstage/sdlc79-release"
|
| 15 |
},
|
| 16 |
{
|
| 17 |
"version": "v84",
|
| 18 |
-
"sha": "
|
| 19 |
"date": "2026-10-24",
|
| 20 |
"subject": "release(sdlc78): v84 QA records the staging build change and what v84 excludes"
|
| 21 |
},
|
| 22 |
{
|
| 23 |
"version": "v83",
|
| 24 |
-
"sha": "
|
| 25 |
"date": "2026-10-21",
|
| 26 |
"subject": "release v83"
|
| 27 |
},
|
| 28 |
{
|
| 29 |
"version": "v82",
|
| 30 |
-
"sha": "
|
| 31 |
"date": "2026-10-17",
|
| 32 |
"subject": "release v82"
|
| 33 |
},
|
| 34 |
{
|
| 35 |
"version": "v81",
|
| 36 |
-
"sha": "
|
| 37 |
"date": "2026-10-15",
|
| 38 |
"subject": "release v81"
|
| 39 |
},
|
| 40 |
{
|
| 41 |
"version": "v80",
|
| 42 |
-
"sha": "
|
| 43 |
"date": "2026-10-13",
|
| 44 |
"subject": "release v80"
|
| 45 |
},
|
| 46 |
{
|
| 47 |
"version": "v79",
|
| 48 |
-
"sha": "
|
| 49 |
"date": "2026-10-13",
|
| 50 |
"subject": "release v79"
|
| 51 |
},
|
| 52 |
{
|
| 53 |
"version": "v78",
|
| 54 |
-
"sha": "
|
| 55 |
"date": "2026-10-09",
|
| 56 |
"subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs"
|
| 57 |
},
|
| 58 |
{
|
| 59 |
"version": "v77",
|
| 60 |
-
"sha": "
|
| 61 |
"date": "2026-10-05",
|
| 62 |
"subject": "release v77"
|
| 63 |
-
},
|
| 64 |
-
{
|
| 65 |
-
"version": "v76",
|
| 66 |
-
"sha": "7990f03e0254",
|
| 67 |
-
"date": "2026-10-01",
|
| 68 |
-
"subject": "release v76"
|
| 69 |
-
},
|
| 70 |
-
{
|
| 71 |
-
"version": "v75",
|
| 72 |
-
"sha": "2c5146981ccf",
|
| 73 |
-
"date": "2026-09-19",
|
| 74 |
-
"subject": "release v75"
|
| 75 |
-
},
|
| 76 |
-
{
|
| 77 |
-
"version": "v74",
|
| 78 |
-
"sha": "40dceba9262e",
|
| 79 |
-
"date": "2026-09-14",
|
| 80 |
-
"subject": "release v74: waves 58+59 incl. the Amazon credential door (W59-T21/22/23) and sync producer (W59-T42); hotfix promote so"
|
| 81 |
-
},
|
| 82 |
-
{
|
| 83 |
-
"version": "v73",
|
| 84 |
-
"sha": "bb94de5665e6",
|
| 85 |
-
"date": "2026-09-13",
|
| 86 |
-
"subject": "release v73"
|
| 87 |
-
},
|
| 88 |
-
{
|
| 89 |
-
"version": "v71",
|
| 90 |
-
"sha": "96143dc6fb65",
|
| 91 |
-
"date": "2026-09-04",
|
| 92 |
-
"subject": "release v71 - wave 51, QA'd 24 PASS / 6 FAIL / 5 UNPROVEN"
|
| 93 |
-
},
|
| 94 |
-
{
|
| 95 |
-
"version": "v70",
|
| 96 |
-
"sha": "eedfb6a00163",
|
| 97 |
-
"date": "2026-09-03",
|
| 98 |
-
"subject": "release v70 - wave 50 (grid analytics, the Sheet as a spreadsheet, view trash and audit)"
|
| 99 |
-
},
|
| 100 |
-
{
|
| 101 |
-
"version": "v69",
|
| 102 |
-
"sha": "8598f56293a0",
|
| 103 |
-
"date": "2026-09-02",
|
| 104 |
-
"subject": "release v69: wave48-continuation validated + hotfixes v67/v68/d4e3e31 + go-live QA repairs (Supplier option limit, optio"
|
| 105 |
-
},
|
| 106 |
-
{
|
| 107 |
-
"version": "v68",
|
| 108 |
-
"sha": "62b065a2610a",
|
| 109 |
-
"date": "2026-09-01",
|
| 110 |
-
"subject": "release v68: unified holiday date offsets"
|
| 111 |
-
},
|
| 112 |
-
{
|
| 113 |
-
"version": "v67",
|
| 114 |
-
"sha": "3b2241982cf7",
|
| 115 |
-
"date": "2026-09-01",
|
| 116 |
-
"subject": "release v67"
|
| 117 |
-
},
|
| 118 |
-
{
|
| 119 |
-
"version": "v66",
|
| 120 |
-
"sha": "73c94f076740",
|
| 121 |
-
"date": "2026-08-31",
|
| 122 |
-
"subject": "release v66"
|
| 123 |
-
},
|
| 124 |
-
{
|
| 125 |
-
"version": "v65",
|
| 126 |
-
"sha": "9f6cfedb8a43",
|
| 127 |
-
"date": "2026-08-31",
|
| 128 |
-
"subject": "release v65"
|
| 129 |
-
},
|
| 130 |
-
{
|
| 131 |
-
"version": "v64",
|
| 132 |
-
"sha": "bb8985c6e73e",
|
| 133 |
-
"date": "2026-08-31",
|
| 134 |
-
"subject": "release v64"
|
| 135 |
-
},
|
| 136 |
-
{
|
| 137 |
-
"version": "v63",
|
| 138 |
-
"sha": "ec75cb8f3412",
|
| 139 |
-
"date": "2026-08-31",
|
| 140 |
-
"subject": "release v63"
|
| 141 |
-
},
|
| 142 |
-
{
|
| 143 |
-
"version": "v62",
|
| 144 |
-
"sha": "2c8f702c964f",
|
| 145 |
-
"date": "2026-08-31",
|
| 146 |
-
"subject": "release v62"
|
| 147 |
-
},
|
| 148 |
-
{
|
| 149 |
-
"version": "v61",
|
| 150 |
-
"sha": "bbc4c0760ab5",
|
| 151 |
-
"date": "2026-08-30",
|
| 152 |
-
"subject": "v61 - wave 46 hotfix: D-590 schema overlay reaches the grid read (all three assemblies), D-599 route to zone association"
|
| 153 |
-
},
|
| 154 |
-
{
|
| 155 |
-
"version": "v60",
|
| 156 |
-
"sha": "dd1b0620210e",
|
| 157 |
-
"date": "2026-08-30",
|
| 158 |
-
"subject": "v60 - wave 46"
|
| 159 |
-
},
|
| 160 |
-
{
|
| 161 |
-
"version": "v59",
|
| 162 |
-
"sha": "3cb73ce8021e",
|
| 163 |
-
"date": "2026-08-29",
|
| 164 |
-
"subject": "release v59"
|
| 165 |
-
},
|
| 166 |
-
{
|
| 167 |
-
"version": "v58",
|
| 168 |
-
"sha": "c526cede82f9",
|
| 169 |
-
"date": "2026-08-29",
|
| 170 |
-
"subject": "release v58"
|
| 171 |
-
},
|
| 172 |
-
{
|
| 173 |
-
"version": "v57",
|
| 174 |
-
"sha": "4859c035e6a7",
|
| 175 |
-
"date": "2026-08-29",
|
| 176 |
-
"subject": "release v57"
|
| 177 |
-
},
|
| 178 |
-
{
|
| 179 |
-
"version": "v56",
|
| 180 |
-
"sha": "2970b729e8f0",
|
| 181 |
-
"date": "2026-08-28",
|
| 182 |
-
"subject": "release v56"
|
| 183 |
-
},
|
| 184 |
-
{
|
| 185 |
-
"version": "v55",
|
| 186 |
-
"sha": "b766b904b8fa",
|
| 187 |
-
"date": "2026-08-28",
|
| 188 |
-
"subject": "release v55"
|
| 189 |
-
},
|
| 190 |
-
{
|
| 191 |
-
"version": "v54",
|
| 192 |
-
"sha": "6a8e99a064d9",
|
| 193 |
-
"date": "2026-08-27",
|
| 194 |
-
"subject": "release v54"
|
| 195 |
-
},
|
| 196 |
-
{
|
| 197 |
-
"version": "v53",
|
| 198 |
-
"sha": "89ed2cb867da",
|
| 199 |
-
"date": "2026-08-24",
|
| 200 |
-
"subject": "release v53"
|
| 201 |
-
},
|
| 202 |
-
{
|
| 203 |
-
"version": "v52",
|
| 204 |
-
"sha": "826349376241",
|
| 205 |
-
"date": "2026-08-23",
|
| 206 |
-
"subject": "release v52"
|
| 207 |
-
},
|
| 208 |
-
{
|
| 209 |
-
"version": "v51",
|
| 210 |
-
"sha": "5740d20fa585",
|
| 211 |
-
"date": "2026-08-23",
|
| 212 |
-
"subject": "release v51"
|
| 213 |
-
},
|
| 214 |
-
{
|
| 215 |
-
"version": "v50",
|
| 216 |
-
"sha": "8027c83d1da6",
|
| 217 |
-
"date": "2026-08-23",
|
| 218 |
-
"subject": "release v50"
|
| 219 |
-
},
|
| 220 |
-
{
|
| 221 |
-
"version": "v49",
|
| 222 |
-
"sha": "9a14c01a48a4",
|
| 223 |
-
"date": "2026-08-23",
|
| 224 |
-
"subject": "release v49"
|
| 225 |
-
},
|
| 226 |
-
{
|
| 227 |
-
"version": "v48",
|
| 228 |
-
"sha": "3c69fd93da7c",
|
| 229 |
-
"date": "2026-08-23",
|
| 230 |
-
"subject": "release v48"
|
| 231 |
-
},
|
| 232 |
-
{
|
| 233 |
-
"version": "v47",
|
| 234 |
-
"sha": "917f9a0eb509",
|
| 235 |
-
"date": "2026-08-23",
|
| 236 |
-
"subject": "release v47"
|
| 237 |
-
},
|
| 238 |
-
{
|
| 239 |
-
"version": "v46",
|
| 240 |
-
"sha": "a8dc531b5078",
|
| 241 |
-
"date": "2026-08-23",
|
| 242 |
-
"subject": "release v46"
|
| 243 |
-
},
|
| 244 |
-
{
|
| 245 |
-
"version": "v45",
|
| 246 |
-
"sha": "0b2b5010d86f",
|
| 247 |
-
"date": "2026-08-23",
|
| 248 |
-
"subject": "release v45"
|
| 249 |
-
},
|
| 250 |
-
{
|
| 251 |
-
"version": "v44",
|
| 252 |
-
"sha": "a7877aaf20ff",
|
| 253 |
-
"date": "2026-08-23",
|
| 254 |
-
"subject": "Hotfix: allow renaming the active Map view"
|
| 255 |
-
},
|
| 256 |
-
{
|
| 257 |
-
"version": "v43",
|
| 258 |
-
"sha": "e2ba4616fa0f",
|
| 259 |
-
"date": "2026-08-23",
|
| 260 |
-
"subject": "release v43"
|
| 261 |
-
},
|
| 262 |
-
{
|
| 263 |
-
"version": "v42",
|
| 264 |
-
"sha": "25b885e6373a",
|
| 265 |
-
"date": "2026-08-23",
|
| 266 |
-
"subject": "release v42"
|
| 267 |
-
},
|
| 268 |
-
{
|
| 269 |
-
"version": "v41",
|
| 270 |
-
"sha": "1d6adcc3c3ac",
|
| 271 |
-
"date": "2026-08-22",
|
| 272 |
-
"subject": "v41 vertical route navigation"
|
| 273 |
-
},
|
| 274 |
-
{
|
| 275 |
-
"version": "v40",
|
| 276 |
-
"sha": "1451837168f3",
|
| 277 |
-
"date": "2026-08-22",
|
| 278 |
-
"subject": "release v40 route UI correction"
|
| 279 |
-
},
|
| 280 |
-
{
|
| 281 |
-
"version": "v39",
|
| 282 |
-
"sha": "a78bdf00e38e",
|
| 283 |
-
"date": "2026-08-21",
|
| 284 |
-
"subject": "W39 clean stale migration targets"
|
| 285 |
-
},
|
| 286 |
-
{
|
| 287 |
-
"version": "v38",
|
| 288 |
-
"sha": "a556631cf979",
|
| 289 |
-
"date": "2026-08-21",
|
| 290 |
-
"subject": "W39 reversible Live store cutover"
|
| 291 |
-
},
|
| 292 |
-
{
|
| 293 |
-
"version": "v37",
|
| 294 |
-
"sha": "1ddd8cf82379",
|
| 295 |
-
"date": "2026-08-21",
|
| 296 |
-
"subject": "W39: coalesce derived workspace reads"
|
| 297 |
-
},
|
| 298 |
-
{
|
| 299 |
-
"version": "v36",
|
| 300 |
-
"sha": "51db3b7fee2c",
|
| 301 |
-
"date": "2026-08-21",
|
| 302 |
-
"subject": "W39 share-field oid repair"
|
| 303 |
-
},
|
| 304 |
-
{
|
| 305 |
-
"version": "v35",
|
| 306 |
-
"sha": "0f642129c8f7",
|
| 307 |
-
"date": "2026-08-21",
|
| 308 |
-
"subject": "W39 visual QA fixture"
|
| 309 |
-
},
|
| 310 |
-
{
|
| 311 |
-
"version": "v34",
|
| 312 |
-
"sha": "cb4caa188e05",
|
| 313 |
-
"date": "2026-08-21",
|
| 314 |
-
"subject": "W39 Staging PostgreSQL egress controls"
|
| 315 |
-
},
|
| 316 |
-
{
|
| 317 |
-
"version": "v33",
|
| 318 |
-
"sha": "2b115e573d6a",
|
| 319 |
-
"date": "2026-08-21",
|
| 320 |
-
"subject": "W39: stop nonproduction Postgres background egress"
|
| 321 |
-
},
|
| 322 |
-
{
|
| 323 |
-
"version": "v32",
|
| 324 |
-
"sha": "b47b27ca7b2a",
|
| 325 |
-
"date": "2026-08-21",
|
| 326 |
-
"subject": "W39: repair Staging Postgres projection"
|
| 327 |
-
},
|
| 328 |
-
{
|
| 329 |
-
"version": "v31",
|
| 330 |
-
"sha": "8880a49a244e",
|
| 331 |
-
"date": "2026-08-21",
|
| 332 |
-
"subject": "v31: Wave 39 staging candidate"
|
| 333 |
-
},
|
| 334 |
-
{
|
| 335 |
-
"version": "v30",
|
| 336 |
-
"sha": "71776ca6c8df",
|
| 337 |
-
"date": "2026-08-20",
|
| 338 |
-
"subject": "v30: W39-T29 Member-owned View first-share hotfix"
|
| 339 |
-
},
|
| 340 |
-
{
|
| 341 |
-
"version": "v29",
|
| 342 |
-
"sha": "55ced5018ff2",
|
| 343 |
-
"date": "2026-08-18",
|
| 344 |
-
"subject": "v29: the build staging has been running since 2026-08-18 (55ced50)."
|
| 345 |
-
},
|
| 346 |
-
{
|
| 347 |
-
"version": "v28",
|
| 348 |
-
"sha": "2793e4ba53dc",
|
| 349 |
-
"date": "2026-08-18",
|
| 350 |
-
"subject": "release v28"
|
| 351 |
-
},
|
| 352 |
-
{
|
| 353 |
-
"version": "v27",
|
| 354 |
-
"sha": "c05a794837fc",
|
| 355 |
-
"date": "2026-08-18",
|
| 356 |
-
"subject": "release v27"
|
| 357 |
-
},
|
| 358 |
-
{
|
| 359 |
-
"version": "v25",
|
| 360 |
-
"sha": "bc248295ed66",
|
| 361 |
-
"date": "2026-08-14",
|
| 362 |
-
"subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)"
|
| 363 |
-
},
|
| 364 |
-
{
|
| 365 |
-
"version": "v26",
|
| 366 |
-
"sha": "6a642e139cfa",
|
| 367 |
-
"date": "2026-08-14",
|
| 368 |
-
"subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)"
|
| 369 |
-
},
|
| 370 |
-
{
|
| 371 |
-
"version": "v24",
|
| 372 |
-
"sha": "514d8292ce73",
|
| 373 |
-
"date": "2026-08-09",
|
| 374 |
-
"subject": "release v24"
|
| 375 |
-
},
|
| 376 |
-
{
|
| 377 |
-
"version": "v23",
|
| 378 |
-
"sha": "44cc28380fca",
|
| 379 |
-
"date": "2026-08-09",
|
| 380 |
-
"subject": "release v23"
|
| 381 |
-
},
|
| 382 |
-
{
|
| 383 |
-
"version": "v22",
|
| 384 |
-
"sha": "f14e199ff32f",
|
| 385 |
-
"date": "2026-08-09",
|
| 386 |
-
"subject": "release v22"
|
| 387 |
-
},
|
| 388 |
-
{
|
| 389 |
-
"version": "v21",
|
| 390 |
-
"sha": "5fdffce8b558",
|
| 391 |
-
"date": "2026-08-09",
|
| 392 |
-
"subject": "release v21"
|
| 393 |
-
},
|
| 394 |
-
{
|
| 395 |
-
"version": "v20",
|
| 396 |
-
"sha": "442e4a1430a7",
|
| 397 |
-
"date": "2026-08-09",
|
| 398 |
-
"subject": "release v20"
|
| 399 |
-
},
|
| 400 |
-
{
|
| 401 |
-
"version": "v19",
|
| 402 |
-
"sha": "94cb4a4dfdfe",
|
| 403 |
-
"date": "2026-08-09",
|
| 404 |
-
"subject": "release v19"
|
| 405 |
-
},
|
| 406 |
-
{
|
| 407 |
-
"version": "v18",
|
| 408 |
-
"sha": "0106ef949805",
|
| 409 |
-
"date": "2026-08-09",
|
| 410 |
-
"subject": "release v18"
|
| 411 |
-
},
|
| 412 |
-
{
|
| 413 |
-
"version": "v17",
|
| 414 |
-
"sha": "308240cc75e8",
|
| 415 |
-
"date": "2026-08-09",
|
| 416 |
-
"subject": "release v17"
|
| 417 |
-
},
|
| 418 |
-
{
|
| 419 |
-
"version": "v16",
|
| 420 |
-
"sha": "a804df54430f",
|
| 421 |
-
"date": "2026-08-09",
|
| 422 |
-
"subject": "release v16"
|
| 423 |
-
},
|
| 424 |
-
{
|
| 425 |
-
"version": "v15",
|
| 426 |
-
"sha": "6b84d7264c81",
|
| 427 |
-
"date": "2026-08-09",
|
| 428 |
-
"subject": "release v15"
|
| 429 |
-
},
|
| 430 |
-
{
|
| 431 |
-
"version": "v14",
|
| 432 |
-
"sha": "bee5eaf5b5cb",
|
| 433 |
-
"date": "2026-08-07",
|
| 434 |
-
"subject": "release v14"
|
| 435 |
-
},
|
| 436 |
-
{
|
| 437 |
-
"version": "v13",
|
| 438 |
-
"sha": "314c0e368b1d",
|
| 439 |
-
"date": "2026-08-06",
|
| 440 |
-
"subject": "release v13"
|
| 441 |
-
},
|
| 442 |
-
{
|
| 443 |
-
"version": "v12",
|
| 444 |
-
"sha": "4908f399785d",
|
| 445 |
-
"date": "2026-08-06",
|
| 446 |
-
"subject": "release v12"
|
| 447 |
-
},
|
| 448 |
-
{
|
| 449 |
-
"version": "v11",
|
| 450 |
-
"sha": "312661712b98",
|
| 451 |
-
"date": "2026-08-05",
|
| 452 |
-
"subject": "release v11"
|
| 453 |
-
},
|
| 454 |
-
{
|
| 455 |
-
"version": "v10",
|
| 456 |
-
"sha": "43f804287496",
|
| 457 |
-
"date": "2026-08-05",
|
| 458 |
-
"subject": "release v10"
|
| 459 |
-
},
|
| 460 |
-
{
|
| 461 |
-
"version": "v9",
|
| 462 |
-
"sha": "2e26f57396b1",
|
| 463 |
-
"date": "2026-08-05",
|
| 464 |
-
"subject": "release v9"
|
| 465 |
-
},
|
| 466 |
-
{
|
| 467 |
-
"version": "v8",
|
| 468 |
-
"sha": "8073b3ecbac3",
|
| 469 |
-
"date": "2026-08-05",
|
| 470 |
-
"subject": "release v8"
|
| 471 |
-
},
|
| 472 |
-
{
|
| 473 |
-
"version": "v7",
|
| 474 |
-
"sha": "b1c5a7e9b848",
|
| 475 |
-
"date": "2026-08-05",
|
| 476 |
-
"subject": "release v7"
|
| 477 |
-
},
|
| 478 |
-
{
|
| 479 |
-
"version": "v6",
|
| 480 |
-
"sha": "a344020f114a",
|
| 481 |
-
"date": "2026-08-05",
|
| 482 |
-
"subject": "release v6"
|
| 483 |
-
},
|
| 484 |
-
{
|
| 485 |
-
"version": "v5",
|
| 486 |
-
"sha": "c0b0fbb40a7b",
|
| 487 |
-
"date": "2026-08-05",
|
| 488 |
-
"subject": "release v5"
|
| 489 |
-
},
|
| 490 |
-
{
|
| 491 |
-
"version": "v4",
|
| 492 |
-
"sha": "de5037ff1c8f",
|
| 493 |
-
"date": "2026-08-05",
|
| 494 |
-
"subject": "release v4"
|
| 495 |
-
},
|
| 496 |
-
{
|
| 497 |
-
"version": "v3",
|
| 498 |
-
"sha": "6b9fa62d93e8",
|
| 499 |
-
"date": "2026-08-05",
|
| 500 |
-
"subject": "release v3"
|
| 501 |
-
},
|
| 502 |
-
{
|
| 503 |
-
"version": "v2",
|
| 504 |
-
"sha": "fd0586199858",
|
| 505 |
-
"date": "2026-08-04",
|
| 506 |
-
"subject": "release v2"
|
| 507 |
-
},
|
| 508 |
-
{
|
| 509 |
-
"version": "v1",
|
| 510 |
-
"sha": "5b4e2c4bc3f3",
|
| 511 |
-
"date": "2026-08-04",
|
| 512 |
-
"subject": "release v1"
|
| 513 |
}
|
| 514 |
]
|
| 515 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "v86 (8587070fe)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
| 6 |
+
"sha": "8587070fe",
|
| 7 |
+
"date": "2026-11-10",
|
| 8 |
+
"subject": "release v86"
|
| 9 |
},
|
| 10 |
{
|
| 11 |
"version": "v85",
|
| 12 |
+
"sha": "b36dd4992",
|
| 13 |
"date": "2026-10-25",
|
| 14 |
"subject": "Merge remote-tracking branch 'origin/master' into hfstage/sdlc79-release"
|
| 15 |
},
|
| 16 |
{
|
| 17 |
"version": "v84",
|
| 18 |
+
"sha": "7bb3a2e09",
|
| 19 |
"date": "2026-10-24",
|
| 20 |
"subject": "release(sdlc78): v84 QA records the staging build change and what v84 excludes"
|
| 21 |
},
|
| 22 |
{
|
| 23 |
"version": "v83",
|
| 24 |
+
"sha": "9b7471968",
|
| 25 |
"date": "2026-10-21",
|
| 26 |
"subject": "release v83"
|
| 27 |
},
|
| 28 |
{
|
| 29 |
"version": "v82",
|
| 30 |
+
"sha": "9c61b1ee9",
|
| 31 |
"date": "2026-10-17",
|
| 32 |
"subject": "release v82"
|
| 33 |
},
|
| 34 |
{
|
| 35 |
"version": "v81",
|
| 36 |
+
"sha": "e141ccc85",
|
| 37 |
"date": "2026-10-15",
|
| 38 |
"subject": "release v81"
|
| 39 |
},
|
| 40 |
{
|
| 41 |
"version": "v80",
|
| 42 |
+
"sha": "869b8fcc6",
|
| 43 |
"date": "2026-10-13",
|
| 44 |
"subject": "release v80"
|
| 45 |
},
|
| 46 |
{
|
| 47 |
"version": "v79",
|
| 48 |
+
"sha": "406417b8d",
|
| 49 |
"date": "2026-10-13",
|
| 50 |
"subject": "release v79"
|
| 51 |
},
|
| 52 |
{
|
| 53 |
"version": "v78",
|
| 54 |
+
"sha": "aad6561d6",
|
| 55 |
"date": "2026-10-09",
|
| 56 |
"subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs"
|
| 57 |
},
|
| 58 |
{
|
| 59 |
"version": "v77",
|
| 60 |
+
"sha": "e741e5ca6",
|
| 61 |
"date": "2026-10-05",
|
| 62 |
"subject": "release v77"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
}
|
| 64 |
]
|
| 65 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
v86 (8587070fe)
|
api/ads_intent_groups.py
CHANGED
|
@@ -4,12 +4,14 @@ import json
|
|
| 4 |
import os
|
| 5 |
from decimal import Decimal
|
| 6 |
|
|
|
|
|
|
|
| 7 |
NOT_GROUPED = "Not grouped"
|
| 8 |
TIMEOUT_SECONDS = 15
|
| 9 |
|
| 10 |
|
| 11 |
def _scrub(value):
|
| 12 |
-
return
|
| 13 |
|
| 14 |
|
| 15 |
def _cost(terms):
|
|
@@ -46,18 +48,21 @@ def _parse(answer, terms):
|
|
| 46 |
return groups
|
| 47 |
|
| 48 |
|
| 49 |
-
def _ask(messages, tools):
|
| 50 |
"""Use the flow provider ladder with a bounded wait on each transport."""
|
| 51 |
import ai_review
|
| 52 |
import providers
|
| 53 |
import requests
|
|
|
|
| 54 |
problems = []
|
| 55 |
for p in ai_review.flow_providers():
|
|
|
|
| 56 |
try:
|
| 57 |
if p["shape"] == "anthropic":
|
| 58 |
req = providers.anthropic_request(model=p["model"], key=os.environ[p["env"]].strip(),
|
| 59 |
system=None, messages=messages, tools=tools, max_tokens=4096, tool_choice="required")
|
| 60 |
status, body, _transport = providers.anthropic_send(req, timeout=TIMEOUT_SECONDS)
|
|
|
|
| 61 |
if status != 200:
|
| 62 |
problems.append(providers.refusal_sentence(p["name"], status, json.dumps(body)[:600]))
|
| 63 |
continue
|
|
@@ -69,10 +74,16 @@ def _ask(messages, tools):
|
|
| 69 |
headers={"Authorization": "Bearer " + os.environ[p["env"]].strip()},
|
| 70 |
json={"model": p["model"], "messages": messages, "tools": tools,
|
| 71 |
"tool_choice": "required", "temperature": 0, "max_tokens": 4096})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
if response.status_code != 200:
|
| 73 |
problems.append(providers.refusal_sentence(p["name"], response.status_code, response.text[:600]))
|
| 74 |
continue
|
| 75 |
-
calls =
|
| 76 |
if len(calls) != 1 or calls[0]["function"]["name"] != "group_intents":
|
| 77 |
raise ValueError("The model did not return one intent-group answer.")
|
| 78 |
answer = json.loads(calls[0]["function"]["arguments"])
|
|
@@ -81,10 +92,16 @@ def _ask(messages, tools):
|
|
| 81 |
raise ValueError("The model returned an unreadable intent-group answer.") from None
|
| 82 |
except Exception as exc:
|
| 83 |
problems.append(f"{p['name']}: {type(exc).__name__}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
raise RuntimeError(_scrub(" ".join(problems)) or "No intent-group provider is configured.")
|
| 85 |
|
| 86 |
|
| 87 |
-
def group(terms, chat=None):
|
| 88 |
"""Return each term once, with omitted or failed terms under Not grouped and a reason."""
|
| 89 |
terms = list(terms)
|
| 90 |
if len({t["id"] for t in terms}) != len(terms):
|
|
@@ -102,7 +119,8 @@ def group(terms, chat=None):
|
|
| 102 |
"Return group_intents. Leave a term out if its intent is unknown.")},
|
| 103 |
{"role": "user", "content": json.dumps([{"id": t["id"], "term": t["term"]} for t in terms])}]
|
| 104 |
try:
|
| 105 |
-
|
|
|
|
| 106 |
return {"state": "ready", "reason": None, "groups": groups, "total_spend": _cost(terms)}
|
| 107 |
except Exception as exc:
|
| 108 |
reason = _scrub(f"Intent grouping failed: {type(exc).__name__}: {exc}")
|
|
|
|
| 4 |
import os
|
| 5 |
from decimal import Decimal
|
| 6 |
|
| 7 |
+
import dash_norm
|
| 8 |
+
|
| 9 |
NOT_GROUPED = "Not grouped"
|
| 10 |
TIMEOUT_SECONDS = 15
|
| 11 |
|
| 12 |
|
| 13 |
def _scrub(value):
|
| 14 |
+
return dash_norm.plain(value).strip()
|
| 15 |
|
| 16 |
|
| 17 |
def _cost(terms):
|
|
|
|
| 48 |
return groups
|
| 49 |
|
| 50 |
|
| 51 |
+
def _ask(messages, tools, *, st=None, user=""):
|
| 52 |
"""Use the flow provider ladder with a bounded wait on each transport."""
|
| 53 |
import ai_review
|
| 54 |
import providers
|
| 55 |
import requests
|
| 56 |
+
import usage_ledger
|
| 57 |
problems = []
|
| 58 |
for p in ai_review.flow_providers():
|
| 59 |
+
body, received = None, False
|
| 60 |
try:
|
| 61 |
if p["shape"] == "anthropic":
|
| 62 |
req = providers.anthropic_request(model=p["model"], key=os.environ[p["env"]].strip(),
|
| 63 |
system=None, messages=messages, tools=tools, max_tokens=4096, tool_choice="required")
|
| 64 |
status, body, _transport = providers.anthropic_send(req, timeout=TIMEOUT_SECONDS)
|
| 65 |
+
received = True
|
| 66 |
if status != 200:
|
| 67 |
problems.append(providers.refusal_sentence(p["name"], status, json.dumps(body)[:600]))
|
| 68 |
continue
|
|
|
|
| 74 |
headers={"Authorization": "Bearer " + os.environ[p["env"]].strip()},
|
| 75 |
json={"model": p["model"], "messages": messages, "tools": tools,
|
| 76 |
"tool_choice": "required", "temperature": 0, "max_tokens": 4096})
|
| 77 |
+
received = True
|
| 78 |
+
try:
|
| 79 |
+
body = response.json()
|
| 80 |
+
except ValueError:
|
| 81 |
+
if response.status_code == 200:
|
| 82 |
+
raise
|
| 83 |
if response.status_code != 200:
|
| 84 |
problems.append(providers.refusal_sentence(p["name"], response.status_code, response.text[:600]))
|
| 85 |
continue
|
| 86 |
+
calls = body["choices"][0]["message"].get("tool_calls") or []
|
| 87 |
if len(calls) != 1 or calls[0]["function"]["name"] != "group_intents":
|
| 88 |
raise ValueError("The model did not return one intent-group answer.")
|
| 89 |
answer = json.loads(calls[0]["function"]["arguments"])
|
|
|
|
| 92 |
raise ValueError("The model returned an unreadable intent-group answer.") from None
|
| 93 |
except Exception as exc:
|
| 94 |
problems.append(f"{p['name']}: {type(exc).__name__}")
|
| 95 |
+
finally:
|
| 96 |
+
if received:
|
| 97 |
+
measured = body if isinstance(body, dict) else {}
|
| 98 |
+
ins, outs = usage_ledger.tokens_from(measured)
|
| 99 |
+
usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs,
|
| 100 |
+
total=usage_ledger.total_from(measured), st=st, user=user)
|
| 101 |
raise RuntimeError(_scrub(" ".join(problems)) or "No intent-group provider is configured.")
|
| 102 |
|
| 103 |
|
| 104 |
+
def group(terms, chat=None, *, st=None, user=""):
|
| 105 |
"""Return each term once, with omitted or failed terms under Not grouped and a reason."""
|
| 106 |
terms = list(terms)
|
| 107 |
if len({t["id"] for t in terms}) != len(terms):
|
|
|
|
| 119 |
"Return group_intents. Leave a term out if its intent is unknown.")},
|
| 120 |
{"role": "user", "content": json.dumps([{"id": t["id"], "term": t["term"]} for t in terms])}]
|
| 121 |
try:
|
| 122 |
+
answer = chat(messages, tools) if chat else _ask(messages, tools, st=st, user=user)
|
| 123 |
+
groups = _parse(answer, terms)
|
| 124 |
return {"state": "ready", "reason": None, "groups": groups, "total_spend": _cost(terms)}
|
| 125 |
except Exception as exc:
|
| 126 |
reason = _scrub(f"Intent grouping failed: {type(exc).__name__}: {exc}")
|
api/agent_directory.py
CHANGED
|
@@ -166,11 +166,28 @@ def _filter_sentences(tree, seen, label):
|
|
| 166 |
return [LIMITED if p is None else prefix + p for p in parts]
|
| 167 |
|
| 168 |
|
| 169 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
out = []
|
| 171 |
labels = users.allowed_bus_labels(rec)
|
| 172 |
if "All" not in labels and labels:
|
| 173 |
-
out.append("
|
| 174 |
agent = perms.scope_agent(rec)
|
| 175 |
if agent:
|
| 176 |
out.append(f"Sales agent is {agent}")
|
|
@@ -210,7 +227,7 @@ def _entry(session, uname, rec, ctx):
|
|
| 210 |
if unnamed:
|
| 211 |
named.append(f"{unnamed} database{'' if unnamed == 1 else 's'} you cannot open")
|
| 212 |
if not admin and not perm_scope.is_migrated(principal):
|
| 213 |
-
limits = _legacy_limits(rec) + limits
|
| 214 |
return {"username": str(rec.get("username_original") or uname),
|
| 215 |
"name": str(rec.get("name") or uname),
|
| 216 |
"role": "Administrator" if admin else "Member",
|
|
|
|
| 166 |
return [LIMITED if p is None else prefix + p for p in parts]
|
| 167 |
|
| 168 |
|
| 169 |
+
#: The Odoo field a legacy team scope narrows, and Odoo's own name for it (R66).
|
| 170 |
+
_TEAM_TABLE, _TEAM_FIELD, _TEAM_LABEL = "ut_odoo_orders", "team", "Sales team"
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _team_label(st, caller):
|
| 174 |
+
"""The caller-visible team label, with Odoo's name for an absent field."""
|
| 175 |
+
try:
|
| 176 |
+
from core import user_tables
|
| 177 |
+
fields = (user_tables.get_one(_TEAM_TABLE, st) or {}).get("fields") or ()
|
| 178 |
+
field = next((f for f in fields if isinstance(f, dict) and f.get("key") == _TEAM_FIELD), {})
|
| 179 |
+
if field:
|
| 180 |
+
return _name_field(_fields(caller, _TEAM_TABLE, st), _TEAM_FIELD)
|
| 181 |
+
return _TEAM_LABEL
|
| 182 |
+
except Exception: # noqa: BLE001
|
| 183 |
+
return _TEAM_LABEL
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _legacy_limits(rec, st=None, caller=None):
|
| 187 |
out = []
|
| 188 |
labels = users.allowed_bus_labels(rec)
|
| 189 |
if "All" not in labels and labels:
|
| 190 |
+
out.append(f"{_team_label(st, caller)} is " + " or ".join(labels))
|
| 191 |
agent = perms.scope_agent(rec)
|
| 192 |
if agent:
|
| 193 |
out.append(f"Sales agent is {agent}")
|
|
|
|
| 227 |
if unnamed:
|
| 228 |
named.append(f"{unnamed} database{'' if unnamed == 1 else 's'} you cannot open")
|
| 229 |
if not admin and not perm_scope.is_migrated(principal):
|
| 230 |
+
limits = _legacy_limits(rec, st, caller) + limits
|
| 231 |
return {"username": str(rec.get("username_original") or uname),
|
| 232 |
"name": str(rec.get("name") or uname),
|
| 233 |
"role": "Administrator" if admin else "Member",
|
api/amazon_quotes_loop.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Price and rank every rival once a day per tenant in the sync worker, outside the sync pass (C7).
|
| 2 |
+
|
| 3 |
+
The loop holds the SP-API client and passes its reads into `core.amazon_rivals.run_quotes`, because
|
| 4 |
+
core imports no harness module. It never joins `amazon_store.AFTER_SYNC` and never runs as an HF job.
|
| 5 |
+
"""
|
| 6 |
+
from contextlib import contextmanager
|
| 7 |
+
import importlib
|
| 8 |
+
import logging
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import threading
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
JOB = "amazon-rival-quotes"
|
| 14 |
+
#: The loop asks each tenant whether its daily run is due this often.
|
| 15 |
+
POLL_SECONDS = 600
|
| 16 |
+
|
| 17 |
+
#: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
|
| 18 |
+
#: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
|
| 19 |
+
#: Command Center patches the competitor cells of the rows those ASINs move (S82-T89).
|
| 20 |
+
AFTER_QUOTES = ("modules.amazon_profit:on_quotes", "modules.competitor_watch:on_quotes")
|
| 21 |
+
|
| 22 |
+
_log = logging.getLogger(__name__)
|
| 23 |
+
_missing = False
|
| 24 |
+
_unavailable = set()
|
| 25 |
+
_thread = None
|
| 26 |
+
_stop = None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _say(message):
|
| 30 |
+
print(f"[{JOB}] {message}", flush=True)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def readers(fields):
|
| 34 |
+
"""The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs."""
|
| 35 |
+
from harness import amazon_client as cx
|
| 36 |
+
return {"offers": lambda asins: cx.item_offers(fields, asins),
|
| 37 |
+
# The pricing sync draws this same `competitive_summary` pacer in this worker process.
|
| 38 |
+
"summaries": lambda asins: cx.competitive_summaries(fields, asins),
|
| 39 |
+
"ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@contextmanager
|
| 43 |
+
def _mirror(runtime):
|
| 44 |
+
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
|
| 45 |
+
from harness import runtime as R
|
| 46 |
+
if not Path(runtime.datastore_path).is_file():
|
| 47 |
+
yield None
|
| 48 |
+
return
|
| 49 |
+
cur = R.mirror_cursor(runtime, require_ready=False)
|
| 50 |
+
try:
|
| 51 |
+
yield cur
|
| 52 |
+
finally:
|
| 53 |
+
cur.close()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _after_quotes(tenant, changed):
|
| 57 |
+
for target in AFTER_QUOTES:
|
| 58 |
+
try:
|
| 59 |
+
module_name, _, name = target.partition(":")
|
| 60 |
+
getattr(importlib.import_module(module_name), name)(tenant, changed)
|
| 61 |
+
_say(f"tenant={tenant} after-quotes callback {target} completed")
|
| 62 |
+
except Exception as exc: # noqa: BLE001
|
| 63 |
+
_say(f"tenant={tenant} after-quotes callback {target} failed ({type(exc).__name__}); "
|
| 64 |
+
"the run result is unchanged.")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
|
| 68 |
+
"""Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
|
| 69 |
+
from core import amazon_rivals as rivals
|
| 70 |
+
try:
|
| 71 |
+
if not rivals.due(runtime, now=now):
|
| 72 |
+
return None
|
| 73 |
+
except rivals.BackendUnavailable:
|
| 74 |
+
if tenant not in _unavailable:
|
| 75 |
+
_unavailable.add(tenant)
|
| 76 |
+
_say(f"tenant={tenant} has no PostgreSQL store, so no rival quotes run")
|
| 77 |
+
return None
|
| 78 |
+
if fields is None:
|
| 79 |
+
from harness import amazon_store
|
| 80 |
+
fields = amazon_store.credential(tenant, runtime)
|
| 81 |
+
if not fields:
|
| 82 |
+
return None
|
| 83 |
+
result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
|
| 84 |
+
seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
|
| 85 |
+
log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)
|
| 86 |
+
if result["cause"] is None:
|
| 87 |
+
_after_quotes(tenant, result["changed"])
|
| 88 |
+
return result
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def run_once(*, for_each_tenant=None, now=None):
|
| 92 |
+
"""Admit one independent per-tenant job without waiting for its body."""
|
| 93 |
+
global _missing
|
| 94 |
+
if for_each_tenant is None:
|
| 95 |
+
try:
|
| 96 |
+
from tenant_loops import for_each_tenant
|
| 97 |
+
except ImportError:
|
| 98 |
+
if not _missing:
|
| 99 |
+
_log.warning("Rival quotes unavailable: tenant_loops is not installed.")
|
| 100 |
+
_missing = True
|
| 101 |
+
return {}
|
| 102 |
+
_missing = False
|
| 103 |
+
return for_each_tenant(lambda tenant, runtime: run_tenant(tenant, runtime, now=now),
|
| 104 |
+
name=JOB, logger=_log, kind="heavy")
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def start():
|
| 108 |
+
"""Start the daily loop only when process_role admits its heavy coordinator (the sync worker)."""
|
| 109 |
+
global _thread, _stop
|
| 110 |
+
import process_role
|
| 111 |
+
if _thread is not None and _thread.is_alive():
|
| 112 |
+
return _thread
|
| 113 |
+
stop_event = threading.Event()
|
| 114 |
+
_stop = stop_event
|
| 115 |
+
|
| 116 |
+
def run():
|
| 117 |
+
while not stop_event.is_set():
|
| 118 |
+
run_once()
|
| 119 |
+
stop_event.wait(POLL_SECONDS)
|
| 120 |
+
|
| 121 |
+
# A literal name: verify_background_loops classifies a job by it.
|
| 122 |
+
_thread = process_role.start("amazon-rival-quotes", "heavy", run)
|
| 123 |
+
return _thread
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def stop():
|
| 127 |
+
if _stop is not None:
|
| 128 |
+
_stop.set()
|
api/amazon_relational.py
CHANGED
|
@@ -734,7 +734,7 @@ def _read_through(cur, key, count, *, columns=None):
|
|
| 734 |
if key in READ_THROUGH or count > ut.MAX_ROWS:
|
| 735 |
return True
|
| 736 |
if not count:
|
| 737 |
-
return
|
| 738 |
names = [name for name, _dtype in (columns if columns is not None
|
| 739 |
else mirror_columns(cur, SOURCE[key]))]
|
| 740 |
sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
|
|
@@ -1106,6 +1106,13 @@ def plan(cur, rt=None):
|
|
| 1106 |
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 1107 |
except Exception: # noqa: BLE001
|
| 1108 |
continue # absent = never synced
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1109 |
if _read_through(cur, key, n):
|
| 1110 |
built[key] = {"readThrough": True,
|
| 1111 |
"fields": _fields_for(cur, key, read_through=True), "count": n}
|
|
@@ -1412,7 +1419,7 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1412 |
not_stored.clear()
|
| 1413 |
live_columns = {key: {field["key"] for field in table.get("fields", ())}
|
| 1414 |
for key, table in doc.items() if key in SOURCE and isinstance(table, dict)}
|
| 1415 |
-
#
|
| 1416 |
unborn = {key for key, _label, _fields, rows in plans
|
| 1417 |
if isinstance(rows, list) and not rows and not isinstance(doc.get(key), dict)}
|
| 1418 |
live_columns.update({key: {field["key"] for field in fields}
|
|
@@ -1506,6 +1513,12 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1506 |
saved = stored.get(row["_id"])
|
| 1507 |
if numeric and isinstance(saved, dict):
|
| 1508 |
saved.update({name: row[name] for name in numeric if name in row})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1509 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1510 |
# mutable display label or from the process having imported this module.
|
| 1511 |
doc[key]["connector"] = source_attribution(key)
|
|
|
|
| 734 |
if key in READ_THROUGH or count > ut.MAX_ROWS:
|
| 735 |
return True
|
| 736 |
if not count:
|
| 737 |
+
return key == PREFIX + "inventory"
|
| 738 |
names = [name for name, _dtype in (columns if columns is not None
|
| 739 |
else mirror_columns(cur, SOURCE[key]))]
|
| 740 |
sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
|
|
|
|
| 1106 |
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 1107 |
except Exception: # noqa: BLE001
|
| 1108 |
continue # absent = never synced
|
| 1109 |
+
if key == PREFIX + "inventory" and not n:
|
| 1110 |
+
# A measured empty inventory report remains a declared database with a source reason.
|
| 1111 |
+
status = amazon_store.inventory_table_status(cur)
|
| 1112 |
+
built[key] = {"readThrough": True,
|
| 1113 |
+
"fields": _fields_for(cur, key, read_through=True), "count": n,
|
| 1114 |
+
"sourceStatus": {name: status[name] for name in ("empty", "code", "reason")}}
|
| 1115 |
+
continue
|
| 1116 |
if _read_through(cur, key, n):
|
| 1117 |
built[key] = {"readThrough": True,
|
| 1118 |
"fields": _fields_for(cur, key, read_through=True), "count": n}
|
|
|
|
| 1419 |
not_stored.clear()
|
| 1420 |
live_columns = {key: {field["key"] for field in table.get("fields", ())}
|
| 1421 |
for key, table in doc.items() if key in SOURCE and isinstance(table, dict)}
|
| 1422 |
+
# Empty materialized snapshots reconcile existing definitions and create none.
|
| 1423 |
unborn = {key for key, _label, _fields, rows in plans
|
| 1424 |
if isinstance(rows, list) and not rows and not isinstance(doc.get(key), dict)}
|
| 1425 |
live_columns.update({key: {field["key"] for field in fields}
|
|
|
|
| 1513 |
saved = stored.get(row["_id"])
|
| 1514 |
if numeric and isinstance(saved, dict):
|
| 1515 |
saved.update({name: row[name] for name in numeric if name in row})
|
| 1516 |
+
if key == PREFIX + "inventory":
|
| 1517 |
+
status = rows.get("sourceStatus") if isinstance(rows, dict) else None
|
| 1518 |
+
if status is not None:
|
| 1519 |
+
doc[key]["sourceStatus"] = dict(status)
|
| 1520 |
+
else:
|
| 1521 |
+
doc[key].pop("sourceStatus", None)
|
| 1522 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 1523 |
# mutable display label or from the process having imported this module.
|
| 1524 |
doc[key]["connector"] = source_attribution(key)
|
api/app_templates.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""C1: the shipped App template catalogue
|
| 2 |
|
| 3 |
The registry's template contracts and `core.external_tools`' templateKind enum derive from or
|
| 4 |
mirror `TEMPLATES`; `verify_app_templates.py` holds all three name for name.
|
|
@@ -40,6 +40,10 @@ def source_connectors(connector, sources, optional_sources=(), optional_connecto
|
|
| 40 |
|
| 41 |
def validate_connectors(row):
|
| 42 |
"""Reject unordered, unknown, duplicated or incomplete connector declarations for any row."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
connectors = row.get("connectors")
|
| 44 |
if (not isinstance(connectors, (tuple, list)) or not connectors
|
| 45 |
or connectors[0] != row.get("connector")
|
|
@@ -78,11 +82,12 @@ TEMPLATES = (
|
|
| 78 |
"module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
|
| 79 |
"proposes": False},
|
| 80 |
_amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
|
| 81 |
-
"Acquire", ("ut_amzn_brand_analytics",), proposes=False),
|
| 82 |
_amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
|
| 83 |
"Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
|
| 84 |
"ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
|
| 85 |
-
"ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False
|
|
|
|
| 86 |
_amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
|
| 87 |
"Acquire", ("ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns", "ut_amzn_ads_sd_campaigns",
|
| 88 |
"ut_amzn_economics"), connector="amazon_ads", proposes=False),
|
|
@@ -100,11 +105,15 @@ TEMPLATES = (
|
|
| 100 |
"Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings"),
|
| 101 |
optional_connectors=("odoo",)),
|
| 102 |
_amazon("returns_refunds", "Returns and refunds", "See why buyers return and track each fix.",
|
| 103 |
-
"Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_fba_shipped_sales", "
|
| 104 |
-
"
|
|
|
|
|
|
|
| 105 |
optional_connectors=("odoo",)),
|
| 106 |
_amazon("repeat_buyers", "Repeat buyers", "See who buys again and what they buy together.",
|
| 107 |
-
"Retain", ("ut_amzn_repeat_purchase", "ut_amzn_market_basket"), proposes=False
|
|
|
|
|
|
|
| 108 |
_amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
|
| 109 |
"Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions"),
|
| 110 |
optional_connectors=("odoo",)),
|
|
@@ -115,10 +124,15 @@ TEMPLATES = (
|
|
| 115 |
"ut_amzn_fba_shipped_sales", "ut_amzn_mfn_returns", "ut_amzn_settlements",
|
| 116 |
"ut_amzn_transactions", "ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns",
|
| 117 |
"ut_amzn_ads_sd_campaigns"), optional_sources=("product_data",), proposes=False),
|
| 118 |
-
#
|
| 119 |
# source: a cold mirror must not refuse an App whose rows live in the store.
|
| 120 |
-
_amazon("amazon_profit", "
|
| 121 |
"Create value", (), optional_connectors=("odoo",), proposes=False),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
)
|
| 123 |
if not all(validate_connectors(row) for row in TEMPLATES):
|
| 124 |
raise ValueError("An App template omits a source connector.")
|
|
|
|
| 1 |
+
"""C1: the shipped App template catalogue and each template's source or permission contract.
|
| 2 |
|
| 3 |
The registry's template contracts and `core.external_tools`' templateKind enum derive from or
|
| 4 |
mirror `TEMPLATES`; `verify_app_templates.py` holds all three name for name.
|
|
|
|
| 40 |
|
| 41 |
def validate_connectors(row):
|
| 42 |
"""Reject unordered, unknown, duplicated or incomplete connector declarations for any row."""
|
| 43 |
+
if row.get("kind") == "alerts":
|
| 44 |
+
return (row.get("connector") is None and row.get("permission") == "automation"
|
| 45 |
+
and all(row.get(key) == () for key in
|
| 46 |
+
("connectors", "sources", "optional_sources", "optional_connectors")))
|
| 47 |
connectors = row.get("connectors")
|
| 48 |
if (not isinstance(connectors, (tuple, list)) or not connectors
|
| 49 |
or connectors[0] != row.get("connector")
|
|
|
|
| 82 |
"module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
|
| 83 |
"proposes": False},
|
| 84 |
_amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
|
| 85 |
+
"Acquire", ("ut_amzn_brand_analytics", "ut_amzn_listings"), proposes=False),
|
| 86 |
_amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
|
| 87 |
"Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
|
| 88 |
"ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
|
| 89 |
+
"ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False,
|
| 90 |
+
optional_sources=("ut_amzn_search_query_performance", "ut_amzn_listing_items")),
|
| 91 |
_amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
|
| 92 |
"Acquire", ("ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns", "ut_amzn_ads_sd_campaigns",
|
| 93 |
"ut_amzn_economics"), connector="amazon_ads", proposes=False),
|
|
|
|
| 105 |
"Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings"),
|
| 106 |
optional_connectors=("odoo",)),
|
| 107 |
_amazon("returns_refunds", "Returns and refunds", "See why buyers return and track each fix.",
|
| 108 |
+
"Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_fba_shipped_sales", "ut_amzn_mfn_returns",
|
| 109 |
+
"ut_amzn_settlements", "ut_amzn_transactions", "ut_amzn_order_items"),
|
| 110 |
+
optional_sources=("ut_amzn_fba_inventory", "ut_amzn_removal_orders",
|
| 111 |
+
"ut_amzn_removal_shipments", "ut_amzn_storage_fees"),
|
| 112 |
optional_connectors=("odoo",)),
|
| 113 |
_amazon("repeat_buyers", "Repeat buyers", "See who buys again and what they buy together.",
|
| 114 |
+
"Retain", ("ut_amzn_repeat_purchase", "ut_amzn_market_basket"), proposes=False,
|
| 115 |
+
optional_sources=("ut_amzn_replenishment", "ut_amzn_subscribe_save_performance",
|
| 116 |
+
"ut_amzn_fba_shipped_sales", "ut_amzn_listings")),
|
| 117 |
_amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
|
| 118 |
"Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions"),
|
| 119 |
optional_connectors=("odoo",)),
|
|
|
|
| 124 |
"ut_amzn_fba_shipped_sales", "ut_amzn_mfn_returns", "ut_amzn_settlements",
|
| 125 |
"ut_amzn_transactions", "ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns",
|
| 126 |
"ut_amzn_ads_sd_campaigns"), optional_sources=("product_data",), proposes=False),
|
| 127 |
+
# Project management reads its own module tables (ut_amazon_*), so it declares no
|
| 128 |
# source: a cold mirror must not refuse an App whose rows live in the store.
|
| 129 |
+
_amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.",
|
| 130 |
"Create value", (), optional_connectors=("odoo",), proposes=False),
|
| 131 |
+
{"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.",
|
| 132 |
+
"connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"],
|
| 133 |
+
"sources": (), "optional_sources": (), "optional_connectors": (), "process": "None",
|
| 134 |
+
"mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts",
|
| 135 |
+
"tools": (), "proposes": False},
|
| 136 |
)
|
| 137 |
if not all(validate_connectors(row) for row in TEMPLATES):
|
| 138 |
raise ValueError("An App template omits a source connector.")
|
api/application_registry.py
CHANGED
|
@@ -183,8 +183,20 @@ def _connector_gate(connector):
|
|
| 183 |
return gate
|
| 184 |
|
| 185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
# Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
|
| 187 |
_TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
|
|
|
|
|
|
|
| 188 |
else _connector_gate(contract["connector"])
|
| 189 |
for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
|
| 190 |
#: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
|
|
@@ -552,7 +564,8 @@ def load(runtime, *, strict=False):
|
|
| 552 |
_AVAILABILITY_MESSAGES = {
|
| 553 |
"router_unavailable": "This App cannot open because its routes are unavailable.",
|
| 554 |
"no_grant": "Your access limit prevents this App from reading its source.",
|
| 555 |
-
"not_connected": "Connect the App's connector to open this App.",
|
|
|
|
| 556 |
"no_rows": "This App has no source rows yet.",
|
| 557 |
"source_unavailable": "This App cannot read its source right now.",
|
| 558 |
"held": "This App is not available yet.",
|
|
@@ -565,7 +578,7 @@ def availability_reason(state, connector=None):
|
|
| 565 |
if state == "not_connected":
|
| 566 |
label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector)
|
| 567 |
if label:
|
| 568 |
-
message = f"Connect {label} to open this App."
|
| 569 |
return {"state": state, "message": message,
|
| 570 |
"lock": "Access limit" if state == "no_grant" else None}
|
| 571 |
|
|
@@ -628,13 +641,121 @@ def _readable_template_app(session, app_id):
|
|
| 628 |
catalogue = _app_templates.BY_KIND.get(str(kind or ""))
|
| 629 |
if not catalogue or not catalogue["view"]:
|
| 630 |
raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
|
|
|
|
|
|
|
| 631 |
return app, catalogue
|
| 632 |
|
| 633 |
|
| 634 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
from core import perm_scope
|
| 636 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 637 |
|
|
|
|
|
|
|
| 638 |
kind = app["templateKind"]
|
| 639 |
if _app_templates.held(kind):
|
| 640 |
return availability_reason("held")
|
|
@@ -643,28 +764,54 @@ def _availability_for_app(session, app, catalogue):
|
|
| 643 |
return mount
|
| 644 |
if not _app_templates.seams_ready(kind):
|
| 645 |
return availability_reason("router_unavailable")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 646 |
try:
|
| 647 |
sources = catalogue["sources"]
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 654 |
types, _scopes, failure = _entry_types(session.runtime, session.uname)
|
| 655 |
if failure is not None:
|
| 656 |
-
return
|
| 657 |
connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ())
|
| 658 |
if not connected and catalogue["connector"] == "odoo":
|
| 659 |
from routes_keychain import env_odoo_available
|
| 660 |
connected = env_odoo_available(session.runtime)
|
| 661 |
if not connected:
|
| 662 |
-
return
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 668 |
|
| 669 |
|
| 670 |
def availability(session, app_id):
|
|
@@ -785,7 +932,10 @@ def _user_wall(runtime, user):
|
|
| 785 |
record = (users.registry() or {}).get(user)
|
| 786 |
if not isinstance(record, dict) or record.get("active") is False:
|
| 787 |
return lambda _key: False
|
| 788 |
-
|
|
|
|
|
|
|
|
|
|
| 789 |
except Exception:
|
| 790 |
return lambda _key: False
|
| 791 |
|
|
@@ -1179,7 +1329,8 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
|
|
| 1179 |
mount = _mount_reason(row.get("templateKind"))
|
| 1180 |
if mount:
|
| 1181 |
row["availability"] = mount
|
| 1182 |
-
elif
|
|
|
|
| 1183 |
row["availability"] = availability_reason("no_grant")
|
| 1184 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 1185 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
|
@@ -1232,6 +1383,9 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1232 |
continue
|
| 1233 |
if _app_templates.held(app):
|
| 1234 |
continue
|
|
|
|
|
|
|
|
|
|
| 1235 |
if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
|
| 1236 |
continue
|
| 1237 |
if database:
|
|
@@ -1261,6 +1415,16 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1261 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 1262 |
|
| 1263 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1264 |
def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None):
|
| 1265 |
"""The App home read: stored records the caller owns or was shared, never a code projection."""
|
| 1266 |
return [{"id": app["id"], "name": app.get("name"),
|
|
@@ -1271,7 +1435,7 @@ def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_da
|
|
| 1271 |
"origin": "tenant",
|
| 1272 |
"compatibleDatabases": app.get("compatibleDatabases") or [],
|
| 1273 |
"openTarget": app.get("openTarget"),
|
| 1274 |
-
"navPlacement":
|
| 1275 |
"capabilities": app.get("capabilities"), "starred": app.get("starred"),
|
| 1276 |
"isDefault": app.get("isDefault")}
|
| 1277 |
for index, app in enumerate(list_apps(runtime, user, is_admin,
|
|
@@ -1450,6 +1614,9 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr
|
|
| 1450 |
"""Create one App. A template App is one live copy per tenant: a second add returns the first."""
|
| 1451 |
if getattr(runtime, "key", tenant) != tenant or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant:
|
| 1452 |
raise PermissionError("an Application cannot be created for another tenant")
|
|
|
|
|
|
|
|
|
|
| 1453 |
app = _build_application(
|
| 1454 |
tenant, user, payload, semantic_records=semantic_records, field_contracts=field_contracts,
|
| 1455 |
allowed_databases=allowed_databases, origin_door=origin_door, via=via)
|
|
@@ -1644,6 +1811,129 @@ def migrate_returns_split(runtime, tenant):
|
|
| 1644 |
return {"outcome": "migrated", "itemIds": copy.deepcopy(copied["itemIds"])}
|
| 1645 |
|
| 1646 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1647 |
def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
|
| 1648 |
"""R15: boot migrations, once per tenant, never from an Application read.
|
| 1649 |
|
|
@@ -1731,8 +2021,93 @@ def _ensure_route_seed(runtime, tenant):
|
|
| 1731 |
COMMAND_CENTER_MARKER = "commandCenterApplication"
|
| 1732 |
|
| 1733 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1734 |
def ensure_command_center_application(runtime, tenant):
|
| 1735 |
-
"""
|
| 1736 |
|
| 1737 |
A tenant without the table gets no marker, so a later boot seeds it once the table exists.
|
| 1738 |
"""
|
|
@@ -1742,8 +2117,12 @@ def ensure_command_center_application(runtime, tenant):
|
|
| 1742 |
from core import user_tables
|
| 1743 |
from modules.amazon_profit import TABLE_KEY
|
| 1744 |
|
|
|
|
|
|
|
| 1745 |
base = _clean_doc(_read_strict(runtime, _KEY))
|
| 1746 |
if COMMAND_CENTER_MARKER in (base.get("migrations") or {}):
|
|
|
|
|
|
|
| 1747 |
return {"outcome": "unchanged"}
|
| 1748 |
records = _record_documents(runtime)
|
| 1749 |
|
|
@@ -1777,7 +2156,7 @@ def ensure_command_center_application(runtime, tenant):
|
|
| 1777 |
return raw
|
| 1778 |
app = _build_application(
|
| 1779 |
tenant, "loopable",
|
| 1780 |
-
{"name": "
|
| 1781 |
"navPlacement": {"section": "app", "order": 0}})
|
| 1782 |
app["share"] = {"mode": "tenant", "users": []}
|
| 1783 |
doc["applications"][app["id"]] = app
|
|
@@ -1785,7 +2164,7 @@ def ensure_command_center_application(runtime, tenant):
|
|
| 1785 |
migrations[COMMAND_CENTER_MARKER] = {"at": write_execution.utcnow().isoformat(), **result}
|
| 1786 |
return doc
|
| 1787 |
if _update_document(runtime, apply) is False:
|
| 1788 |
-
raise ApplicationError("The
|
| 1789 |
return result
|
| 1790 |
|
| 1791 |
|
|
|
|
| 183 |
return gate
|
| 184 |
|
| 185 |
|
| 186 |
+
def _automation_offered(session):
|
| 187 |
+
"""Use Automation's current account and tenant wall without a connector requirement."""
|
| 188 |
+
try:
|
| 189 |
+
session.require("automation")
|
| 190 |
+
return True
|
| 191 |
+
except Exception as exc:
|
| 192 |
+
raise_if_resource_error(exc)
|
| 193 |
+
return False
|
| 194 |
+
|
| 195 |
+
|
| 196 |
# Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
|
| 197 |
_TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
|
| 198 |
+
else _automation_offered if kind == "alerts"
|
| 199 |
+
else _connector_gate("amazon") if kind == "amazon_operator"
|
| 200 |
else _connector_gate(contract["connector"])
|
| 201 |
for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
|
| 202 |
#: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
|
|
|
|
| 564 |
_AVAILABILITY_MESSAGES = {
|
| 565 |
"router_unavailable": "This App cannot open because its routes are unavailable.",
|
| 566 |
"no_grant": "Your access limit prevents this App from reading its source.",
|
| 567 |
+
"not_connected": "Connect the App's connector in Connectors to open this App.",
|
| 568 |
+
"refreshing": "This App's source is refreshing and opens when the refresh ends",
|
| 569 |
"no_rows": "This App has no source rows yet.",
|
| 570 |
"source_unavailable": "This App cannot read its source right now.",
|
| 571 |
"held": "This App is not available yet.",
|
|
|
|
| 578 |
if state == "not_connected":
|
| 579 |
label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector)
|
| 580 |
if label:
|
| 581 |
+
message = f"Connect {label} in Connectors to open this App."
|
| 582 |
return {"state": state, "message": message,
|
| 583 |
"lock": "Access limit" if state == "no_grant" else None}
|
| 584 |
|
|
|
|
| 641 |
catalogue = _app_templates.BY_KIND.get(str(kind or ""))
|
| 642 |
if not catalogue or not catalogue["view"]:
|
| 643 |
raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
|
| 644 |
+
if kind == "alerts":
|
| 645 |
+
session.require("automation")
|
| 646 |
return app, catalogue
|
| 647 |
|
| 648 |
|
| 649 |
+
#: R62: App kinds whose readers apply a source row and field wall to their own rows.
|
| 650 |
+
#: Any other kind refuses its App while such a wall applies, because it cannot narrow.
|
| 651 |
+
NARROWING_KINDS = frozenset({"account_health", "opportunity_finder", "ranking_diagnostic", "repeat_buyers"})
|
| 652 |
+
|
| 653 |
+
|
| 654 |
+
def app_access(user, runtime, app, sources):
|
| 655 |
+
"""R46, R62: the one source-wall answer for an App, its Dashboard and its readers.
|
| 656 |
+
|
| 657 |
+
Takes a user record, not a session, so a digest can ask for any recipient. Returns
|
| 658 |
+
`app` and `dashboard` ("ready" or refused), the Dashboard refusal's `status`, `code`
|
| 659 |
+
and one-line `message`, and `narrow`: per source, the row wall and hidden field keys.
|
| 660 |
+
"""
|
| 661 |
+
from core import perm_scope, perms, route_system, user_tables
|
| 662 |
+
from core.request_timing import remaining_budget
|
| 663 |
+
|
| 664 |
+
user = user if isinstance(user, dict) else {}
|
| 665 |
+
kind = app.get("templateKind") if isinstance(app, dict) else None
|
| 666 |
+
granted = may_read(app, user.get("username"), perms.is_admin(user))
|
| 667 |
+
refusal = []
|
| 668 |
+
|
| 669 |
+
def refuse(status, code, message):
|
| 670 |
+
refusal.append((status, code, message))
|
| 671 |
+
|
| 672 |
+
all_units = "All" in perms.allowed_bu_labels(user)
|
| 673 |
+
raw_units = user.get("bus", "all")
|
| 674 |
+
if raw_units != "all":
|
| 675 |
+
from core import users
|
| 676 |
+
by_label = {label: key for key, label in users.BU_LABELS.items()}
|
| 677 |
+
identities = []
|
| 678 |
+
valid_units = isinstance(raw_units, (tuple, list)) and bool(raw_units)
|
| 679 |
+
for unit in raw_units if valid_units else ():
|
| 680 |
+
if type(unit) is int and unit in users.BU_LABELS:
|
| 681 |
+
identities.append(unit)
|
| 682 |
+
elif isinstance(unit, str) and unit in by_label:
|
| 683 |
+
identities.append(by_label[unit])
|
| 684 |
+
elif unit != "All":
|
| 685 |
+
valid_units = False
|
| 686 |
+
all_units = bool(valid_units and identities) and "All" in perms.allowed_bu_labels(dict(user, bus=identities))
|
| 687 |
+
if not all_units:
|
| 688 |
+
refuse(403, "app_dashboard_scope", "The Dashboard needs access to every business unit.")
|
| 689 |
+
elif perms.scope_agent(user) is not None:
|
| 690 |
+
refuse(403, "app_dashboard_scope", "The Dashboard needs access to every salesperson's records.")
|
| 691 |
+
if not isinstance(sources, (tuple, list)) or not sources:
|
| 692 |
+
refuse(503, "app_dashboard_sources", "The Dashboard has no source to read.")
|
| 693 |
+
sources = ()
|
| 694 |
+
config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
|
| 695 |
+
enabled = config.get("modules", "all")
|
| 696 |
+
narrow = {}
|
| 697 |
+
for source in sources:
|
| 698 |
+
remaining_budget()
|
| 699 |
+
label = _source_label(source) or "a source database"
|
| 700 |
+
definition = user_tables.get_definition(source, st=runtime)
|
| 701 |
+
fields = definition.get("fields") if isinstance(definition, dict) else None
|
| 702 |
+
declared = perm_scope.entry(user, source) or {}
|
| 703 |
+
walled = bool(perm_scope.wall_declared(user, source, st=runtime, defn=definition))
|
| 704 |
+
rows = bool(declared.get("filter")) or bool(
|
| 705 |
+
fields and perm_scope.row_scope_applies(user, source, st=runtime, defn=definition))
|
| 706 |
+
hidden = set(declared.get("hiddenFields") or ()) | set(
|
| 707 |
+
perm_scope.hidden_keys(user, source, fields, st=runtime) if fields else ())
|
| 708 |
+
narrow[source] = {"rows": rows or (walled and not hidden), "hidden": tuple(sorted(map(str, hidden)))}
|
| 709 |
+
# The Dashboard totals the raw source, so it keeps the source grant and refuses any narrowing.
|
| 710 |
+
if (enabled != "all" and source not in (enabled or ())) or not perm_scope.may_read(user, source, st=runtime):
|
| 711 |
+
refuse(403, "app_dashboard_source", f"The Dashboard needs read access to {label}.")
|
| 712 |
+
elif not fields:
|
| 713 |
+
refuse(503, "app_dashboard_source", f"The Dashboard cannot read {label} right now.")
|
| 714 |
+
elif (declared.get("access") is False or walled or rows or hidden
|
| 715 |
+
or route_system.is_managed(definition) or route_system.is_managed(source)):
|
| 716 |
+
refuse(403, "app_dashboard_scope", f"The Dashboard needs {label} without a row or field limit.")
|
| 717 |
+
narrowed = any(wall["rows"] or wall["hidden"] for wall in narrow.values())
|
| 718 |
+
status, code, message = refusal[0] if refusal else (None, None, None)
|
| 719 |
+
return {"app": "ready" if granted and (kind in NARROWING_KINDS or not narrowed) else "no_grant",
|
| 720 |
+
"dashboard": "ready" if not refusal else "refused",
|
| 721 |
+
"status": status, "code": code, "message": message, "narrow": narrow}
|
| 722 |
+
|
| 723 |
+
|
| 724 |
+
def app_source_scope(user, runtime, source, *, app, records=None):
|
| 725 |
+
"""Return source fields and row identities after live admission of the named App."""
|
| 726 |
from core import perm_scope
|
| 727 |
+
# Importing the API adapter registers the core admission callback.
|
| 728 |
+
import routes_app_dashboard
|
| 729 |
+
fields, rows = perm_scope.scoped_app_source(
|
| 730 |
+
user, source, st=runtime, app_id=app.get("id") if isinstance(app, dict) else None, records=records)
|
| 731 |
+
identity = "__scope_key" if records is not None else "pid"
|
| 732 |
+
return {str(field["key"]) for field in fields if field.get("key")}, {
|
| 733 |
+
str(row[identity]) for row in rows if identity in row}
|
| 734 |
+
|
| 735 |
+
|
| 736 |
+
def readable_app(runtime, tenant, uname, is_admin, kind):
|
| 737 |
+
"""The caller's live readable App of `kind` in `tenant`, their own copy first, or None."""
|
| 738 |
+
apps = [row for _key, row in sorted(load(runtime)["applications"].items())
|
| 739 |
+
if isinstance(row, dict) and row.get("templateKind") == kind and row.get("tenant") == tenant
|
| 740 |
+
and row.get("retired") is not True and may_read(row, uname, is_admin)]
|
| 741 |
+
return next((row for row in apps if row.get("createdBy") == uname), apps[0] if apps else None)
|
| 742 |
+
|
| 743 |
+
|
| 744 |
+
def admit_app(session, app, sources, *, code):
|
| 745 |
+
"""R62: admit an App read on the App grant plus `app_access`, never the source open grant."""
|
| 746 |
+
from deps import err
|
| 747 |
+
if (session.tenant != getattr(session.runtime, "key", None)
|
| 748 |
+
or session.user.get("tenant", session.tenant) != session.tenant
|
| 749 |
+
or not isinstance(app, dict) or app.get("tenant") != session.tenant
|
| 750 |
+
or app.get("retired") is True or not may_read(app, session.uname, session.admin)):
|
| 751 |
+
raise err(404, f"{code}_app_not_found", "This App is unavailable or is not shared with you.")
|
| 752 |
+
access = app_access(session.user, session.runtime, app, sources)
|
| 753 |
+
if access["app"] != "ready":
|
| 754 |
+
raise err(403, f"{code}_limited", "This App cannot apply your access limit on its source.")
|
| 755 |
+
return access
|
| 756 |
|
| 757 |
+
|
| 758 |
+
def _availability_for_app(session, app, catalogue):
|
| 759 |
kind = app["templateKind"]
|
| 760 |
if _app_templates.held(kind):
|
| 761 |
return availability_reason("held")
|
|
|
|
| 764 |
return mount
|
| 765 |
if not _app_templates.seams_ready(kind):
|
| 766 |
return availability_reason("router_unavailable")
|
| 767 |
+
if kind == "alerts":
|
| 768 |
+
return availability_reason("ready" if _automation_offered(session) else "no_grant")
|
| 769 |
+
from routes_connectors import _entry_types
|
| 770 |
+
amazon = catalogue.get("connector") in ("amazon", "amazon_ads")
|
| 771 |
+
dashboard = {"state": "source_unavailable", "message": _AVAILABILITY_MESSAGES["source_unavailable"]}
|
| 772 |
+
|
| 773 |
+
def reason(state, connector=None):
|
| 774 |
+
result = availability_reason(state, connector)
|
| 775 |
+
return {**result, "dashboard": dashboard} if amazon else result
|
| 776 |
+
|
| 777 |
try:
|
| 778 |
sources = catalogue["sources"]
|
| 779 |
+
if amazon:
|
| 780 |
+
# Availability and the Dashboard use one source-wall answer.
|
| 781 |
+
access = app_access(session.user, session.runtime, app, sources)
|
| 782 |
+
dashboard = {"state": "ready" if access["dashboard"] == "ready" else
|
| 783 |
+
"no_grant" if access["status"] == 403 else "source_unavailable",
|
| 784 |
+
"message": access["message"]}
|
| 785 |
+
if access["app"] != "ready":
|
| 786 |
+
return reason("no_grant")
|
| 787 |
+
else:
|
| 788 |
+
from core import perm_scope
|
| 789 |
+
for source in sources:
|
| 790 |
+
if (not perm_scope.may_read(session.user, source, st=session.runtime)
|
| 791 |
+
or perm_scope.wall_declared(session.user, source, st=session.runtime)):
|
| 792 |
+
return reason("no_grant")
|
| 793 |
types, _scopes, failure = _entry_types(session.runtime, session.uname)
|
| 794 |
if failure is not None:
|
| 795 |
+
return reason("source_unavailable")
|
| 796 |
connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ())
|
| 797 |
if not connected and catalogue["connector"] == "odoo":
|
| 798 |
from routes_keychain import env_odoo_available
|
| 799 |
connected = env_odoo_available(session.runtime)
|
| 800 |
if not connected:
|
| 801 |
+
return reason("not_connected", catalogue["connector"])
|
| 802 |
+
row_sources = ((*sources, *catalogue.get("optional_sources", ()))
|
| 803 |
+
if kind == "repeat_buyers" else sources)
|
| 804 |
+
if amazon:
|
| 805 |
+
from harness.mirror_store import restore_in_progress
|
| 806 |
+
if restore_in_progress(session.tenant, catalogue["connector"]):
|
| 807 |
+
return reason("refreshing")
|
| 808 |
+
if row_sources and not _source_rows_exist(session, row_sources):
|
| 809 |
+
return reason("no_rows")
|
| 810 |
+
except Exception as exc: # noqa: BLE001
|
| 811 |
+
if amazon:
|
| 812 |
+
raise_if_resource_error(exc)
|
| 813 |
+
return reason("source_unavailable")
|
| 814 |
+
return reason("ready")
|
| 815 |
|
| 816 |
|
| 817 |
def availability(session, app_id):
|
|
|
|
| 932 |
record = (users.registry() or {}).get(user)
|
| 933 |
if not isinstance(record, dict) or record.get("active") is False:
|
| 934 |
return lambda _key: False
|
| 935 |
+
public = users._public(user, record)
|
| 936 |
+
if public.get("tenant") != getattr(runtime, "key", None):
|
| 937 |
+
return lambda _key: False
|
| 938 |
+
return _account_wall(runtime, public)
|
| 939 |
except Exception:
|
| 940 |
return lambda _key: False
|
| 941 |
|
|
|
|
| 1329 |
mount = _mount_reason(row.get("templateKind"))
|
| 1330 |
if mount:
|
| 1331 |
row["availability"] = mount
|
| 1332 |
+
elif (catalogue.get("connector") not in ("amazon", "amazon_ads")
|
| 1333 |
+
and not _database_free(app) and not compatible):
|
| 1334 |
row["availability"] = availability_reason("no_grant")
|
| 1335 |
if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
|
| 1336 |
# A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
|
|
|
|
| 1383 |
continue
|
| 1384 |
if _app_templates.held(app):
|
| 1385 |
continue
|
| 1386 |
+
if (isinstance(app, dict) and app.get("templateKind") == "alerts"
|
| 1387 |
+
and not (may_open or _user_wall(runtime, user))("automation")):
|
| 1388 |
+
continue
|
| 1389 |
if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
|
| 1390 |
continue
|
| 1391 |
if database:
|
|
|
|
| 1415 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 1416 |
|
| 1417 |
|
| 1418 |
+
def _nav_placement(app, index):
|
| 1419 |
+
"""Put default Project management entries first without rewriting stored placements."""
|
| 1420 |
+
placement = app.get("navPlacement")
|
| 1421 |
+
if (app.get("templateKind") == "amazon_profit"
|
| 1422 |
+
and (not placement or placement == {"section": "app", "order": 0})):
|
| 1423 |
+
# A zero placement already asks for first place. Break its default tie in the rail.
|
| 1424 |
+
return {"section": "app", "order": -1}
|
| 1425 |
+
return placement or {"section": "app", "order": index}
|
| 1426 |
+
|
| 1427 |
+
|
| 1428 |
def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None):
|
| 1429 |
"""The App home read: stored records the caller owns or was shared, never a code projection."""
|
| 1430 |
return [{"id": app["id"], "name": app.get("name"),
|
|
|
|
| 1435 |
"origin": "tenant",
|
| 1436 |
"compatibleDatabases": app.get("compatibleDatabases") or [],
|
| 1437 |
"openTarget": app.get("openTarget"),
|
| 1438 |
+
"navPlacement": _nav_placement(app, index),
|
| 1439 |
"capabilities": app.get("capabilities"), "starred": app.get("starred"),
|
| 1440 |
"isDefault": app.get("isDefault")}
|
| 1441 |
for index, app in enumerate(list_apps(runtime, user, is_admin,
|
|
|
|
| 1614 |
"""Create one App. A template App is one live copy per tenant: a second add returns the first."""
|
| 1615 |
if getattr(runtime, "key", tenant) != tenant or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant:
|
| 1616 |
raise PermissionError("an Application cannot be created for another tenant")
|
| 1617 |
+
if (isinstance(payload, dict) and str(payload.get("templateKind") or "").strip().lower() == "alerts"
|
| 1618 |
+
and not _user_wall(runtime, user)("automation")):
|
| 1619 |
+
raise PermissionError("Your current rights do not allow Automation.")
|
| 1620 |
app = _build_application(
|
| 1621 |
tenant, user, payload, semantic_records=semantic_records, field_contracts=field_contracts,
|
| 1622 |
allowed_databases=allowed_databases, origin_door=origin_door, via=via)
|
|
|
|
| 1811 |
return {"outcome": "migrated", "itemIds": copy.deepcopy(copied["itemIds"])}
|
| 1812 |
|
| 1813 |
|
| 1814 |
+
#: Marker of the one-copy step: each removed later copy and the kept copy that holds its slot now.
|
| 1815 |
+
TEMPLATE_ONE_COPY_MARKER = "templateOneCopy"
|
| 1816 |
+
#: The one-copy step deletes as the account Loopable seeds Apps under.
|
| 1817 |
+
_ONE_COPY_ACTOR = "loopable"
|
| 1818 |
+
|
| 1819 |
+
|
| 1820 |
+
def _template_double_plan(documents):
|
| 1821 |
+
"""`[(identity, (key, kept), [(key, later), ...])]` per tenant-wide template with two live copies.
|
| 1822 |
+
|
| 1823 |
+
Copies sort as `_refuse_template_held` sorts them. Routing is one private copy per account by design.
|
| 1824 |
+
"""
|
| 1825 |
+
groups = {}
|
| 1826 |
+
for key, doc in documents.items():
|
| 1827 |
+
for app_id, app in doc["applications"].items():
|
| 1828 |
+
if not isinstance(app, dict) or app.get("retired") is True:
|
| 1829 |
+
continue
|
| 1830 |
+
identity = template_identity(app)
|
| 1831 |
+
if identity and identity != "route":
|
| 1832 |
+
groups.setdefault(identity, {}).setdefault(app_id, (key, app))
|
| 1833 |
+
plan = []
|
| 1834 |
+
for identity, copies in sorted(groups.items()):
|
| 1835 |
+
rows = sorted(copies.values(), key=lambda row: (str(row[1].get("createdAt") or ""),
|
| 1836 |
+
str(row[1].get("id") or "")))
|
| 1837 |
+
if len(rows) > 1:
|
| 1838 |
+
plan.append((identity, rows[0], rows[1:]))
|
| 1839 |
+
return plan
|
| 1840 |
+
|
| 1841 |
+
|
| 1842 |
+
def _fold_template_copy(runtime, kept_key, kept_id, later):
|
| 1843 |
+
"""Give the kept copy the later copy's readers and stars, and repoint its defaults."""
|
| 1844 |
+
def fold(raw):
|
| 1845 |
+
doc = _clean_doc(raw)
|
| 1846 |
+
app = doc["applications"].get(kept_id)
|
| 1847 |
+
if not isinstance(app, dict) or app.get("retired") is True:
|
| 1848 |
+
raise ApplicationError("The kept App copy is unavailable.")
|
| 1849 |
+
before = copy.deepcopy(app)
|
| 1850 |
+
share = app.get("share") if isinstance(app.get("share"), dict) else {}
|
| 1851 |
+
other = later.get("share") if isinstance(later.get("share"), dict) else {}
|
| 1852 |
+
if share.get("mode") != "tenant":
|
| 1853 |
+
if other.get("mode") == "tenant":
|
| 1854 |
+
app["share"] = {"mode": "tenant", "users": []}
|
| 1855 |
+
else:
|
| 1856 |
+
users = [user for user in _users([
|
| 1857 |
+
*(share.get("users") or () if share.get("mode") == "users" else ()),
|
| 1858 |
+
*(other.get("users") or () if other.get("mode") == "users" else ()),
|
| 1859 |
+
later.get("createdBy")]) if user != app.get("createdBy")]
|
| 1860 |
+
if users:
|
| 1861 |
+
app["share"] = {"mode": "users", "users": users}
|
| 1862 |
+
stars = {*(app.get("starredBy") or ()), *(later.get("starredBy") or ())}
|
| 1863 |
+
if stars != set(app.get("starredBy") or ()):
|
| 1864 |
+
app["starredBy"] = sorted(stars)
|
| 1865 |
+
if app != before:
|
| 1866 |
+
app["revision"] = int(app.get("revision") or 0) + 1
|
| 1867 |
+
app["updatedAt"] = write_execution.utcnow().isoformat()
|
| 1868 |
+
return doc
|
| 1869 |
+
return raw
|
| 1870 |
+
_update_document(runtime, fold, key=kept_key)
|
| 1871 |
+
|
| 1872 |
+
def repoint(raw):
|
| 1873 |
+
doc = _clean_doc(raw)
|
| 1874 |
+
names = [name for name, value in doc["defaults"].items() if value == later.get("id")]
|
| 1875 |
+
if not names:
|
| 1876 |
+
return raw
|
| 1877 |
+
for name in names:
|
| 1878 |
+
doc["defaults"][name] = kept_id
|
| 1879 |
+
return doc
|
| 1880 |
+
if later.get("id") in _clean_doc(_read_strict(runtime, _KEY))["defaults"].values():
|
| 1881 |
+
_update_document(runtime, repoint)
|
| 1882 |
+
index = _read_strict(runtime, _APP_INDEX_KEY)
|
| 1883 |
+
owners = index.get("users") if isinstance(index, dict) else None
|
| 1884 |
+
if isinstance(owners, dict):
|
| 1885 |
+
_move_placements(runtime, {name: {later["id"]: kept_id} for name in owners})
|
| 1886 |
+
|
| 1887 |
+
|
| 1888 |
+
def remove_template_doubles(runtime, tenant):
|
| 1889 |
+
"""Keep the oldest live copy of each tenant-wide template and delete every later copy.
|
| 1890 |
+
|
| 1891 |
+
`delete` keeps the later copy's databases, Views and records. A second run writes nothing.
|
| 1892 |
+
Never raises: a failed copy is reported and retried at the next boot.
|
| 1893 |
+
"""
|
| 1894 |
+
if (getattr(runtime, "key", None) != tenant
|
| 1895 |
+
or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant):
|
| 1896 |
+
return {"outcome": "skipped", "removed": {}, "reason": "the runtime belongs to another tenant"}
|
| 1897 |
+
try:
|
| 1898 |
+
documents = {_KEY: _clean_doc(_read_strict(runtime, _KEY))}
|
| 1899 |
+
documents.update(_record_documents(runtime))
|
| 1900 |
+
except Exception as exc:
|
| 1901 |
+
return {"outcome": "skipped", "removed": {},
|
| 1902 |
+
"reason": f"registry read failed: {type(exc).__name__}: {exc}"}
|
| 1903 |
+
removed, failed = {}, []
|
| 1904 |
+
for identity, (kept_key, kept), later in _template_double_plan(documents):
|
| 1905 |
+
for _key, app in later:
|
| 1906 |
+
try:
|
| 1907 |
+
_fold_template_copy(runtime, kept_key, kept["id"], app)
|
| 1908 |
+
delete(runtime, app["id"], _ONE_COPY_ACTOR, True)
|
| 1909 |
+
except Exception as exc:
|
| 1910 |
+
failed.append(f"{app['id']}: {type(exc).__name__}")
|
| 1911 |
+
print(f"[aios-api] one template copy {tenant}: {app['id']} not removed: "
|
| 1912 |
+
f"{type(exc).__name__}")
|
| 1913 |
+
continue
|
| 1914 |
+
removed[app["id"]] = kept["id"]
|
| 1915 |
+
print(f"[aios-api] one template copy {tenant}: removed {app['id']}, "
|
| 1916 |
+
f"kept {kept['id']} ({identity})")
|
| 1917 |
+
if removed:
|
| 1918 |
+
def mark(raw):
|
| 1919 |
+
doc = _clean_doc(raw)
|
| 1920 |
+
migrations = doc.setdefault("migrations", {})
|
| 1921 |
+
marker = migrations.get(TEMPLATE_ONE_COPY_MARKER)
|
| 1922 |
+
marker = copy.deepcopy(marker) if isinstance(marker, dict) else {}
|
| 1923 |
+
held = marker.get("removed") if isinstance(marker.get("removed"), dict) else {}
|
| 1924 |
+
marker.update(removed={**held, **removed}, at=write_execution.utcnow().isoformat())
|
| 1925 |
+
migrations[TEMPLATE_ONE_COPY_MARKER] = marker
|
| 1926 |
+
return doc
|
| 1927 |
+
try:
|
| 1928 |
+
_update_document(runtime, mark)
|
| 1929 |
+
except Exception as exc:
|
| 1930 |
+
failed.append(f"marker: {type(exc).__name__}")
|
| 1931 |
+
outcome = "removed" if removed else ("failed" if failed else "unchanged")
|
| 1932 |
+
reason = ", ".join(f"{later_id} into {kept_id}" for later_id, kept_id in removed.items())
|
| 1933 |
+
return {"outcome": outcome, "removed": removed,
|
| 1934 |
+
"reason": "; ".join(part for part in (reason, ", ".join(failed)) if part)}
|
| 1935 |
+
|
| 1936 |
+
|
| 1937 |
def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
|
| 1938 |
"""R15: boot migrations, once per tenant, never from an Application read.
|
| 1939 |
|
|
|
|
| 2021 |
COMMAND_CENTER_MARKER = "commandCenterApplication"
|
| 2022 |
|
| 2023 |
|
| 2024 |
+
def _migrate_project_management_names(runtime):
|
| 2025 |
+
"""Rename only the two original defaults in each tenant-bound App document."""
|
| 2026 |
+
def original(app):
|
| 2027 |
+
return (isinstance(app, dict) and app.get("tenant") == runtime.key
|
| 2028 |
+
and app.get("templateKind") == "amazon_profit"
|
| 2029 |
+
and app.get("createdBy") == "loopable" and app.get("revision") == 1
|
| 2030 |
+
and app.get("name") in ("Command Center", "Amazon Command Center"))
|
| 2031 |
+
|
| 2032 |
+
documents = {_KEY: _clean_doc(_read_strict(runtime, _KEY)), **_record_documents(runtime)}
|
| 2033 |
+
count = 0
|
| 2034 |
+
for key, snapshot in documents.items():
|
| 2035 |
+
if not any(original(app) for app in snapshot["applications"].values()):
|
| 2036 |
+
continue
|
| 2037 |
+
changed = []
|
| 2038 |
+
|
| 2039 |
+
def apply(raw):
|
| 2040 |
+
changed.clear()
|
| 2041 |
+
doc = _clean_doc(raw)
|
| 2042 |
+
for app in doc["applications"].values():
|
| 2043 |
+
if original(app):
|
| 2044 |
+
app["name"] = "Project management"
|
| 2045 |
+
changed.append(app["id"])
|
| 2046 |
+
return doc
|
| 2047 |
+
|
| 2048 |
+
if _update_document(runtime, apply, key=key) is False:
|
| 2049 |
+
raise ApplicationError("The Project management name was not saved.")
|
| 2050 |
+
count += len(changed)
|
| 2051 |
+
return count
|
| 2052 |
+
|
| 2053 |
+
|
| 2054 |
+
def _adopt_project_field_order(runtime):
|
| 2055 |
+
"""Move Projects beside each managed product ID without rewriting fields, rows or saved views."""
|
| 2056 |
+
from core import user_tables
|
| 2057 |
+
from modules import amazon_profit
|
| 2058 |
+
|
| 2059 |
+
specs = amazon_profit.table_specs()
|
| 2060 |
+
|
| 2061 |
+
def ordered(spec, definition):
|
| 2062 |
+
if not isinstance(definition, dict) or definition.get("source") != user_tables.AUTOMATION_SOURCE:
|
| 2063 |
+
return None
|
| 2064 |
+
fields = definition.get("fields") or []
|
| 2065 |
+
keys = [field.get("key") for field in fields]
|
| 2066 |
+
if len(keys) != len(set(keys)) or not keys or keys[0] != spec["keyColumn"]:
|
| 2067 |
+
return None
|
| 2068 |
+
identity = fields[0].get("automation") or {}
|
| 2069 |
+
if identity.get("flowId") != amazon_profit.VIA or "projects" not in keys:
|
| 2070 |
+
return None
|
| 2071 |
+
wanted = amazon_profit._project_field_order(keys)
|
| 2072 |
+
defaults = [field["key"] for field in spec["fields"] if field["key"] != "projects"]
|
| 2073 |
+
# A saved definition with another order belongs to its author.
|
| 2074 |
+
if [key for key in keys if key != "projects"] != defaults:
|
| 2075 |
+
return None
|
| 2076 |
+
return wanted if wanted != keys else None
|
| 2077 |
+
|
| 2078 |
+
todo = [spec for spec in specs if ordered(spec, user_tables.get_definition(spec["key"], st=runtime))]
|
| 2079 |
+
if not todo:
|
| 2080 |
+
return 0
|
| 2081 |
+
changed = []
|
| 2082 |
+
|
| 2083 |
+
def apply(raw):
|
| 2084 |
+
changed.clear()
|
| 2085 |
+
tables = raw if isinstance(raw, dict) else {}
|
| 2086 |
+
for spec in todo:
|
| 2087 |
+
table = tables.get(spec["key"])
|
| 2088 |
+
wanted = ordered(spec, table)
|
| 2089 |
+
if wanted:
|
| 2090 |
+
fields = {field["key"]: field for field in table["fields"]}
|
| 2091 |
+
table["fields"] = [fields[key] for key in wanted]
|
| 2092 |
+
changed.append(spec["key"])
|
| 2093 |
+
return tables
|
| 2094 |
+
|
| 2095 |
+
if runtime.update(user_tables.STORE_KEY, apply, flush="sync") is False:
|
| 2096 |
+
raise ApplicationError("The Project management field order was not saved.")
|
| 2097 |
+
return len(changed)
|
| 2098 |
+
|
| 2099 |
+
|
| 2100 |
+
def reconcile_project_management_defaults(runtime, tenant):
|
| 2101 |
+
"""Correct only original App names and managed field defaults, without seeding or reading rows."""
|
| 2102 |
+
if (getattr(runtime, "key", None) != tenant
|
| 2103 |
+
or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant):
|
| 2104 |
+
raise PermissionError("The App migration belongs to another tenant.")
|
| 2105 |
+
return {"renamed": _migrate_project_management_names(runtime),
|
| 2106 |
+
"reordered": _adopt_project_field_order(runtime)}
|
| 2107 |
+
|
| 2108 |
+
|
| 2109 |
def ensure_command_center_application(runtime, tenant):
|
| 2110 |
+
"""Reconcile Project management defaults and seed one shared App when its table exists.
|
| 2111 |
|
| 2112 |
A tenant without the table gets no marker, so a later boot seeds it once the table exists.
|
| 2113 |
"""
|
|
|
|
| 2117 |
from core import user_tables
|
| 2118 |
from modules.amazon_profit import TABLE_KEY
|
| 2119 |
|
| 2120 |
+
corrected = reconcile_project_management_defaults(runtime, tenant)
|
| 2121 |
+
renamed, reordered = corrected["renamed"], corrected["reordered"]
|
| 2122 |
base = _clean_doc(_read_strict(runtime, _KEY))
|
| 2123 |
if COMMAND_CENTER_MARKER in (base.get("migrations") or {}):
|
| 2124 |
+
if renamed or reordered:
|
| 2125 |
+
return {"outcome": "migrated", "renamed": renamed, "reordered": reordered}
|
| 2126 |
return {"outcome": "unchanged"}
|
| 2127 |
records = _record_documents(runtime)
|
| 2128 |
|
|
|
|
| 2156 |
return raw
|
| 2157 |
app = _build_application(
|
| 2158 |
tenant, "loopable",
|
| 2159 |
+
{"name": "Project management", "templateKind": "amazon_profit",
|
| 2160 |
"navPlacement": {"section": "app", "order": 0}})
|
| 2161 |
app["share"] = {"mode": "tenant", "users": []}
|
| 2162 |
doc["applications"][app["id"]] = app
|
|
|
|
| 2164 |
migrations[COMMAND_CENTER_MARKER] = {"at": write_execution.utcnow().isoformat(), **result}
|
| 2165 |
return doc
|
| 2166 |
if _update_document(runtime, apply) is False:
|
| 2167 |
+
raise ApplicationError("The Project management App was not saved.")
|
| 2168 |
return result
|
| 2169 |
|
| 2170 |
|
api/assistant_feed.py
CHANGED
|
@@ -147,11 +147,9 @@ def post(rt, username, item):
|
|
| 147 |
if not (rt.get(cutover_key) or {}).get("purged"):
|
| 148 |
if not row_mode:
|
| 149 |
def purge(data):
|
| 150 |
-
|
| 151 |
-
data.pop(str(username), None)
|
| 152 |
-
return data
|
| 153 |
rt.update(alerts.NOTIFICATIONS_KEY, purge, flush="sync")
|
| 154 |
-
rt.update(cutover_key, lambda _data: {"version": 2, "purged": True}, flush="sync")
|
| 155 |
turn_id = assistant_log.append(rt, username, turn)
|
| 156 |
turn["id"] = turn_id
|
| 157 |
if _internal_only(rt, item):
|
|
|
|
| 147 |
if not (rt.get(cutover_key) or {}).get("purged"):
|
| 148 |
if not row_mode:
|
| 149 |
def purge(data):
|
| 150 |
+
return {}
|
|
|
|
|
|
|
| 151 |
rt.update(alerts.NOTIFICATIONS_KEY, purge, flush="sync")
|
| 152 |
+
rt.update(cutover_key, lambda _data: {"version": 2 if row_mode else 1, "purged": True}, flush="sync")
|
| 153 |
turn_id = assistant_log.append(rt, username, turn)
|
| 154 |
turn["id"] = turn_id
|
| 155 |
if _internal_only(rt, item):
|
api/assistant_memory.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
"""C13: a user's sourced facts, separate from the C12 conversation transcript."""
|
|
|
|
| 2 |
import copy
|
| 3 |
import datetime
|
| 4 |
import json
|
|
@@ -126,7 +127,7 @@ def _write_company(rt, user, text, *, sources, door, kind=None, subject=None):
|
|
| 126 |
# R105: every active member writes company notes. The author's role decides which note prevails.
|
| 127 |
if not isinstance(user, dict) or not isinstance(user.get("username"), str) or not user["username"]:
|
| 128 |
raise PermissionError("Only an active member of this workspace can share a company note.")
|
| 129 |
-
if door not in {"app", "whatsapp", "slack", "wake"}:
|
| 130 |
raise ValueError("That Assistant memory door is not available.")
|
| 131 |
subject = _subject(subject)
|
| 132 |
sources = _sources(sources)
|
|
@@ -493,6 +494,74 @@ def recall(rt, principal, query=None, limit=20):
|
|
| 493 |
return result
|
| 494 |
|
| 495 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
#: One compaction reads and sends at most one log chunk. A tail that grew while compaction could
|
| 497 |
#: not run (no model, no credit) is worked through in order, one bounded batch per attempt, so
|
| 498 |
#: neither the store read nor the model prompt grows with the conversation (D-803).
|
|
@@ -765,6 +834,132 @@ def memory_tools():
|
|
| 765 |
"properties": {"query": {"type": "string"}}}, _recall_memory, effect="read")
|
| 766 |
|
| 767 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 768 |
try:
|
| 769 |
memory_tools()
|
| 770 |
except (ImportError, ValueError) as exc:
|
|
|
|
| 1 |
"""C13: a user's sourced facts, separate from the C12 conversation transcript."""
|
| 2 |
+
import base64
|
| 3 |
import copy
|
| 4 |
import datetime
|
| 5 |
import json
|
|
|
|
| 127 |
# R105: every active member writes company notes. The author's role decides which note prevails.
|
| 128 |
if not isinstance(user, dict) or not isinstance(user.get("username"), str) or not user["username"]:
|
| 129 |
raise PermissionError("Only an active member of this workspace can share a company note.")
|
| 130 |
+
if door not in {"app", "whatsapp", "slack", "wake", "mcp"}:
|
| 131 |
raise ValueError("That Assistant memory door is not available.")
|
| 132 |
subject = _subject(subject)
|
| 133 |
sources = _sources(sources)
|
|
|
|
| 494 |
return result
|
| 495 |
|
| 496 |
|
| 497 |
+
INSPECT_ORDERS = ("newest", "oldest")
|
| 498 |
+
|
| 499 |
+
|
| 500 |
+
class InspectRequestError(ValueError):
|
| 501 |
+
"""A page request the route answers with 400, raised before any store read."""
|
| 502 |
+
|
| 503 |
+
|
| 504 |
+
def _encode_cursor(key, id):
|
| 505 |
+
stamp, company, position = key
|
| 506 |
+
raw = json.dumps([None if stamp == float("-inf") else stamp, company, position, id], separators=(",", ":"))
|
| 507 |
+
return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=")
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
def _decode_cursor(cursor):
|
| 511 |
+
try:
|
| 512 |
+
stamp, company, position, id = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
|
| 513 |
+
if ((stamp is not None and type(stamp) not in (int, float)) or company not in (0, 1)
|
| 514 |
+
or type(position) is not int or not isinstance(id, str)):
|
| 515 |
+
raise TypeError
|
| 516 |
+
return (float("-inf") if stamp is None else float(stamp), company, position), id
|
| 517 |
+
except Exception:
|
| 518 |
+
raise InspectRequestError("That memory page is out of date. Reload the list.") from None
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def inspect_page(rt, principal, *, cursor=None, limit=MAX_ENTRIES, order="newest", personal_only=False):
|
| 522 |
+
"""Page the whole authorized memory for the inspect and erase panel. Turn context keeps recall."""
|
| 523 |
+
if order not in INSPECT_ORDERS:
|
| 524 |
+
raise InspectRequestError("Choose newest or oldest first.")
|
| 525 |
+
after = _decode_cursor(cursor) if cursor else None
|
| 526 |
+
limit = max(1, min(MAX_ENTRIES, int(limit)))
|
| 527 |
+
if principal["tenant"] != assistant_log._tenant_of(rt):
|
| 528 |
+
return {"entries": [], "total": 0, "next": None}
|
| 529 |
+
doc = document(rt, principal["username"])
|
| 530 |
+
company = _document({}) if personal_only else company_document(rt)
|
| 531 |
+
entries = [(entry, "personal", doc["tombstones"], 0, n) for n, entry in enumerate(doc["entries"])]
|
| 532 |
+
entries.extend((entry, "company", company["tombstones"], 1, n) for n, entry in enumerate(company["entries"]))
|
| 533 |
+
# Same walls as recall: the prevailing note is chosen before the source filter.
|
| 534 |
+
superseded = _superseded(company["entries"], company["tombstones"])
|
| 535 |
+
readable, rows = {}, []
|
| 536 |
+
for entry, scope, tombstones, rank, position in entries:
|
| 537 |
+
if (not isinstance(entry, dict) or not all(key in entry for key in ("id", "text", "sources", "door", "at"))
|
| 538 |
+
or not isinstance(entry["text"], str) or not isinstance(entry["id"], str)):
|
| 539 |
+
continue
|
| 540 |
+
if scope == "company" and id(entry) in superseded:
|
| 541 |
+
continue
|
| 542 |
+
if _normalized(entry["text"]) in tombstones:
|
| 543 |
+
continue
|
| 544 |
+
# Permissions are fixed for one request, so one answer per source set is enough.
|
| 545 |
+
wall = (scope, json.dumps(entry["sources"], sort_keys=True, default=str))
|
| 546 |
+
if wall not in readable:
|
| 547 |
+
readable[wall] = _readable(rt, principal, entry["sources"], company=scope == "company")
|
| 548 |
+
if readable[wall]:
|
| 549 |
+
# Age, then document position: the order recall uses, so equal stamps never reorder.
|
| 550 |
+
rows.append(((_oldest(entry), rank, position), entry, scope))
|
| 551 |
+
newest = order == "newest"
|
| 552 |
+
rows.sort(key=lambda row: row[0], reverse=newest)
|
| 553 |
+
start = 0
|
| 554 |
+
if after is not None:
|
| 555 |
+
key, anchor = after
|
| 556 |
+
# Resume after the anchor id. If it was erased, resume after its sort key.
|
| 557 |
+
start = next((n + 1 for n, row in enumerate(rows) if row[1]["id"] == anchor and row[0][1] == key[1]),
|
| 558 |
+
next((n for n, row in enumerate(rows) if (row[0] < key if newest else row[0] > key)), len(rows)))
|
| 559 |
+
page = rows[start:start + limit]
|
| 560 |
+
more = start + limit < len(rows)
|
| 561 |
+
return {"entries": [{**copy.deepcopy(entry), "scope": scope} for _, entry, scope in page],
|
| 562 |
+
"total": len(rows), "next": _encode_cursor(page[-1][0], page[-1][1]["id"]) if more else None}
|
| 563 |
+
|
| 564 |
+
|
| 565 |
#: One compaction reads and sends at most one log chunk. A tail that grew while compaction could
|
| 566 |
#: not run (no model, no credit) is worked through in order, one bounded batch per attempt, so
|
| 567 |
#: neither the store read nor the model prompt grows with the conversation (D-803).
|
|
|
|
| 834 |
"properties": {"query": {"type": "string"}}}, _recall_memory, effect="read")
|
| 835 |
|
| 836 |
|
| 837 |
+
def mcp_memory(session, action, args):
|
| 838 |
+
"""Use the existing personal document with fresh account, credential and source walls."""
|
| 839 |
+
import intelligence_store as files
|
| 840 |
+
from core import external_access
|
| 841 |
+
from core import write_execution
|
| 842 |
+
|
| 843 |
+
session = files.fresh_personal_session(session)
|
| 844 |
+
principal = {"tenant": session.tenant, "username": session.uname}
|
| 845 |
+
rt = session.runtime
|
| 846 |
+
|
| 847 |
+
def readable(sources):
|
| 848 |
+
_, scope = files.credential_scope(session)
|
| 849 |
+
return (all(external_access.database_allowed(scope, source["database"]) for source in sources)
|
| 850 |
+
and _readable(rt, principal, sources))
|
| 851 |
+
|
| 852 |
+
if action == "memory_recall":
|
| 853 |
+
page = inspect_page(rt, principal, cursor=args.get("cursor"),
|
| 854 |
+
limit=args.get("limit", 50), order=args.get("order", "newest"), personal_only=True)
|
| 855 |
+
# The MCP door is personal. Company publication retains its separate approval policy.
|
| 856 |
+
page["entries"] = [entry for entry in page["entries"] if entry["scope"] == "personal"
|
| 857 |
+
and readable(entry["sources"])
|
| 858 |
+
and all(word in entry["text"].casefold()
|
| 859 |
+
for word in args.get("query", "").casefold().split())]
|
| 860 |
+
page.pop("total", None)
|
| 861 |
+
return page
|
| 862 |
+
id = args.get("id")
|
| 863 |
+
old = next((entry for entry in document(rt, session.uname)["entries"] if entry.get("id") == id), None)
|
| 864 |
+
if action in {"memory_correct", "memory_forget"}:
|
| 865 |
+
if old is None or not readable(old["sources"]):
|
| 866 |
+
raise files.IntelligenceError("memory_not_found", "That personal memory was not found.", 404)
|
| 867 |
+
if action == "memory_forget":
|
| 868 |
+
return {"deleted": forget(rt, session.uname, id)}
|
| 869 |
+
elif action != "memory_remember":
|
| 870 |
+
raise ValueError("That personal memory action is unavailable.")
|
| 871 |
+
text = args["text"]
|
| 872 |
+
if not isinstance(text, str) or not text.strip() or len(text) > MAX_TEXT:
|
| 873 |
+
raise ValueError("A personal fact needs 1 to 500 characters.")
|
| 874 |
+
sources = _sources([*(old or {}).get("sources", []),
|
| 875 |
+
*_capture_sources(session.user, args.get("sources", []))])
|
| 876 |
+
if not readable(sources):
|
| 877 |
+
raise files.IntelligenceError("forbidden", "This fact's sources are unavailable.", 403)
|
| 878 |
+
duplicates = {entry["id"]: copy.deepcopy(entry["sources"])
|
| 879 |
+
for entry in document(rt, session.uname)["entries"] if _normalized(entry["text"]) == _normalized(text)}
|
| 880 |
+
if any(not readable(value) for value in duplicates.values()):
|
| 881 |
+
raise files.IntelligenceError("forbidden", "This fact's sources are unavailable.", 403)
|
| 882 |
+
landed = {}
|
| 883 |
+
|
| 884 |
+
def apply(raw):
|
| 885 |
+
doc = _document(raw)
|
| 886 |
+
if _normalized(text) in doc["tombstones"]:
|
| 887 |
+
raise ValueError("That text was erased from memory.")
|
| 888 |
+
if old is not None:
|
| 889 |
+
current = next((entry for entry in doc["entries"] if entry.get("id") == id), None)
|
| 890 |
+
if current != old:
|
| 891 |
+
raise files.IntelligenceError("memory_stale", "Read the personal fact again before correcting it.", 409)
|
| 892 |
+
if _normalized(current["text"]) == _normalized(text):
|
| 893 |
+
landed.update(current)
|
| 894 |
+
return doc
|
| 895 |
+
doc["entries"].remove(current)
|
| 896 |
+
doc["tombstones"] = list(dict.fromkeys([*doc["tombstones"], _normalized(old["text"])]))
|
| 897 |
+
duplicate = next((entry for entry in doc["entries"] if _normalized(entry["text"]) == _normalized(text)), None)
|
| 898 |
+
if duplicate:
|
| 899 |
+
if duplicate["id"] not in duplicates or duplicate["sources"] != duplicates[duplicate["id"]]:
|
| 900 |
+
raise files.IntelligenceError("memory_stale", "Read the personal fact again before changing it.", 409)
|
| 901 |
+
duplicate["sources"] = _sources([*duplicate["sources"], *sources])
|
| 902 |
+
landed.update(duplicate)
|
| 903 |
+
else:
|
| 904 |
+
if len(doc["entries"]) >= MAX_ENTRIES:
|
| 905 |
+
raise ValueError("Personal memory is full. Erase or consolidate a fact first.")
|
| 906 |
+
entry = {"id": write_execution.new_id(), "text": text.strip(), "sources": sources, "door": "mcp",
|
| 907 |
+
"at": _now(), "kind": _kind(text), "provenance": {"user": session.uname, "door": "mcp"}}
|
| 908 |
+
if args.get("_captureSubject"):
|
| 909 |
+
entry["captureSubject"] = args["_captureSubject"]
|
| 910 |
+
entry["provenance"]["interaction"] = args["_interaction"]
|
| 911 |
+
if old:
|
| 912 |
+
entry["supersedes"] = old["id"]
|
| 913 |
+
doc["entries"].append(entry)
|
| 914 |
+
landed.update(entry)
|
| 915 |
+
doc["updatedAt"] = _now()
|
| 916 |
+
return doc
|
| 917 |
+
|
| 918 |
+
rt.update(_key(rt, session.uname), apply, flush="sync")
|
| 919 |
+
return {**copy.deepcopy(landed), "scope": "personal"}
|
| 920 |
+
|
| 921 |
+
|
| 922 |
+
def capture_mcp_context(session, context):
|
| 923 |
+
"""Capture bounded delivered human statements without treating instructions or secrets as facts."""
|
| 924 |
+
import hashlib
|
| 925 |
+
import intelligence_store as files
|
| 926 |
+
from core import external_access
|
| 927 |
+
session = files.fresh_personal_session(session)
|
| 928 |
+
if not external_access.may_write(files.credential_scope(session)[0]):
|
| 929 |
+
return {"state": "ignored", "reason": "read_only"}
|
| 930 |
+
if (not isinstance(context, dict) or set(context) - {"text", "interaction_id"}
|
| 931 |
+
or not isinstance(context.get("text"), str) or not isinstance(context.get("interaction_id"), str)
|
| 932 |
+
or not 1 <= len(context["interaction_id"]) <= 128 or len(context["text"]) > MAX_TEXT):
|
| 933 |
+
return {"state": "ignored", "reason": "invalid_context"}
|
| 934 |
+
text = context["text"].strip()
|
| 935 |
+
unsafe = (r"password|passwd|secret|token|api[_ -]?key|credential|bearer|-----BEGIN|"
|
| 936 |
+
r"ignor\w*|overrid\w*|system|developer|instruction|permission|grant|execute|https?://|"
|
| 937 |
+
r"\b(?:must|always|never|you|assistant|model|tool|print\w*|send\w*|delet\w*|eras\w*|run|skip|bypass\w*|obey\w*|follow\w*)\b|"
|
| 938 |
+
r"[\r\n]|[A-Za-z0-9_+/=-]{32,}")
|
| 939 |
+
if re.search(unsafe, text, re.I):
|
| 940 |
+
return {"state": "ignored", "reason": "unsafe_statement"}
|
| 941 |
+
correction = bool(re.match(r"^correction:\s*", text, re.I))
|
| 942 |
+
text = re.sub(r"^correction:\s*", "", text, flags=re.I)
|
| 943 |
+
match = re.fullmatch(r"(my (?:name|role|timezone|language) is|i prefer|i work as|call me) ([^.!?]{1,400})[.]?", text, re.I)
|
| 944 |
+
if not match:
|
| 945 |
+
return {"state": "ignored", "reason": "no_personal_fact"}
|
| 946 |
+
subject = _normalized(match[1])
|
| 947 |
+
args = {"text": text, "_captureSubject": subject,
|
| 948 |
+
"_interaction": hashlib.sha256(context["interaction_id"].encode()).hexdigest()}
|
| 949 |
+
action = "memory_remember"
|
| 950 |
+
if correction:
|
| 951 |
+
entries = [entry for entry in document(session.runtime, session.uname)["entries"]
|
| 952 |
+
if entry.get("captureSubject") == subject and _normalized(entry["text"]) != _normalized(text)]
|
| 953 |
+
if len(entries) != 1:
|
| 954 |
+
return {"state": "ignored", "reason": "ambiguous_correction"}
|
| 955 |
+
action, args["id"] = "memory_correct", entries[0]["id"]
|
| 956 |
+
try:
|
| 957 |
+
saved = mcp_memory(session, action, args)
|
| 958 |
+
except ValueError:
|
| 959 |
+
return {"state": "ignored", "reason": "erased_or_full"}
|
| 960 |
+
return {"state": "saved", "id": saved["id"], "scope": "personal"}
|
| 961 |
+
|
| 962 |
+
|
| 963 |
try:
|
| 964 |
memory_tools()
|
| 965 |
except (ImportError, ValueError) as exc:
|
api/assistant_mirror.py
CHANGED
|
@@ -58,6 +58,12 @@ def _rot():
|
|
| 58 |
return routes_odoo_tables
|
| 59 |
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
def open_cursor(session, table_key=None):
|
| 62 |
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
|
| 63 |
|
|
@@ -77,17 +83,21 @@ def open_cursor(session, table_key=None):
|
|
| 77 |
try:
|
| 78 |
cur, mismatch = rot._mirror_cur(session, table_key)
|
| 79 |
except RuntimeError as exc:
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
except Exception as exc: # noqa: BLE001
|
| 82 |
# ⚠ A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from
|
| 83 |
# `deps`, which every fixture and gate in the repo imports and drives with runtimes that
|
| 84 |
# are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on
|
| 85 |
# whatever it is handed. An unexpected failure there is a 500 on an assistant answer,
|
| 86 |
# which reads to a user as the product being broken rather than the mirror being shut.
|
| 87 |
-
# Named and fail-closed beats a stack trace.
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
if mismatch:
|
| 92 |
return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch))
|
| 93 |
return cur, None
|
|
@@ -167,17 +177,14 @@ def unavailable_reason(table_key, definition=None, *, snapshot=None):
|
|
| 167 |
if key.startswith("ut_amzn_"):
|
| 168 |
import amazon_relational as amazon
|
| 169 |
if key in amazon.SOURCE:
|
| 170 |
-
if snapshot is not None
|
| 171 |
-
|
| 172 |
-
return amazon.missing_source(key)["missing"]
|
| 173 |
spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
|
| 174 |
if spec and not spec.get("missing"):
|
| 175 |
-
|
| 176 |
-
return (f"{label} is still loading into this server after a restart, "
|
| 177 |
-
"so ask again in an hour.")
|
| 178 |
except Exception: # noqa: BLE001
|
| 179 |
pass
|
| 180 |
-
return
|
| 181 |
|
| 182 |
|
| 183 |
def still_loading(table_key, *, cur=None, snapshot=None):
|
|
|
|
| 58 |
return routes_odoo_tables
|
| 59 |
|
| 60 |
|
| 61 |
+
def _cause(cause):
|
| 62 |
+
"""The closed-set refusal sentence `deps` owns, imported lazily to break the `deps` cycle."""
|
| 63 |
+
import deps
|
| 64 |
+
return deps._assistant_source_message(cause)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
def open_cursor(session, table_key=None):
|
| 68 |
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
|
| 69 |
|
|
|
|
| 83 |
try:
|
| 84 |
cur, mismatch = rot._mirror_cur(session, table_key)
|
| 85 |
except RuntimeError as exc:
|
| 86 |
+
if "could not be restored" in str(exc):
|
| 87 |
+
return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("restore_blocked"))
|
| 88 |
+
if "has not been attached" in str(exc):
|
| 89 |
+
return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("loading"))
|
| 90 |
+
return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("syncing"))
|
| 91 |
except Exception as exc: # noqa: BLE001
|
| 92 |
# ⚠ A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from
|
| 93 |
# `deps`, which every fixture and gate in the repo imports and drives with runtimes that
|
| 94 |
# are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on
|
| 95 |
# whatever it is handed. An unexpected failure there is a 500 on an assistant answer,
|
| 96 |
# which reads to a user as the product being broken rather than the mirror being shut.
|
| 97 |
+
# Named and fail-closed beats a stack trace. The class goes to the server log, never the reply.
|
| 98 |
+
import deps
|
| 99 |
+
deps._assistant_log_cause("assistant mirror open", exc)
|
| 100 |
+
return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("mirror_open"))
|
| 101 |
if mismatch:
|
| 102 |
return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch))
|
| 103 |
return cur, None
|
|
|
|
| 177 |
if key.startswith("ut_amzn_"):
|
| 178 |
import amazon_relational as amazon
|
| 179 |
if key in amazon.SOURCE:
|
| 180 |
+
pending = snapshot.amazon_pending() if snapshot is not None else None
|
| 181 |
+
return amazon.missing_source(key, pending=pending)["missing"]
|
|
|
|
| 182 |
spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
|
| 183 |
if spec and not spec.get("missing"):
|
| 184 |
+
return _cause("loading")
|
|
|
|
|
|
|
| 185 |
except Exception: # noqa: BLE001
|
| 186 |
pass
|
| 187 |
+
return _cause("no_binding")
|
| 188 |
|
| 189 |
|
| 190 |
def still_loading(table_key, *, cur=None, snapshot=None):
|
api/automation_engine.py
CHANGED
|
@@ -2803,6 +2803,282 @@ def all_definitions(rt, cached=False):
|
|
| 2803 |
return out
|
| 2804 |
|
| 2805 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2806 |
def automation_runs(rt, auto_id):
|
| 2807 |
"""Read run history after a once definition has retired."""
|
| 2808 |
row = (rt.get(STORE_KEY) or {}).get(str(auto_id))
|
|
@@ -6804,14 +7080,22 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 6804 |
rows, merged = _merge_candidates(rows)
|
| 6805 |
stats["merged"] += merged
|
| 6806 |
|
|
|
|
|
|
|
| 6807 |
def _up(cur, _f=fields, _r=rows):
|
|
|
|
| 6808 |
cur = cur or {}
|
| 6809 |
t = dict(cur.get(table_key) or {})
|
|
|
|
| 6810 |
t["fields"], t["rows"] = _f, _r
|
|
|
|
|
|
|
|
|
|
| 6811 |
cur[table_key] = t
|
| 6812 |
return cur
|
| 6813 |
|
| 6814 |
rt.update(UT_STORE_KEY, _up)
|
|
|
|
| 6815 |
# ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
|
| 6816 |
#
|
| 6817 |
# W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early
|
|
@@ -14211,6 +14495,10 @@ def _module_table(rt, key, memo, want_rows=False):
|
|
| 14211 |
"check this tenant's Odoo connector under Settings -> Connectors; the linked column "
|
| 14212 |
"keeps its last resolved value until the source answers again")
|
| 14213 |
table["rows"] = {str(r.get("pid")): r for r in (pool or []) if r.get("pid") is not None}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14214 |
for rid, row in list(table['rows'].items()):
|
| 14215 |
table['rows'][rid] = {**row, **(table.get('_if_shared_cells', {}).get(rid) or {})}
|
| 14216 |
table["_rows_built"] = True
|
|
@@ -14323,6 +14611,16 @@ def _mirror_table(rt, key, defn):
|
|
| 14323 |
return table, None
|
| 14324 |
|
| 14325 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14326 |
def _resolve_table(rt, store, key, memo, want_rows=False):
|
| 14327 |
"""A database by key, from EITHER namespace — `(table_or_None, limit_or_None)`.
|
| 14328 |
|
|
@@ -14356,28 +14654,60 @@ def _resolve_table(rt, store, key, memo, want_rows=False):
|
|
| 14356 |
if cached is None:
|
| 14357 |
cached = memo[("mirror", key)] = _mirror_table(rt, key, hit)
|
| 14358 |
return cached
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14359 |
return hit, None
|
| 14360 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 14361 |
|
| 14362 |
|
| 14363 |
-
def _link_from_key(fields, bag):
|
| 14364 |
"""Which column on THIS table supplies the join value.
|
| 14365 |
|
| 14366 |
Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
|
| 14367 |
fallback chain is what makes an Instagram database link up with no configuration at all —
|
| 14368 |
the "automatically" in the owner's instruction — and it is the same chain the grid already
|
| 14369 |
uses to decide a table's identity column, rather than a second opinion about it.
|
|
|
|
|
|
|
| 14370 |
"""
|
| 14371 |
declared = str((bag or {}).get("from") or "").strip()
|
| 14372 |
if declared:
|
| 14373 |
return declared
|
| 14374 |
-
|
|
|
|
| 14375 |
if prof:
|
| 14376 |
return str(prof.get("key") or "")
|
| 14377 |
-
pin = next((f for f in
|
| 14378 |
return str(pin.get("key") or "") if pin else ""
|
| 14379 |
|
| 14380 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14381 |
def _join_norm(value):
|
| 14382 |
"""ONE normalisation of an `on` join value, applied to BOTH sides of the join.
|
| 14383 |
|
|
@@ -14541,9 +14871,9 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14541 |
ifs = [f for f in fields if f.get('type') == 'if']
|
| 14542 |
if not links and not rollups and not ifs:
|
| 14543 |
return {}, limits
|
| 14544 |
-
rows = (t
|
| 14545 |
-
if
|
| 14546 |
-
|
| 14547 |
# A registry source: NOW the pool is worth its cost, because this table really does
|
| 14548 |
# declare relations. One fetch, memoised, shared with every link and rollup below.
|
| 14549 |
# ⭐⭐ W53-T40 (d) — AND A READ-THROUGH SOURCE TAKES THE SAME SECOND FETCH, for a reason
|
|
@@ -14564,6 +14894,18 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14564 |
# --- resolve every link field ONCE per table, not once per row.
|
| 14565 |
# `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
|
| 14566 |
resolved, linked_types, refused = {}, {}, set()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14567 |
for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
|
| 14568 |
lk_key = str((f or {}).get("key") or "")
|
| 14569 |
if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
|
|
@@ -14630,7 +14972,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14630 |
on_key = str(bag.get("on") or "")
|
| 14631 |
hop_in = str(bag.get("throughOn") or "")
|
| 14632 |
hop_out = str(bag.get("throughTo") or "")
|
| 14633 |
-
from_key =
|
| 14634 |
if not (on_key and hop_in and hop_out and from_key):
|
| 14635 |
# ⛔ REFUSED, NOT BLANK — the same law the one-hop arm below follows. A route with
|
| 14636 |
# a missing leg has no second hop to make; resolving it to `[]` would render a
|
|
@@ -14668,7 +15010,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14668 |
target_idx = _linked_rows_by_join(linked, on_key)
|
| 14669 |
cap = int(_ut().LINK_THROUGH_MAX_IDS)
|
| 14670 |
|
| 14671 |
-
def _walk(row, _bidx=bridge_idx, _tidx=target_idx, _cap=cap):
|
| 14672 |
"""One source row's DISTINCT target records, and whether the ceiling stopped it.
|
| 14673 |
|
| 14674 |
⚠ DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines;
|
|
@@ -14676,7 +15018,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14676 |
label of a set of PRODUCTS.
|
| 14677 |
"""
|
| 14678 |
seen, out = set(), []
|
| 14679 |
-
for _bid, brow in _bidx.get(_join_norm((row
|
| 14680 |
for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()):
|
| 14681 |
if tid in seen:
|
| 14682 |
continue
|
|
@@ -14688,7 +15030,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14688 |
|
| 14689 |
two_hop, capped = {}, 0
|
| 14690 |
for rid, row in rows.items():
|
| 14691 |
-
hits_2, hit_cap = _walk(row)
|
| 14692 |
two_hop[str(rid)] = hits_2
|
| 14693 |
capped += 1 if hit_cap else 0
|
| 14694 |
resolved[lk_key] = two_hop
|
|
@@ -14712,7 +15054,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14712 |
effect="capped", limit=cap))
|
| 14713 |
elif bag.get("on"):
|
| 14714 |
idx = _linked_rows_by_join(linked, str(bag["on"]))
|
| 14715 |
-
from_key =
|
| 14716 |
if not from_key:
|
| 14717 |
# ⛔ REFUSED, NOT BLANK. `_link_from_key` falls back to the PROFILE-flagged column
|
| 14718 |
# and then the PINNED one; a table with neither — which a registry module can be,
|
|
@@ -14728,7 +15070,7 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14728 |
"database that is the id column, e.g. `partner_id` or `product_id`)"))
|
| 14729 |
continue
|
| 14730 |
resolved[lk_key] = {
|
| 14731 |
-
str(rid): idx.get(_join_norm((row
|
| 14732 |
for rid, row in rows.items()}
|
| 14733 |
else:
|
| 14734 |
# An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
|
|
@@ -15024,6 +15366,136 @@ def related_product_metric_values(rt, customer_id, products, fields, offer, *,
|
|
| 15024 |
return out, list(dict.fromkeys(reasons))
|
| 15025 |
|
| 15026 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15027 |
def compute_relation_cells(rt, table_key, tables=None):
|
| 15028 |
"""Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
|
| 15029 |
|
|
@@ -15050,7 +15522,10 @@ def compute_relation_cells(rt, table_key, tables=None):
|
|
| 15050 |
another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the
|
| 15051 |
caller that renders them reads them, and nothing about this door changes shape.
|
| 15052 |
"""
|
| 15053 |
-
changes,
|
|
|
|
|
|
|
|
|
|
| 15054 |
if not changes:
|
| 15055 |
return 0
|
| 15056 |
if str(table_key) in LINKABLE_MODULES:
|
|
@@ -15093,28 +15568,85 @@ def _refresh_relations_inplace(tables, log=print, rt=None):
|
|
| 15093 |
keyword this function briefly added (see `compute_relation_cells`).
|
| 15094 |
"""
|
| 15095 |
touched = 0
|
|
|
|
| 15096 |
for tk, table in list((tables or {}).items()):
|
| 15097 |
fields = (table or {}).get("fields") or []
|
| 15098 |
if not any(f.get("type") in ('rollup', 'if') or _ut().is_derived_link(f) for f in fields):
|
|
|
|
|
|
|
| 15099 |
continue
|
|
|
|
| 15100 |
try:
|
| 15101 |
touched += compute_relation_cells(rt, tk, tables=tables)
|
|
|
|
| 15102 |
except Exception as e: # noqa: BLE001
|
| 15103 |
log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
|
|
|
|
|
|
|
|
|
|
| 15104 |
return touched
|
| 15105 |
|
| 15106 |
|
| 15107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15108 |
"""Compute relations outside the write lock and merge only unchanged derived cells."""
|
| 15109 |
-
snapshot =
|
| 15110 |
# Keep authored inputs before computation, including nested cell values.
|
| 15111 |
-
before = copy.deepcopy(snapshot)
|
| 15112 |
local = _refresh_relations_inplace(snapshot, log=log, rt=rt)
|
| 15113 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15114 |
return local
|
| 15115 |
patches = {}
|
| 15116 |
for key, old in before.items():
|
| 15117 |
-
if not isinstance(old, dict):
|
| 15118 |
continue
|
| 15119 |
fields = old.get('fields') or []
|
| 15120 |
derived = {str(f['key']) for f in fields if f.get('key')
|
|
@@ -15129,31 +15661,60 @@ def refresh_relations(rt, log=print, tables=None, persist=True):
|
|
| 15129 |
if changed:
|
| 15130 |
patches.setdefault(str(key), {})[str(rid)] = (row, changed)
|
| 15131 |
if not patches:
|
|
|
|
| 15132 |
return 0
|
| 15133 |
actual = [0]
|
|
|
|
|
|
|
| 15134 |
|
| 15135 |
def _up(cur):
|
| 15136 |
actual[0] = 0
|
|
|
|
|
|
|
| 15137 |
lock = getattr(cur, 'lock_records', None)
|
| 15138 |
if callable(lock):
|
| 15139 |
lock({key: list(rows) for key, rows in patches.items()})
|
| 15140 |
for key, rows in patches.items():
|
| 15141 |
table = (cur or {}).get(key)
|
| 15142 |
if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'):
|
|
|
|
| 15143 |
continue
|
|
|
|
| 15144 |
current_rows = table.get('rows')
|
| 15145 |
if current_rows is None:
|
|
|
|
| 15146 |
continue
|
| 15147 |
for rid, (old_row, changes) in rows.items():
|
| 15148 |
current = current_rows.get(rid)
|
| 15149 |
# A changed authored input or a deleted row waits for the next refresh.
|
| 15150 |
if not isinstance(current, dict) or current != old_row:
|
|
|
|
| 15151 |
continue
|
|
|
|
|
|
|
| 15152 |
current.update(changes)
|
| 15153 |
actual[0] += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15154 |
return cur
|
| 15155 |
|
| 15156 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15157 |
return actual[0]
|
| 15158 |
|
| 15159 |
|
|
@@ -16256,13 +16817,153 @@ def _automation_slack_reply_text(result):
|
|
| 16256 |
return "An authorized automation completed."
|
| 16257 |
|
| 16258 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16259 |
def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
|
| 16260 |
"""Deliver one authorized event result, with readable linked records for an enters_view cohort."""
|
| 16261 |
definition = defn if isinstance(defn, dict) else {}
|
| 16262 |
aid = str(definition.get("id") or "").strip()
|
| 16263 |
if not aid:
|
| 16264 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16265 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16266 |
if not isinstance(stored, dict):
|
| 16267 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16268 |
trigger = stored.get("trigger") if isinstance(stored.get("trigger"), dict) else {}
|
|
@@ -16286,6 +16987,22 @@ def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
|
|
| 16286 |
context = routes_slack.automation_answer_context(rt, aid)
|
| 16287 |
if context is None:
|
| 16288 |
return {"sent": False, "reason": "not_authorized"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16289 |
provenance = {"basis": "model_guidance", "citationIds": [], "modelGuidance": False}
|
| 16290 |
if trigger_key == "enters_view":
|
| 16291 |
module = str(trigger.get("table") or _flow_table(stored) or "")
|
|
|
|
| 2803 |
return out
|
| 2804 |
|
| 2805 |
|
| 2806 |
+
def alert_principal(rt, username):
|
| 2807 |
+
"""Resolve the active rule owner from the current tenant account rows."""
|
| 2808 |
+
from core import account_rows, users
|
| 2809 |
+
resolve = getattr(rt, '_document_store', None)
|
| 2810 |
+
record = account_rows.get(username, st=resolve() if callable(resolve) else rt, fresh=True)
|
| 2811 |
+
if not record or record.get('active', True) is False:
|
| 2812 |
+
raise PermissionError('This alert owner is no longer active in this workspace.')
|
| 2813 |
+
principal = users._public(username, record)
|
| 2814 |
+
if principal['tenant'] != rt.key:
|
| 2815 |
+
raise PermissionError('This alert owner is no longer active in this workspace.')
|
| 2816 |
+
return principal
|
| 2817 |
+
|
| 2818 |
+
|
| 2819 |
+
def alert_schema(rt, principal, database):
|
| 2820 |
+
"""Return only the database fields the current principal can read."""
|
| 2821 |
+
from core import perm_scope
|
| 2822 |
+
if not principal or not perm_scope.may_read(principal, database, st=rt):
|
| 2823 |
+
raise PermissionError('This database is unavailable to your account.')
|
| 2824 |
+
return perm_scope.scoped_fields(principal, database, st=rt)
|
| 2825 |
+
|
| 2826 |
+
|
| 2827 |
+
def clean_alert(rt, principal, raw):
|
| 2828 |
+
"""Normalize a received-change rule through the existing condition and formula evaluators."""
|
| 2829 |
+
import formula_runtime
|
| 2830 |
+
if not isinstance(raw, dict):
|
| 2831 |
+
raise ValueError('Supply an alert configuration.')
|
| 2832 |
+
name = raw.get('name')
|
| 2833 |
+
if not isinstance(name, str) or not name.strip() or len(name) > MAX_NAME:
|
| 2834 |
+
raise ValueError('Give this alert a short name.')
|
| 2835 |
+
database = str(raw.get('database') or '')
|
| 2836 |
+
fields = alert_schema(rt, principal, database)
|
| 2837 |
+
available = {str(f['key']) for f in fields if f.get('key')}
|
| 2838 |
+
watched = raw.get('fields')
|
| 2839 |
+
if (not isinstance(watched, list) or not watched or len(watched) > 100
|
| 2840 |
+
or any(not isinstance(k, str) or k not in available for k in watched)):
|
| 2841 |
+
raise ValueError('Choose fields you can read in this database.')
|
| 2842 |
+
condition, problem = clean_cond(raw.get('condition'))
|
| 2843 |
+
if problem:
|
| 2844 |
+
raise ValueError(problem)
|
| 2845 |
+
formula = None
|
| 2846 |
+
if raw.get('formula'):
|
| 2847 |
+
if not isinstance(raw['formula'], str):
|
| 2848 |
+
raise ValueError('Use a formula expression.')
|
| 2849 |
+
formula = formula_runtime.parse_formula(raw['formula'])
|
| 2850 |
+
if formula.get('cells'):
|
| 2851 |
+
raise ValueError('Alert formulas use database fields, not sheet addresses.')
|
| 2852 |
+
if condition is None and formula is None:
|
| 2853 |
+
raise ValueError('Choose a condition or a formula.')
|
| 2854 |
+
reads = cond_fields(condition) | set((formula or {}).get('refs') or [])
|
| 2855 |
+
if not reads <= available:
|
| 2856 |
+
raise PermissionError('The condition uses a field you cannot read.')
|
| 2857 |
+
when = raw.get('when', 'updated')
|
| 2858 |
+
if when not in ('updated', 'created'):
|
| 2859 |
+
raise ValueError('Choose a received record update or creation.')
|
| 2860 |
+
destination = alert_destination(rt, principal, raw.get('destination'))
|
| 2861 |
+
return {'name': name.strip(), 'database': database, 'fields': list(dict.fromkeys(watched)),
|
| 2862 |
+
'condition': condition, 'formula': (formula or {}).get('source', ''),
|
| 2863 |
+
'when': when, 'destination': destination}
|
| 2864 |
+
|
| 2865 |
+
|
| 2866 |
+
def alert_destination(rt, principal, raw):
|
| 2867 |
+
"""Validate a saved destination reference without accepting inline secrets or URLs."""
|
| 2868 |
+
from core import keychain, notify_destinations
|
| 2869 |
+
if not isinstance(raw, dict) or set(raw) != {'kind', 'id'}:
|
| 2870 |
+
raise ValueError('Choose a saved Slack or webhook destination.')
|
| 2871 |
+
kind, identity = raw.get('kind'), raw.get('id')
|
| 2872 |
+
if not isinstance(identity, str) or not identity:
|
| 2873 |
+
raise ValueError('Choose a saved destination.')
|
| 2874 |
+
if kind == 'slack':
|
| 2875 |
+
preference = notify_destinations.get(rt, principal['username'], 'automation_run')
|
| 2876 |
+
if identity not in preference['slackChannels']:
|
| 2877 |
+
raise PermissionError('Choose one of your saved Slack destinations.')
|
| 2878 |
+
elif kind == 'webhook':
|
| 2879 |
+
entry = next((entry for entry in keychain.list_entries(rt, 'generic')
|
| 2880 |
+
if entry['id'] == identity and entry.get('createdBy') == principal['username']), None)
|
| 2881 |
+
if not entry:
|
| 2882 |
+
raise PermissionError('This webhook destination is unavailable.')
|
| 2883 |
+
secret = keychain.read_fields(rt, identity)
|
| 2884 |
+
if not secret or secret.get('purpose') != 'alert_webhook' or not secret.get('url'):
|
| 2885 |
+
raise ValueError('Choose a saved alert webhook destination.')
|
| 2886 |
+
# DNS and redirect checks run again at dispatch through fetch_json.
|
| 2887 |
+
parsed = urlparse(secret['url'])
|
| 2888 |
+
if parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password:
|
| 2889 |
+
raise ValueError('This webhook destination requires a public HTTPS URL.')
|
| 2890 |
+
else:
|
| 2891 |
+
raise ValueError('Choose Slack or webhook.')
|
| 2892 |
+
return {'kind': kind, 'id': identity}
|
| 2893 |
+
|
| 2894 |
+
|
| 2895 |
+
def save_alert(rt, principal, raw, aid=None):
|
| 2896 |
+
"""Save an owner-bound Automation rule starting after the current received-change watermark."""
|
| 2897 |
+
import secrets
|
| 2898 |
+
from core import received_changes
|
| 2899 |
+
username = principal.get('username')
|
| 2900 |
+
current = alert_principal(rt, username)
|
| 2901 |
+
config = clean_alert(rt, current, raw)
|
| 2902 |
+
identity = aid or 'alert_' + secrets.token_hex(12)
|
| 2903 |
+
sequence = received_changes.watermark(rt)
|
| 2904 |
+
result = []
|
| 2905 |
+
|
| 2906 |
+
def change(doc):
|
| 2907 |
+
result.clear()
|
| 2908 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 2909 |
+
old = doc.get(identity)
|
| 2910 |
+
if aid and (not old or not old.get('alert') or old.get('createdBy') != username):
|
| 2911 |
+
raise PermissionError('This alert is unavailable to your account.')
|
| 2912 |
+
if not aid and len(doc) >= MAX_AUTOMATIONS:
|
| 2913 |
+
raise ValueError('This workspace has reached its Automation limit.')
|
| 2914 |
+
state = copy.deepcopy((old or {}).get('state') or {})
|
| 2915 |
+
state.update(alertWatermark=sequence, alertRevision=int(state.get('alertRevision', 0)) + 1)
|
| 2916 |
+
definition = {'id': identity, 'name': config['name'], 'kind': 'plain', 'createdBy': username,
|
| 2917 |
+
'alert': config, 'trigger': {'key': 'received_change', 'configured': True,
|
| 2918 |
+
'enabled': not bool(raw.get('paused', False))}, 'state': state,
|
| 2919 |
+
'config': {}, 'flow': {'actions': []}, 'schedule': {'enabled': False}}
|
| 2920 |
+
doc[identity] = definition
|
| 2921 |
+
result.append(copy.deepcopy(definition))
|
| 2922 |
+
return doc
|
| 2923 |
+
|
| 2924 |
+
_store_update(rt, change)
|
| 2925 |
+
return result[0]
|
| 2926 |
+
|
| 2927 |
+
|
| 2928 |
+
def alert_wire(definition):
|
| 2929 |
+
"""Project saved rule configuration without private event or delivery state."""
|
| 2930 |
+
return {'id': definition['id'], **copy.deepcopy(definition['alert']),
|
| 2931 |
+
'paused': not definition.get('trigger', {}).get('enabled', True)}
|
| 2932 |
+
|
| 2933 |
+
|
| 2934 |
+
def received_alert_match(rt, definition, event):
|
| 2935 |
+
"""Recheck fields and row scope before evaluating one committed received change."""
|
| 2936 |
+
import formula_runtime
|
| 2937 |
+
from core import perm_scope
|
| 2938 |
+
cfg = definition.get('alert')
|
| 2939 |
+
if (not cfg or event.get('tenant') != rt.key or event.get('database') != cfg.get('database')
|
| 2940 |
+
or not definition.get('trigger', {}).get('enabled')
|
| 2941 |
+
or int(event.get('sequence', 0)) <= max(
|
| 2942 |
+
int(definition.get('state', {}).get('alertWatermark', 0)),
|
| 2943 |
+
int(definition.get('state', {}).get('alertDeliveryFloor', 0)))
|
| 2944 |
+
or not set(cfg['fields']).intersection(event.get('changes') or {})):
|
| 2945 |
+
return None
|
| 2946 |
+
expected_operation = 'created' if cfg.get('when') == 'created' else 'updated'
|
| 2947 |
+
if event.get('operation') != expected_operation:
|
| 2948 |
+
return None
|
| 2949 |
+
if (cfg['database'] in LINKABLE_MODULES and event.get('source') == 'typed'
|
| 2950 |
+
and event.get('actor') != definition.get('createdBy')):
|
| 2951 |
+
return None # Native module overlays can contain personal values under the same field key.
|
| 2952 |
+
principal = alert_principal(rt, definition['createdBy'])
|
| 2953 |
+
admitted = clean_alert(rt, principal, cfg)
|
| 2954 |
+
fields = alert_schema(rt, principal, cfg['database'])
|
| 2955 |
+
# Use the existing scoped reader for row and BU restrictions, including connector decoding.
|
| 2956 |
+
row = next((row for row in perm_scope.scoped_table(principal, cfg['database'], st=rt)
|
| 2957 |
+
if str(row.get('pid')) == str(event.get('rowId'))), None)
|
| 2958 |
+
if row is None:
|
| 2959 |
+
return None
|
| 2960 |
+
available = {str(f['key']) for f in fields if f.get('key')}
|
| 2961 |
+
# Event values may have advanced again by evaluation time. Only admitted delta fields replace them.
|
| 2962 |
+
row = {key: value for key, value in row.items() if key in available}
|
| 2963 |
+
for key, delta in (event.get('changes') or {}).items():
|
| 2964 |
+
if key in available:
|
| 2965 |
+
row[key] = delta.get('after')
|
| 2966 |
+
if not lane_match(admitted['condition'], row):
|
| 2967 |
+
return None
|
| 2968 |
+
if admitted['formula']:
|
| 2969 |
+
result = formula_runtime.evaluate_result(formula_runtime.parse_formula(admitted['formula']), row, strict=True)
|
| 2970 |
+
if result['error'] or result['value'] not in (True, 'TRUE', 1):
|
| 2971 |
+
return None
|
| 2972 |
+
return {'principal': principal, 'row': {key: row.get(key) for key in admitted['fields']},
|
| 2973 |
+
'config': admitted}
|
| 2974 |
+
|
| 2975 |
+
|
| 2976 |
+
def received_change_hook(rt, event):
|
| 2977 |
+
"""Evaluate received-change Automations without propagating failures into committed writers."""
|
| 2978 |
+
matched = []
|
| 2979 |
+
for aid, definition in all_definitions(rt).items():
|
| 2980 |
+
if not definition.get('alert'):
|
| 2981 |
+
continue
|
| 2982 |
+
try:
|
| 2983 |
+
match = received_alert_match(rt, definition, event)
|
| 2984 |
+
if match is not None:
|
| 2985 |
+
matched.append(aid)
|
| 2986 |
+
dispatch = globals().get('deliver_received_alert')
|
| 2987 |
+
if callable(dispatch):
|
| 2988 |
+
dispatch(rt, aid, event)
|
| 2989 |
+
except (PermissionError, ValueError):
|
| 2990 |
+
continue
|
| 2991 |
+
return matched
|
| 2992 |
+
|
| 2993 |
+
|
| 2994 |
+
def _send_received_alert(rt, definition, event, match):
|
| 2995 |
+
"""Use the saved Slack destination or encrypted webhook through the existing outbound rail."""
|
| 2996 |
+
from core import keychain, notify_destinations
|
| 2997 |
+
cfg = match['config']
|
| 2998 |
+
destination = alert_destination(rt, match['principal'], cfg['destination'])
|
| 2999 |
+
item = notify_destinations.received_change_item(cfg, event, match['row'])
|
| 3000 |
+
if destination['kind'] == 'slack':
|
| 3001 |
+
import notify_destinations as delivery
|
| 3002 |
+
preference = notify_destinations.get(rt, definition['createdBy'], 'automation_run')
|
| 3003 |
+
return delivery._dispatch_slack(rt, preference, destination['id'], 'automation_run', [item])
|
| 3004 |
+
fields = keychain.read_fields(rt, destination['id'])
|
| 3005 |
+
payload = {'version': 1, 'tenant': rt.key, 'ruleId': definition['id'],
|
| 3006 |
+
'eventId': event['id'], 'receivedAt': event['receivedAt'], 'items': [item]}
|
| 3007 |
+
status, _body = fetch_json(fields['url'], payload, timeout=20, max_kb=64,
|
| 3008 |
+
headers={'Idempotency-Key': event['id'] + ':' + definition['id']})
|
| 3009 |
+
return {'state': 'sent' if 200 <= status < 300 else 'refused',
|
| 3010 |
+
'reason': '' if 200 <= status < 300 else 'destination_refused'}
|
| 3011 |
+
|
| 3012 |
+
|
| 3013 |
+
def deliver_received_alert(rt, aid, event):
|
| 3014 |
+
"""Claim a rule/event once before dispatch and hold ambiguous transport outcomes for review."""
|
| 3015 |
+
definition = all_definitions(rt).get(aid)
|
| 3016 |
+
if not definition or not event.get('id'):
|
| 3017 |
+
return {'state': 'refused', 'reason': 'unknown_alert'}
|
| 3018 |
+
match = received_alert_match(rt, definition, event)
|
| 3019 |
+
if match is None:
|
| 3020 |
+
return {'state': 'refused', 'reason': 'not_authorized_or_matching'}
|
| 3021 |
+
revision = definition.get('state', {}).get('alertRevision')
|
| 3022 |
+
identity = event['id']
|
| 3023 |
+
claimed = []
|
| 3024 |
+
|
| 3025 |
+
def claim(doc):
|
| 3026 |
+
claimed.clear()
|
| 3027 |
+
current = doc.get(aid)
|
| 3028 |
+
if (not current or current.get('state', {}).get('alertRevision') != revision
|
| 3029 |
+
or not current.get('trigger', {}).get('enabled')):
|
| 3030 |
+
return doc
|
| 3031 |
+
receipts = current.setdefault('state', {}).setdefault('alertDeliveries', {})
|
| 3032 |
+
from core import received_changes
|
| 3033 |
+
sequence = int(event['sequence'])
|
| 3034 |
+
floor = max(int(current['state'].get('alertDeliveryFloor', 0)),
|
| 3035 |
+
sequence - received_changes.MAX_EVENTS)
|
| 3036 |
+
current['state']['alertDeliveryFloor'] = floor
|
| 3037 |
+
for prior_id in list(receipts):
|
| 3038 |
+
if int(receipts[prior_id].get('sequence', 0)) <= floor:
|
| 3039 |
+
receipts.pop(prior_id)
|
| 3040 |
+
if sequence <= floor:
|
| 3041 |
+
return doc
|
| 3042 |
+
if identity not in receipts:
|
| 3043 |
+
receipts[identity] = {'state': 'claimed', 'sequence': sequence, 'at': _iso()}
|
| 3044 |
+
claimed.append(True)
|
| 3045 |
+
return doc
|
| 3046 |
+
|
| 3047 |
+
_store_update(rt, claim)
|
| 3048 |
+
if not claimed:
|
| 3049 |
+
return {'state': 'duplicate', 'reason': 'already_attempted'}
|
| 3050 |
+
try:
|
| 3051 |
+
# Configuration, grants and the row wall can change while the claim commits.
|
| 3052 |
+
current = all_definitions(rt).get(aid)
|
| 3053 |
+
if not current or current.get('state', {}).get('alertRevision') != revision:
|
| 3054 |
+
outcome = {'state': 'refused', 'reason': 'rule_changed'}
|
| 3055 |
+
else:
|
| 3056 |
+
try:
|
| 3057 |
+
fresh = received_alert_match(rt, current, event)
|
| 3058 |
+
except (PermissionError, ValueError):
|
| 3059 |
+
fresh = None
|
| 3060 |
+
outcome = (_send_received_alert(rt, current, event, fresh) if fresh else
|
| 3061 |
+
{'state': 'refused', 'reason': 'access_revoked'})
|
| 3062 |
+
except PermissionError:
|
| 3063 |
+
outcome = {'state': 'refused', 'reason': 'access_revoked'}
|
| 3064 |
+
except Exception:
|
| 3065 |
+
# The provider may have accepted a timed-out POST. Never send it again automatically.
|
| 3066 |
+
outcome = {'state': 'unconfirmed', 'reason': 'delivery_unconfirmed'}
|
| 3067 |
+
outcome = {'state': outcome.get('state') if outcome.get('state') in
|
| 3068 |
+
('sent', 'refused', 'unconfirmed') else 'unconfirmed',
|
| 3069 |
+
'reason': str(outcome.get('reason') or '')[:80]}
|
| 3070 |
+
|
| 3071 |
+
def finish(doc):
|
| 3072 |
+
current = doc.get(aid)
|
| 3073 |
+
if current and identity in current.get('state', {}).get('alertDeliveries', {}):
|
| 3074 |
+
current['state']['alertDeliveries'][identity] = {
|
| 3075 |
+
**outcome, 'sequence': int(event['sequence']), 'at': _iso()}
|
| 3076 |
+
return doc
|
| 3077 |
+
|
| 3078 |
+
_store_update(rt, finish)
|
| 3079 |
+
return outcome
|
| 3080 |
+
|
| 3081 |
+
|
| 3082 |
def automation_runs(rt, auto_id):
|
| 3083 |
"""Read run history after a once definition has retired."""
|
| 3084 |
row = (rt.get(STORE_KEY) or {}).get(str(auto_id))
|
|
|
|
| 7080 |
rows, merged = _merge_candidates(rows)
|
| 7081 |
stats["merged"] += merged
|
| 7082 |
|
| 7083 |
+
field_changes = {}
|
| 7084 |
+
|
| 7085 |
def _up(cur, _f=fields, _r=rows):
|
| 7086 |
+
field_changes.clear()
|
| 7087 |
cur = cur or {}
|
| 7088 |
t = dict(cur.get(table_key) or {})
|
| 7089 |
+
old_keys = _ut()._field_list_keys(t)
|
| 7090 |
t["fields"], t["rows"] = _f, _r
|
| 7091 |
+
new_keys = _ut()._field_list_keys(t)
|
| 7092 |
+
field_changes.update(removed=[key for key in old_keys if key not in new_keys],
|
| 7093 |
+
added=[key for key in new_keys if key not in old_keys])
|
| 7094 |
cur[table_key] = t
|
| 7095 |
return cur
|
| 7096 |
|
| 7097 |
rt.update(UT_STORE_KEY, _up)
|
| 7098 |
+
_ut()._log_field_list_change(rt, table_key, 'automation', 'migrate_ig_tables', field_changes)
|
| 7099 |
# ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
|
| 7100 |
#
|
| 7101 |
# W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early
|
|
|
|
| 14495 |
"check this tenant's Odoo connector under Settings -> Connectors; the linked column "
|
| 14496 |
"keeps its last resolved value until the source answers again")
|
| 14497 |
table["rows"] = {str(r.get("pid")): r for r in (pool or []) if r.get("pid") is not None}
|
| 14498 |
+
if key == "product_data":
|
| 14499 |
+
from modules import product_data
|
| 14500 |
+
table["rows"] = {str(row["pid"]): row for row in product_data.apply_lead_overlay(
|
| 14501 |
+
list(table["rows"].values()), table.get("_if_shared_cells"))}
|
| 14502 |
for rid, row in list(table['rows'].items()):
|
| 14503 |
table['rows'][rid] = {**row, **(table.get('_if_shared_cells', {}).get(rid) or {})}
|
| 14504 |
table["_rows_built"] = True
|
|
|
|
| 14611 |
return table, None
|
| 14612 |
|
| 14613 |
|
| 14614 |
+
def _relation_native_rows(rt, key):
|
| 14615 |
+
"""Read omitted native rows only through the caller's tenant runtime."""
|
| 14616 |
+
if not _relation_tenant(rt):
|
| 14617 |
+
raise RuntimeError('A tenant runtime is required to read relation rows.')
|
| 14618 |
+
table = _ut().get_one(key, st=rt)
|
| 14619 |
+
if table is not None and 'rows' not in table:
|
| 14620 |
+
raise RuntimeError('The relation row reader returned only a definition.')
|
| 14621 |
+
return table
|
| 14622 |
+
|
| 14623 |
+
|
| 14624 |
def _resolve_table(rt, store, key, memo, want_rows=False):
|
| 14625 |
"""A database by key, from EITHER namespace — `(table_or_None, limit_or_None)`.
|
| 14626 |
|
|
|
|
| 14654 |
if cached is None:
|
| 14655 |
cached = memo[("mirror", key)] = _mirror_table(rt, key, hit)
|
| 14656 |
return cached
|
| 14657 |
+
if want_rows and 'rows' not in hit:
|
| 14658 |
+
if ('native', key) not in memo:
|
| 14659 |
+
memo[('native', key)] = (_relation_native_rows(rt, key), None)
|
| 14660 |
+
return memo[('native', key)]
|
| 14661 |
return hit, None
|
| 14662 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 14663 |
|
| 14664 |
|
| 14665 |
+
def _link_from_key(fields, bag, overlay_fields=None):
|
| 14666 |
"""Which column on THIS table supplies the join value.
|
| 14667 |
|
| 14668 |
Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
|
| 14669 |
fallback chain is what makes an Instagram database link up with no configuration at all —
|
| 14670 |
the "automatically" in the owner's instruction — and it is the same chain the grid already
|
| 14671 |
uses to decide a table's identity column, rather than a second opinion about it.
|
| 14672 |
+
`overlay_fields` (`{key: Field}` from the shared overlay) joins the fallback after the
|
| 14673 |
+
definition's own fields: a typed overlay column can supply the join too (D-11171).
|
| 14674 |
"""
|
| 14675 |
declared = str((bag or {}).get("from") or "").strip()
|
| 14676 |
if declared:
|
| 14677 |
return declared
|
| 14678 |
+
pool = list(fields) + [dict(f, key=k) for k, f in (overlay_fields or {}).items() if isinstance(f, dict)]
|
| 14679 |
+
prof = next((f for f in pool if isinstance(f.get("profile"), dict)), None)
|
| 14680 |
if prof:
|
| 14681 |
return str(prof.get("key") or "")
|
| 14682 |
+
pin = next((f for f in pool if f.get("pinned") is True), None)
|
| 14683 |
return str(pin.get("key") or "") if pin else ""
|
| 14684 |
|
| 14685 |
|
| 14686 |
+
def _overlay_join_rows(rt, table_key, rows):
|
| 14687 |
+
"""`({key: Field}, {row_id: {key: value}})`: this table's shared overlay columns and their
|
| 14688 |
+
typed cells for `rows`. A join reads a typed overlay value from here (D-11171).
|
| 14689 |
+
|
| 14690 |
+
Lenient like every display read: an unreadable overlay is `({}, {})`, so the join reads the
|
| 14691 |
+
stored row value only, exactly as it did before."""
|
| 14692 |
+
try:
|
| 14693 |
+
from core import shared_overlay
|
| 14694 |
+
declared = shared_overlay.fields(table_key, st=rt)
|
| 14695 |
+
if not declared:
|
| 14696 |
+
return {}, {}
|
| 14697 |
+
pids = [str(rid) for rid in rows if str(rid).isdigit()]
|
| 14698 |
+
return dict(declared), shared_overlay.cells(table_key, pids, st=rt)
|
| 14699 |
+
except Exception: # noqa: BLE001
|
| 14700 |
+
return {}, {}
|
| 14701 |
+
|
| 14702 |
+
|
| 14703 |
+
def _join_value(row, rid, from_key, overlay_cells):
|
| 14704 |
+
"""The join value a source row supplies: the stored cell, else its typed overlay cell."""
|
| 14705 |
+
value = (row or {}).get(from_key)
|
| 14706 |
+
if value is None or value == "":
|
| 14707 |
+
value = ((overlay_cells or {}).get(str(rid)) or {}).get(from_key)
|
| 14708 |
+
return value
|
| 14709 |
+
|
| 14710 |
+
|
| 14711 |
def _join_norm(value):
|
| 14712 |
"""ONE normalisation of an `on` join value, applied to BOTH sides of the join.
|
| 14713 |
|
|
|
|
| 14871 |
ifs = [f for f in fields if f.get('type') == 'if']
|
| 14872 |
if not links and not rollups and not ifs:
|
| 14873 |
return {}, limits
|
| 14874 |
+
rows = (t.get("rows") or {}) if t is not None and 'rows' in t else {}
|
| 14875 |
+
if t is not None and ('rows' not in t or (not rows and (
|
| 14876 |
+
not t.get("_rows_built", True) or not _ut().materialises(table_key, defn=t)))):
|
| 14877 |
# A registry source: NOW the pool is worth its cost, because this table really does
|
| 14878 |
# declare relations. One fetch, memoised, shared with every link and rollup below.
|
| 14879 |
# ⭐⭐ W53-T40 (d) — AND A READ-THROUGH SOURCE TAKES THE SAME SECOND FETCH, for a reason
|
|
|
|
| 14894 |
# --- resolve every link field ONCE per table, not once per row.
|
| 14895 |
# `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
|
| 14896 |
resolved, linked_types, refused = {}, {}, set()
|
| 14897 |
+
overlay = []
|
| 14898 |
+
|
| 14899 |
+
def _overlay(bag):
|
| 14900 |
+
"""`(overlay fields, overlay cells, from key)` for one join. The shared overlay is read
|
| 14901 |
+
once, and only when the `from` column is not a field this table defines."""
|
| 14902 |
+
own = _link_from_key(fields, bag)
|
| 14903 |
+
if own and own in by_key:
|
| 14904 |
+
return {}, {}, own
|
| 14905 |
+
if not overlay:
|
| 14906 |
+
overlay.append(_overlay_join_rows(rt, table_key, rows))
|
| 14907 |
+
return overlay[0] + (_link_from_key(fields, bag, overlay[0][0]),)
|
| 14908 |
+
|
| 14909 |
for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
|
| 14910 |
lk_key = str((f or {}).get("key") or "")
|
| 14911 |
if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
|
|
|
|
| 14972 |
on_key = str(bag.get("on") or "")
|
| 14973 |
hop_in = str(bag.get("throughOn") or "")
|
| 14974 |
hop_out = str(bag.get("throughTo") or "")
|
| 14975 |
+
_of, overlay_cells, from_key = _overlay(bag)
|
| 14976 |
if not (on_key and hop_in and hop_out and from_key):
|
| 14977 |
# ⛔ REFUSED, NOT BLANK — the same law the one-hop arm below follows. A route with
|
| 14978 |
# a missing leg has no second hop to make; resolving it to `[]` would render a
|
|
|
|
| 15010 |
target_idx = _linked_rows_by_join(linked, on_key)
|
| 15011 |
cap = int(_ut().LINK_THROUGH_MAX_IDS)
|
| 15012 |
|
| 15013 |
+
def _walk(row, rid, _bidx=bridge_idx, _tidx=target_idx, _cap=cap, _cells=overlay_cells):
|
| 15014 |
"""One source row's DISTINCT target records, and whether the ceiling stopped it.
|
| 15015 |
|
| 15016 |
⚠ DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines;
|
|
|
|
| 15018 |
label of a set of PRODUCTS.
|
| 15019 |
"""
|
| 15020 |
seen, out = set(), []
|
| 15021 |
+
for _bid, brow in _bidx.get(_join_norm(_join_value(row, rid, from_key, _cells)), ()):
|
| 15022 |
for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()):
|
| 15023 |
if tid in seen:
|
| 15024 |
continue
|
|
|
|
| 15030 |
|
| 15031 |
two_hop, capped = {}, 0
|
| 15032 |
for rid, row in rows.items():
|
| 15033 |
+
hits_2, hit_cap = _walk(row, rid)
|
| 15034 |
two_hop[str(rid)] = hits_2
|
| 15035 |
capped += 1 if hit_cap else 0
|
| 15036 |
resolved[lk_key] = two_hop
|
|
|
|
| 15054 |
effect="capped", limit=cap))
|
| 15055 |
elif bag.get("on"):
|
| 15056 |
idx = _linked_rows_by_join(linked, str(bag["on"]))
|
| 15057 |
+
_of, overlay_cells, from_key = _overlay(bag)
|
| 15058 |
if not from_key:
|
| 15059 |
# ⛔ REFUSED, NOT BLANK. `_link_from_key` falls back to the PROFILE-flagged column
|
| 15060 |
# and then the PINNED one; a table with neither — which a registry module can be,
|
|
|
|
| 15070 |
"database that is the id column, e.g. `partner_id` or `product_id`)"))
|
| 15071 |
continue
|
| 15072 |
resolved[lk_key] = {
|
| 15073 |
+
str(rid): idx.get(_join_norm(_join_value(row, rid, from_key, overlay_cells)), [])
|
| 15074 |
for rid, row in rows.items()}
|
| 15075 |
else:
|
| 15076 |
# An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
|
|
|
|
| 15366 |
return out, list(dict.fromkeys(reasons))
|
| 15367 |
|
| 15368 |
|
| 15369 |
+
#: Compatibility memo for direct computation; GET reads the durable provenance document.
|
| 15370 |
+
_RELATION_LIMITS = {}
|
| 15371 |
+
_RELATION_LIMITS_LOCK = threading.Lock()
|
| 15372 |
+
#: The refusals of the table this thread just refreshed, for the refresh log.
|
| 15373 |
+
_RELATION_PASS = threading.local()
|
| 15374 |
+
RELATION_LIMITS_KEY = "relation_refusals"
|
| 15375 |
+
_RELATION_LIMIT_KEYS = ("subject", "effect", "cause", "recommendation", "code", "limit")
|
| 15376 |
+
|
| 15377 |
+
|
| 15378 |
+
def _relation_tenant(rt):
|
| 15379 |
+
rt = getattr(rt, "_runtime", None) or rt
|
| 15380 |
+
return str(getattr(rt, "key", "") or "")
|
| 15381 |
+
|
| 15382 |
+
|
| 15383 |
+
def remember_relation_limits(rt, table_key, limits):
|
| 15384 |
+
"""Keep this table's newest link refusals for its tenant; an empty list clears them."""
|
| 15385 |
+
tenant = _relation_tenant(rt)
|
| 15386 |
+
if not tenant:
|
| 15387 |
+
return
|
| 15388 |
+
with _RELATION_LIMITS_LOCK:
|
| 15389 |
+
if limits:
|
| 15390 |
+
_RELATION_LIMITS[(tenant, str(table_key))] = [dict(limit) for limit in limits]
|
| 15391 |
+
else:
|
| 15392 |
+
_RELATION_LIMITS.pop((tenant, str(table_key)), None)
|
| 15393 |
+
|
| 15394 |
+
|
| 15395 |
+
def relation_limits(rt, table_key):
|
| 15396 |
+
"""Read the last saved relation provenance without refreshing or writing cells."""
|
| 15397 |
+
rt = getattr(rt, "_runtime", None) or rt
|
| 15398 |
+
document = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
|
| 15399 |
+
return copy.deepcopy(document.get(str(table_key), []))
|
| 15400 |
+
|
| 15401 |
+
|
| 15402 |
+
def _relation_read(rt, name):
|
| 15403 |
+
"""Use the bound backend's strict read and its existing bounded store waits."""
|
| 15404 |
+
if not _relation_tenant(rt) or not rt.available():
|
| 15405 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15406 |
+
resolver = getattr(rt, "_document_store", None)
|
| 15407 |
+
if callable(resolver):
|
| 15408 |
+
backend = resolver() # Validates the store handle's tenant before reading.
|
| 15409 |
+
reader = getattr(backend, "_read_strict", None)
|
| 15410 |
+
if callable(reader):
|
| 15411 |
+
return reader(rt.store_key(name))
|
| 15412 |
+
return backend.get(rt.store_key(name), fresh=True)
|
| 15413 |
+
return rt.get(name, fresh=True)
|
| 15414 |
+
|
| 15415 |
+
|
| 15416 |
+
def _relation_document(rt, raw):
|
| 15417 |
+
"""Reject foreign or malformed provenance instead of treating it as a successful pass."""
|
| 15418 |
+
if raw is None or raw == {}:
|
| 15419 |
+
return {}
|
| 15420 |
+
if (not isinstance(raw, dict) or raw.get("tenant") != _relation_tenant(rt)
|
| 15421 |
+
or type(raw.get("version")) is not int or raw.get("version") != 1
|
| 15422 |
+
or not isinstance(raw.get("tables"), dict)):
|
| 15423 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15424 |
+
tables = raw["tables"]
|
| 15425 |
+
if any(not isinstance(limits, list) or any(not isinstance(limit, dict) for limit in limits)
|
| 15426 |
+
for limits in tables.values()):
|
| 15427 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15428 |
+
for limits in tables.values():
|
| 15429 |
+
for limit in limits:
|
| 15430 |
+
if (any(not isinstance(limit.get(key), str) for key in
|
| 15431 |
+
("subject", "effect", "cause", "recommendation", "fieldSignature"))
|
| 15432 |
+
or not re.fullmatch(r"[0-9a-f]{64}", limit["fieldSignature"])):
|
| 15433 |
+
raise RuntimeError("relation_provenance_unavailable")
|
| 15434 |
+
return tables
|
| 15435 |
+
|
| 15436 |
+
|
| 15437 |
+
def relation_field_signature(field):
|
| 15438 |
+
"""Bind saved provenance to the field definition that produced it."""
|
| 15439 |
+
definition = {k: field[k] for k in ("key", "type", "link", "rollup", "if") if k in field}
|
| 15440 |
+
return hashlib.sha256(json.dumps(definition, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
| 15441 |
+
|
| 15442 |
+
|
| 15443 |
+
def _relation_provenance(table, limits):
|
| 15444 |
+
"""Keep only field-bound diagnostics, excluding row identities, cells and exception payloads."""
|
| 15445 |
+
fields = [f for f in (table or {}).get("fields", []) if f.get("key")]
|
| 15446 |
+
out = []
|
| 15447 |
+
for limit in limits:
|
| 15448 |
+
subject = str(limit.get("subject") or "")
|
| 15449 |
+
matched = [f for f in fields if str(f["key"]) == subject]
|
| 15450 |
+
if not matched:
|
| 15451 |
+
matched = [f for f in fields if str((f.get("link") or {}).get("table") or "") == subject]
|
| 15452 |
+
if not matched and subject == str((table or {}).get("key") or ""):
|
| 15453 |
+
matched = [f for f in fields if f.get("type") in ("rollup", "if") or _ut().is_derived_link(f)]
|
| 15454 |
+
for field in matched:
|
| 15455 |
+
clean = {k: str(limit[k]) for k in _RELATION_LIMIT_KEYS if k in limit}
|
| 15456 |
+
if type(limit.get("limit")) is int:
|
| 15457 |
+
clean["limit"] = limit["limit"]
|
| 15458 |
+
clean["subject"] = str(field["key"])
|
| 15459 |
+
# Source exceptions can contain row values, SQL operands or provider response bodies.
|
| 15460 |
+
if any(marker in clean.get("cause", "") for marker in
|
| 15461 |
+
("could not be read:", "could not answer for it:")):
|
| 15462 |
+
clean["cause"] = "The relation source could not be read. Its last resolved value is retained."
|
| 15463 |
+
clean["recommendation"] = "Check this tenant's source connector, then retry the relation refresh."
|
| 15464 |
+
clean["fieldSignature"] = relation_field_signature(field)
|
| 15465 |
+
out.append(clean)
|
| 15466 |
+
return out
|
| 15467 |
+
|
| 15468 |
+
|
| 15469 |
+
def _persist_relation_limits(rt, results):
|
| 15470 |
+
"""Save one small tenant-bound document per pass, including recovery with no changed cells."""
|
| 15471 |
+
if not results:
|
| 15472 |
+
return
|
| 15473 |
+
old = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
|
| 15474 |
+
def merged(tables):
|
| 15475 |
+
tables = copy.deepcopy(tables)
|
| 15476 |
+
for key, limits in results.items():
|
| 15477 |
+
if limits:
|
| 15478 |
+
tables[str(key)] = copy.deepcopy(limits)
|
| 15479 |
+
else:
|
| 15480 |
+
tables.pop(str(key), None)
|
| 15481 |
+
return tables
|
| 15482 |
+
if merged(old) == old:
|
| 15483 |
+
return
|
| 15484 |
+
def up(raw):
|
| 15485 |
+
return {"version": 1, "tenant": _relation_tenant(rt),
|
| 15486 |
+
"tables": merged(_relation_document(rt, raw))}
|
| 15487 |
+
rt.update(RELATION_LIMITS_KEY, up, flush="sync")
|
| 15488 |
+
|
| 15489 |
+
|
| 15490 |
+
def relation_limit_line(table_key, limit):
|
| 15491 |
+
"""One log line for one refused link: the field, the cause and the fix."""
|
| 15492 |
+
limit = limit or {}
|
| 15493 |
+
cause = str(limit.get("cause") or "").strip().rstrip(".")
|
| 15494 |
+
fix = str(limit.get("recommendation") or "").strip().rstrip(".")
|
| 15495 |
+
return (f"[aios-auto] relation refresh {table_key}: link {limit.get('subject')} refused: "
|
| 15496 |
+
f"{cause}. Fix: {fix}.")
|
| 15497 |
+
|
| 15498 |
+
|
| 15499 |
def compute_relation_cells(rt, table_key, tables=None):
|
| 15500 |
"""Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
|
| 15501 |
|
|
|
|
| 15522 |
another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the
|
| 15523 |
caller that renders them reads them, and nothing about this door changes shape.
|
| 15524 |
"""
|
| 15525 |
+
changes, limits = relation_cells(rt, table_key, tables=tables)
|
| 15526 |
+
# D-11172: the refusals leave through state, never through this signature (see above).
|
| 15527 |
+
_RELATION_PASS.limits = [dict(limit) for limit in limits]
|
| 15528 |
+
remember_relation_limits(rt, table_key, limits)
|
| 15529 |
if not changes:
|
| 15530 |
return 0
|
| 15531 |
if str(table_key) in LINKABLE_MODULES:
|
|
|
|
| 15568 |
keyword this function briefly added (see `compute_relation_cells`).
|
| 15569 |
"""
|
| 15570 |
touched = 0
|
| 15571 |
+
_RELATION_PASS.results = {}
|
| 15572 |
for tk, table in list((tables or {}).items()):
|
| 15573 |
fields = (table or {}).get("fields") or []
|
| 15574 |
if not any(f.get("type") in ('rollup', 'if') or _ut().is_derived_link(f) for f in fields):
|
| 15575 |
+
remember_relation_limits(rt, tk, [])
|
| 15576 |
+
_RELATION_PASS.results[str(tk)] = []
|
| 15577 |
continue
|
| 15578 |
+
_RELATION_PASS.limits = []
|
| 15579 |
try:
|
| 15580 |
touched += compute_relation_cells(rt, tk, tables=tables)
|
| 15581 |
+
_RELATION_PASS.results[str(tk)] = _relation_provenance(table, _RELATION_PASS.limits)
|
| 15582 |
except Exception as e: # noqa: BLE001
|
| 15583 |
log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
|
| 15584 |
+
# D-11172: one line per refused link, its cause and its fix (standing rule 1).
|
| 15585 |
+
for limit in getattr(_RELATION_PASS, "limits", None) or ():
|
| 15586 |
+
log(relation_limit_line(tk, limit))
|
| 15587 |
return touched
|
| 15588 |
|
| 15589 |
|
| 15590 |
+
class _RelationSnapshot(dict):
|
| 15591 |
+
"""Load relation dependencies through the existing single-table readers on demand."""
|
| 15592 |
+
|
| 15593 |
+
def __init__(self, rt, definitions=None, supplied=None):
|
| 15594 |
+
super().__init__()
|
| 15595 |
+
self.runtime, self.definitions, self.supplied = rt, definitions or {}, supplied
|
| 15596 |
+
self.missing = set()
|
| 15597 |
+
|
| 15598 |
+
def get(self, key, default=None):
|
| 15599 |
+
key = str(key)
|
| 15600 |
+
if key not in self and key not in self.missing:
|
| 15601 |
+
if self.supplied is not None and key in self.supplied:
|
| 15602 |
+
table = copy.deepcopy(self.supplied[key])
|
| 15603 |
+
if table is not None and 'rows' not in table and _ut().materialises(key, defn=table):
|
| 15604 |
+
table = _relation_native_rows(self.runtime, key)
|
| 15605 |
+
else:
|
| 15606 |
+
definition = self.definitions.get(key)
|
| 15607 |
+
if definition is None:
|
| 15608 |
+
definition = _ut().get_definition(key, st=self.runtime)
|
| 15609 |
+
table = (_relation_native_rows(self.runtime, key)
|
| 15610 |
+
if definition and _ut().materialises(key, defn=definition)
|
| 15611 |
+
else copy.deepcopy(definition))
|
| 15612 |
+
if table is None:
|
| 15613 |
+
self.missing.add(key)
|
| 15614 |
+
else:
|
| 15615 |
+
self[key] = table
|
| 15616 |
+
return super().get(key, default)
|
| 15617 |
+
|
| 15618 |
+
|
| 15619 |
+
def _relation_snapshot(rt, target=None, supplied=None):
|
| 15620 |
+
"""Read target definitions first and rows only for tables that declare relations."""
|
| 15621 |
+
if target is not None:
|
| 15622 |
+
selected = [target] if isinstance(target, str) else list(target)
|
| 15623 |
+
definitions = {str(key): (supplied.get(str(key)) if supplied is not None else
|
| 15624 |
+
_ut().get_definition(str(key), st=rt)) for key in selected}
|
| 15625 |
+
else:
|
| 15626 |
+
definitions = supplied if supplied is not None else _ut().all_defs(st=rt, schema_only=True)
|
| 15627 |
+
snapshot = _RelationSnapshot(rt, definitions, supplied)
|
| 15628 |
+
for key, definition in definitions.items():
|
| 15629 |
+
if any(f.get('type') in ('rollup', 'if') or _ut().is_derived_link(f)
|
| 15630 |
+
for f in (definition or {}).get('fields') or []):
|
| 15631 |
+
snapshot.get(key)
|
| 15632 |
+
return snapshot
|
| 15633 |
+
|
| 15634 |
+
|
| 15635 |
+
def refresh_relations(rt, log=print, tables=None, persist=True, *, target=None):
|
| 15636 |
"""Compute relations outside the write lock and merge only unchanged derived cells."""
|
| 15637 |
+
snapshot = _relation_snapshot(rt, target=target, supplied=tables)
|
| 15638 |
# Keep authored inputs before computation, including nested cell values.
|
| 15639 |
+
before = copy.deepcopy(dict(snapshot))
|
| 15640 |
local = _refresh_relations_inplace(snapshot, log=log, rt=rt)
|
| 15641 |
+
results = copy.deepcopy(_RELATION_PASS.results)
|
| 15642 |
+
if not persist:
|
| 15643 |
+
return local
|
| 15644 |
+
if not local:
|
| 15645 |
+
_persist_relation_limits(rt, results)
|
| 15646 |
return local
|
| 15647 |
patches = {}
|
| 15648 |
for key, old in before.items():
|
| 15649 |
+
if not isinstance(old, dict) or 'rows' not in old:
|
| 15650 |
continue
|
| 15651 |
fields = old.get('fields') or []
|
| 15652 |
derived = {str(f['key']) for f in fields if f.get('key')
|
|
|
|
| 15661 |
if changed:
|
| 15662 |
patches.setdefault(str(key), {})[str(rid)] = (row, changed)
|
| 15663 |
if not patches:
|
| 15664 |
+
_persist_relation_limits(rt, results)
|
| 15665 |
return 0
|
| 15666 |
actual = [0]
|
| 15667 |
+
field_changes = {}
|
| 15668 |
+
committed = {}
|
| 15669 |
|
| 15670 |
def _up(cur):
|
| 15671 |
actual[0] = 0
|
| 15672 |
+
field_changes.clear()
|
| 15673 |
+
committed.clear()
|
| 15674 |
lock = getattr(cur, 'lock_records', None)
|
| 15675 |
if callable(lock):
|
| 15676 |
lock({key: list(rows) for key, rows in patches.items()})
|
| 15677 |
for key, rows in patches.items():
|
| 15678 |
table = (cur or {}).get(key)
|
| 15679 |
if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'):
|
| 15680 |
+
results.pop(key, None)
|
| 15681 |
continue
|
| 15682 |
+
old_keys = _ut()._field_list_keys(table)
|
| 15683 |
current_rows = table.get('rows')
|
| 15684 |
if current_rows is None:
|
| 15685 |
+
results.pop(key, None)
|
| 15686 |
continue
|
| 15687 |
for rid, (old_row, changes) in rows.items():
|
| 15688 |
current = current_rows.get(rid)
|
| 15689 |
# A changed authored input or a deleted row waits for the next refresh.
|
| 15690 |
if not isinstance(current, dict) or current != old_row:
|
| 15691 |
+
results.pop(key, None)
|
| 15692 |
continue
|
| 15693 |
+
delta = {field: {'before': copy.deepcopy(current.get(field)), 'after': copy.deepcopy(value)}
|
| 15694 |
+
for field, value in changes.items() if current.get(field) != value}
|
| 15695 |
current.update(changes)
|
| 15696 |
actual[0] += 1
|
| 15697 |
+
if delta:
|
| 15698 |
+
committed[(key, rid)] = delta
|
| 15699 |
+
new_keys = _ut()._field_list_keys(table)
|
| 15700 |
+
field_changes[key] = {'removed': [name for name in old_keys if name not in new_keys],
|
| 15701 |
+
'added': [name for name in new_keys if name not in old_keys]}
|
| 15702 |
return cur
|
| 15703 |
|
| 15704 |
+
# A cold process must see recovered cells before their old refusal disappears.
|
| 15705 |
+
saved = rt.update(UT_STORE_KEY, _up, flush='sync')
|
| 15706 |
+
if saved is False:
|
| 15707 |
+
raise RuntimeError('The relation refresh was not committed.')
|
| 15708 |
+
_persist_relation_limits(rt, results)
|
| 15709 |
+
for key, change in field_changes.items():
|
| 15710 |
+
_ut()._log_field_list_change(rt, key, 'automation', 'refresh_relations', change)
|
| 15711 |
+
if committed:
|
| 15712 |
+
import secrets
|
| 15713 |
+
from core import received_changes
|
| 15714 |
+
receipt = 'relation:' + secrets.token_hex(16)
|
| 15715 |
+
for (key, rid), delta in committed.items():
|
| 15716 |
+
received_changes.publish(rt, source='relation', database=key, row_id=rid,
|
| 15717 |
+
changes=delta, receipt_id=receipt)
|
| 15718 |
return actual[0]
|
| 15719 |
|
| 15720 |
|
|
|
|
| 16817 |
return "An authorized automation completed."
|
| 16818 |
|
| 16819 |
|
| 16820 |
+
NOTIFICATION_PAYLOAD_VERSION = 1
|
| 16821 |
+
|
| 16822 |
+
|
| 16823 |
+
def _authorized_notification_text(rt, stored, payload, context):
|
| 16824 |
+
"""Authorize the current recipient, channel and every source before composing notification content."""
|
| 16825 |
+
from core import account_rows, notify_destinations, perm_scope, perms, user_tables, users
|
| 16826 |
+
if (not isinstance(payload, dict) or type(payload.get('version')) is not int
|
| 16827 |
+
or payload.get('version') != NOTIFICATION_PAYLOAD_VERSION):
|
| 16828 |
+
return None
|
| 16829 |
+
aid = stored.get('id')
|
| 16830 |
+
expected = {key: payload.get(key) for key in ('version', 'tenant', 'user', 'kind', 'channel')}
|
| 16831 |
+
if (expected != stored.get('notificationDestination') or payload.get('automationId') != aid
|
| 16832 |
+
or expected['tenant'] != getattr(rt, 'key', None) or expected['channel'] != context.get('channel')
|
| 16833 |
+
or not isinstance(expected['user'], str) or not expected['user']
|
| 16834 |
+
or stored.get('createdBy') != expected['user']
|
| 16835 |
+
or stored.get('trigger', {}).get('key') != 'notification'):
|
| 16836 |
+
return None
|
| 16837 |
+
if aid != notify_destinations.automation_id(rt.key, expected['user'], expected['kind'], expected['channel']):
|
| 16838 |
+
return None
|
| 16839 |
+
resolve = getattr(rt, '_document_store', None)
|
| 16840 |
+
record = account_rows.get(expected['user'], st=resolve() if callable(resolve) else rt, fresh=True)
|
| 16841 |
+
if not record or record.get('active', True) is False or record.get('tenant') != rt.key:
|
| 16842 |
+
return None
|
| 16843 |
+
recipient = users._public(expected['user'], record)
|
| 16844 |
+
principal = context.get('principal')
|
| 16845 |
+
if not isinstance(principal, dict) or not principal.get('username'):
|
| 16846 |
+
return None
|
| 16847 |
+
notify_destinations.kind_id(expected['kind'])
|
| 16848 |
+
raw_items = payload.get('items')
|
| 16849 |
+
if not isinstance(raw_items, list) or not raw_items:
|
| 16850 |
+
return None
|
| 16851 |
+
for item in raw_items:
|
| 16852 |
+
source = item.get('source') if isinstance(item, dict) else None
|
| 16853 |
+
if (not isinstance(source, dict) or set(source) != {'table', 'rowIds'}
|
| 16854 |
+
or not isinstance(source['table'], str) or not source['table']
|
| 16855 |
+
or not isinstance(source['rowIds'], list) or not source['rowIds']
|
| 16856 |
+
or any(not isinstance(rid, str) or not rid for rid in source['rowIds'])
|
| 16857 |
+
or not _notification_source_readable(rt, context, source)):
|
| 16858 |
+
return None
|
| 16859 |
+
items = notify_destinations.clean_items(raw_items)
|
| 16860 |
+
for item in items:
|
| 16861 |
+
source = item['source']
|
| 16862 |
+
table, wanted = source['table'], set(source['rowIds'])
|
| 16863 |
+
if table == 'applications':
|
| 16864 |
+
import application_registry
|
| 16865 |
+
import app_templates
|
| 16866 |
+
applications = application_registry.load(rt, strict=True)['applications']
|
| 16867 |
+
for app_id in wanted:
|
| 16868 |
+
app = applications.get(app_id)
|
| 16869 |
+
if (not isinstance(app, dict) or app.get('id') != app_id or app.get('tenant') != rt.key
|
| 16870 |
+
or app.get('retired') is True):
|
| 16871 |
+
return None
|
| 16872 |
+
template = app_templates.BY_KIND.get(app.get('templateKind'))
|
| 16873 |
+
if not template or app_templates.held(app.get('templateKind')):
|
| 16874 |
+
return None
|
| 16875 |
+
sources = (*template['sources'], *template.get('optional_sources', ()))
|
| 16876 |
+
enabled = (getattr(getattr(rt, 'tenant', None), 'config', None) or {}).get('modules', 'all')
|
| 16877 |
+
if any((enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset))
|
| 16878 |
+
or source not in enabled))
|
| 16879 |
+
or not (user_tables.get_definition(source, st=rt) or {}).get('fields')
|
| 16880 |
+
for source in sources):
|
| 16881 |
+
return None
|
| 16882 |
+
for user in (recipient, principal):
|
| 16883 |
+
if not application_registry.may_read(app, user['username'], perms.is_admin(user)):
|
| 16884 |
+
return None
|
| 16885 |
+
access = application_registry.app_access(user, rt, app, sources)
|
| 16886 |
+
if access['app'] != 'ready' or any(wall['rows'] or wall['hidden']
|
| 16887 |
+
for wall in access['narrow'].values()):
|
| 16888 |
+
return None
|
| 16889 |
+
else:
|
| 16890 |
+
for user in (recipient, principal):
|
| 16891 |
+
if not perm_scope.may_read(user, table, st=rt) or perm_scope.wall_declared(user, table, st=rt):
|
| 16892 |
+
return None
|
| 16893 |
+
missing = set(wanted)
|
| 16894 |
+
for row in perm_scope.scoped_table(user, table, st=rt):
|
| 16895 |
+
missing.discard(str(row.get('pid')))
|
| 16896 |
+
if not missing:
|
| 16897 |
+
break
|
| 16898 |
+
if missing:
|
| 16899 |
+
return None
|
| 16900 |
+
return '\n\n'.join('\n'.join(str(item[key]) for key in ('title', 'body', 'url') if item.get(key))
|
| 16901 |
+
for item in items)
|
| 16902 |
+
|
| 16903 |
+
|
| 16904 |
+
def _notification_source_readable(rt, context, source):
|
| 16905 |
+
"""Require destination read access to every source row and field before using supplied content."""
|
| 16906 |
+
import routes_slack
|
| 16907 |
+
from core import perm_scope
|
| 16908 |
+
agent = routes_slack._agents(rt).get(context["agentId"])
|
| 16909 |
+
module, ids = source["table"], set(source["rowIds"])
|
| 16910 |
+
if not agent or agent.get("active", True) is False:
|
| 16911 |
+
return False
|
| 16912 |
+
if module == "applications":
|
| 16913 |
+
import application_registry
|
| 16914 |
+
import app_templates
|
| 16915 |
+
apps = application_registry.load(rt, strict=True)["applications"]
|
| 16916 |
+
for rid in ids:
|
| 16917 |
+
app = apps.get(rid)
|
| 16918 |
+
if (not isinstance(app, dict) or app.get("tenant") != rt.key or app.get("retired")
|
| 16919 |
+
or not application_registry.may_read(app, context["principal"]["username"])):
|
| 16920 |
+
return False
|
| 16921 |
+
template = app_templates.BY_KIND.get(app.get("templateKind")) or {}
|
| 16922 |
+
sources = template.get("sources") or []
|
| 16923 |
+
if not sources or any(not _notification_source_readable(
|
| 16924 |
+
rt, context, {"table": key, "rowIds": []}) for key in sources):
|
| 16925 |
+
return False
|
| 16926 |
+
return bool(ids)
|
| 16927 |
+
if not routes_slack.agent_may_read(agent, module):
|
| 16928 |
+
return False
|
| 16929 |
+
definition = _ut().get_definition(module, st=rt)
|
| 16930 |
+
if not ids:
|
| 16931 |
+
table, refusal = definition, None
|
| 16932 |
+
elif definition is None:
|
| 16933 |
+
table, refusal = _module_table(rt, module, {}, want_rows=True)
|
| 16934 |
+
elif _ut().materialises(module, defn=definition):
|
| 16935 |
+
table, refusal = _ut().get_records(module, ids, st=rt), None
|
| 16936 |
+
else:
|
| 16937 |
+
table, refusal = _resolve_table(rt, {module: definition}, module, {}, want_rows=True)
|
| 16938 |
+
if refusal or not isinstance(table, dict):
|
| 16939 |
+
return False
|
| 16940 |
+
fields = table.get("fields") or []
|
| 16941 |
+
if not fields or perm_scope.hidden_keys(context["principal"], module, fields, st=rt):
|
| 16942 |
+
return False
|
| 16943 |
+
if not ids:
|
| 16944 |
+
return not perm_scope.row_scope_applies(context["principal"], module, st=rt, defn=definition)
|
| 16945 |
+
rows = table.get("rows") or {}
|
| 16946 |
+
if not ids <= set(rows):
|
| 16947 |
+
return False
|
| 16948 |
+
selected = [{**rows[rid], "pid": rid} for rid in ids]
|
| 16949 |
+
readable = routes_slack.agent_rows(agent, selected, module, fields, st=rt)
|
| 16950 |
+
return {str(row.get("pid")) for row in readable} == ids
|
| 16951 |
+
|
| 16952 |
+
|
| 16953 |
def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
|
| 16954 |
"""Deliver one authorized event result, with readable linked records for an enters_view cohort."""
|
| 16955 |
definition = defn if isinstance(defn, dict) else {}
|
| 16956 |
aid = str(definition.get("id") or "").strip()
|
| 16957 |
if not aid:
|
| 16958 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16959 |
+
if re.fullmatch(r"nd_[0-9a-f]{48}", aid):
|
| 16960 |
+
from core import notify_destinations
|
| 16961 |
+
try:
|
| 16962 |
+
stored = notify_destinations.slack_automation(rt, aid)
|
| 16963 |
+
except Exception: # noqa: BLE001
|
| 16964 |
+
return {"sent": False, "reason": "unknown_automation"}
|
| 16965 |
+
else:
|
| 16966 |
+
stored = all_definitions(rt).get(aid)
|
| 16967 |
if not isinstance(stored, dict):
|
| 16968 |
return {"sent": False, "reason": "unknown_automation"}
|
| 16969 |
trigger = stored.get("trigger") if isinstance(stored.get("trigger"), dict) else {}
|
|
|
|
| 16987 |
context = routes_slack.automation_answer_context(rt, aid)
|
| 16988 |
if context is None:
|
| 16989 |
return {"sent": False, "reason": "not_authorized"}
|
| 16990 |
+
if isinstance(result, dict) and "notification" in result:
|
| 16991 |
+
try:
|
| 16992 |
+
text = _authorized_notification_text(rt, stored, result['notification'], context)
|
| 16993 |
+
except Exception:
|
| 16994 |
+
text = None
|
| 16995 |
+
if text is None:
|
| 16996 |
+
return {'sent': False, 'reason': 'notification_not_authorized'}
|
| 16997 |
+
provenance = {'basis': 'authorized_app_data',
|
| 16998 |
+
'citationIds': [item['id'] for item in result['notification']['items']],
|
| 16999 |
+
'modelGuidance': False}
|
| 17000 |
+
delivered = routes_slack.deliver_channel_reply(
|
| 17001 |
+
rt, context['channel'], text, principal=context['principal'], automation_id=aid,
|
| 17002 |
+
provenance=provenance, post=post)
|
| 17003 |
+
if delivered.get('sent') is True and delivered.get('channel') == context['channel']:
|
| 17004 |
+
delivered['notificationVersion'] = NOTIFICATION_PAYLOAD_VERSION
|
| 17005 |
+
return delivered
|
| 17006 |
provenance = {"basis": "model_guidance", "citationIds": [], "modelGuidance": False}
|
| 17007 |
if trigger_key == "enters_view":
|
| 17008 |
module = str(trigger.get("table") or _flow_table(stored) or "")
|
api/case_drafts.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Compose a Seller Support draft from complete, authorized claim facts
|
| 2 |
import json
|
| 3 |
import os
|
| 4 |
import re
|
|
@@ -63,35 +63,50 @@ def _messages(kind, facts):
|
|
| 63 |
{"role": "user", "content": json.dumps({"kind": kind, "facts": facts}, ensure_ascii=True)}]
|
| 64 |
|
| 65 |
|
| 66 |
-
def _provider_chat(provider, messages, tools, timeout):
|
| 67 |
import providers
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
"""Return a verified model draft or the template with a reason safe to show to its authorized reader."""
|
| 96 |
if not facts.get("complete"):
|
| 97 |
return fallback(facts, facts.get("reasonCode") or "incomplete_evidence",
|
|
@@ -123,7 +138,7 @@ def draft(kind, facts, chat=None):
|
|
| 123 |
break
|
| 124 |
try:
|
| 125 |
answer, problem = ((chat(messages, tools), "") if chat is not None else
|
| 126 |
-
_provider_chat(provider, messages, tools, min(TIMEOUT_SECONDS, remaining)))
|
| 127 |
if problem:
|
| 128 |
last_code = problem
|
| 129 |
last_reason = "The draft provider failed (" + problem + "). Use the template."
|
|
|
|
| 1 |
+
"""Compose a Seller Support draft from complete, authorized claim facts."""
|
| 2 |
import json
|
| 3 |
import os
|
| 4 |
import re
|
|
|
|
| 63 |
{"role": "user", "content": json.dumps({"kind": kind, "facts": facts}, ensure_ascii=True)}]
|
| 64 |
|
| 65 |
|
| 66 |
+
def _provider_chat(provider, messages, tools, timeout, *, st=None, user=""):
|
| 67 |
import providers
|
| 68 |
+
import usage_ledger
|
| 69 |
+
body, received = None, False
|
| 70 |
+
try:
|
| 71 |
+
if provider["shape"] == "anthropic":
|
| 72 |
+
request = providers.anthropic_request(
|
| 73 |
+
model=provider["model"], key=os.environ[provider["env"]].strip(), system=None,
|
| 74 |
+
messages=messages, tools=tools, max_tokens=6500, tool_choice="required")
|
| 75 |
+
status, body, _transport = providers.anthropic_send(request, timeout=timeout)
|
| 76 |
+
received = True
|
| 77 |
+
if status != 200:
|
| 78 |
+
return None, f"http_{status}"
|
| 79 |
+
_text, args, refused = providers.anthropic_read(body)
|
| 80 |
+
return (None, "refused") if refused else (args, "")
|
| 81 |
+
import requests
|
| 82 |
+
response = requests.post(
|
| 83 |
+
provider["url"], timeout=timeout, allow_redirects=False,
|
| 84 |
+
headers={"Authorization": f"Bearer {os.environ[provider['env']].strip()}"},
|
| 85 |
+
json={"model": provider["model"], "messages": messages, "tools": tools,
|
| 86 |
+
"tool_choice": "required", "temperature": 0.1, "max_tokens": 6500})
|
| 87 |
+
received = True
|
| 88 |
+
if response.status_code != 200:
|
| 89 |
+
try:
|
| 90 |
+
body = response.json()
|
| 91 |
+
except ValueError:
|
| 92 |
+
pass
|
| 93 |
+
return None, f"http_{response.status_code}"
|
| 94 |
+
body = response.json()
|
| 95 |
+
if (body.get("choices") or [{}])[0].get("finish_reason") == "length":
|
| 96 |
+
return None, "incomplete_answer"
|
| 97 |
+
calls = (((body.get("choices") or [{}])[0].get("message") or {}).get("tool_calls") or [])
|
| 98 |
+
if len(calls) != 1 or calls[0].get("function", {}).get("name") != "compose_case":
|
| 99 |
+
return None, "malformed"
|
| 100 |
+
return json.loads(calls[0]["function"]["arguments"]), ""
|
| 101 |
+
finally:
|
| 102 |
+
if received:
|
| 103 |
+
measured = body if isinstance(body, dict) else {}
|
| 104 |
+
ins, outs = usage_ledger.tokens_from(measured)
|
| 105 |
+
usage_ledger.record("automation_draft", provider["name"], provider["model"], ins, outs,
|
| 106 |
+
total=usage_ledger.total_from(measured), st=st, user=user)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def draft(kind, facts, chat=None, *, st=None, user=""):
|
| 110 |
"""Return a verified model draft or the template with a reason safe to show to its authorized reader."""
|
| 111 |
if not facts.get("complete"):
|
| 112 |
return fallback(facts, facts.get("reasonCode") or "incomplete_evidence",
|
|
|
|
| 138 |
break
|
| 139 |
try:
|
| 140 |
answer, problem = ((chat(messages, tools), "") if chat is not None else
|
| 141 |
+
_provider_chat(provider, messages, tools, min(TIMEOUT_SECONDS, remaining), st=st, user=user))
|
| 142 |
if problem:
|
| 143 |
last_code = problem
|
| 144 |
last_reason = "The draft provider failed (" + problem + "). Use the template."
|
api/connector_onboarding.py
CHANGED
|
@@ -136,14 +136,28 @@ def mirror_cursor(rt, table_key=None):
|
|
| 136 |
if rt is None:
|
| 137 |
return datastore.ro_con()
|
| 138 |
rt.assert_datastore_matches()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
entities = selected_mirror_entities(rt)
|
| 140 |
if entities is None:
|
| 141 |
import odoo_relational as rel
|
| 142 |
if (table_key is not None
|
| 143 |
and rel.base_table_key(table_key) in rel.stock_source_specs()
|
| 144 |
-
and not
|
| 145 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 146 |
-
return
|
| 147 |
if table_key is not None:
|
| 148 |
import odoo_relational as rel
|
| 149 |
import routes_odoo_tables as route
|
|
@@ -152,13 +166,13 @@ def mirror_cursor(rt, table_key=None):
|
|
| 152 |
raise RuntimeError("Enable this database in connection settings before reading it.")
|
| 153 |
source = route._odoo_sources().get(table_key)
|
| 154 |
if source is None:
|
| 155 |
-
return
|
| 156 |
entities = set(source.get("tables", {}).values())
|
| 157 |
if source.get("table"):
|
| 158 |
entities.add(source["table"])
|
| 159 |
if source.get("needs_excluded"):
|
| 160 |
entities.add("res_partner")
|
| 161 |
-
if any(not
|
| 162 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 163 |
# An empty first selection has no readers to run and need not create a mirror file.
|
| 164 |
-
return
|
|
|
|
| 136 |
if rt is None:
|
| 137 |
return datastore.ro_con()
|
| 138 |
rt.assert_datastore_matches()
|
| 139 |
+
mirror = rt.mirror_handle()
|
| 140 |
+
if str(table_key or "").startswith("ut_odoo_model_"):
|
| 141 |
+
import odoo_relational as rel
|
| 142 |
+
config = rt.get(rel.CONFIG_KEY)
|
| 143 |
+
if not config.get("grids", {}).get(table_key, False):
|
| 144 |
+
raise RuntimeError("Enable this database in connection settings before reading it.")
|
| 145 |
+
cur = mirror.ro_cursor()
|
| 146 |
+
try:
|
| 147 |
+
if rel.dynamic_model_source(cur, table_key, tenant_key=rt.key) is None:
|
| 148 |
+
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 149 |
+
return cur
|
| 150 |
+
except Exception:
|
| 151 |
+
cur.close()
|
| 152 |
+
raise
|
| 153 |
entities = selected_mirror_entities(rt)
|
| 154 |
if entities is None:
|
| 155 |
import odoo_relational as rel
|
| 156 |
if (table_key is not None
|
| 157 |
and rel.base_table_key(table_key) in rel.stock_source_specs()
|
| 158 |
+
and not mirror.entity_live("stock_quant")):
|
| 159 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 160 |
+
return mirror.ro_con()
|
| 161 |
if table_key is not None:
|
| 162 |
import odoo_relational as rel
|
| 163 |
import routes_odoo_tables as route
|
|
|
|
| 166 |
raise RuntimeError("Enable this database in connection settings before reading it.")
|
| 167 |
source = route._odoo_sources().get(table_key)
|
| 168 |
if source is None:
|
| 169 |
+
return mirror.ro_con()
|
| 170 |
entities = set(source.get("tables", {}).values())
|
| 171 |
if source.get("table"):
|
| 172 |
entities.add(source["table"])
|
| 173 |
if source.get("needs_excluded"):
|
| 174 |
entities.add("res_partner")
|
| 175 |
+
if any(not mirror.entity_live(key) for key in entities):
|
| 176 |
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 177 |
# An empty first selection has no readers to run and need not create a mirror file.
|
| 178 |
+
return mirror.ro_cursor() if entities else None
|
api/deps.py
CHANGED
|
@@ -180,21 +180,173 @@ def _assistant_refuse(status, code, message):
|
|
| 180 |
raise err(status, code, message)
|
| 181 |
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
def assistant_unknown_field_message(requested, visible_keys):
|
| 184 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
candidates = sorted({key for key in visible_keys if isinstance(key, str) and key})
|
| 186 |
requested = [requested] if isinstance(requested, str) else requested
|
| 187 |
-
matches = []
|
| 188 |
for key in requested if isinstance(requested, (list, tuple)) else ():
|
| 189 |
if isinstance(key, str) and key not in candidates:
|
|
|
|
|
|
|
|
|
|
| 190 |
for match in difflib.get_close_matches(key, candidates, n=5, cutoff=0.6):
|
| 191 |
if match not in matches:
|
| 192 |
matches.append(match)
|
| 193 |
-
message =
|
| 194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
if matches:
|
| 196 |
message += " Close visible field keys: " + ", ".join(matches[:5]) + "."
|
| 197 |
-
return message
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
|
| 199 |
|
| 200 |
def _assistant_tenant_allows(session: Session, database: str) -> bool:
|
|
@@ -217,13 +369,18 @@ def _assistant_preflight_request(session: Session, database: str, canonical_fiel
|
|
| 217 |
"your current permissions do not allow this database")
|
| 218 |
visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, database)
|
| 219 |
if grant is not None else list(canonical_fields or ()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
try:
|
| 221 |
-
perm_scope.validate_assistant_filter((grant or {}).get("filter"),
|
| 222 |
-
canonical_fields if permission_fields is None else permission_fields)
|
| 223 |
-
_assistant_requested_fields(requested_fields, visible)
|
| 224 |
perm_scope.validate_assistant_filter(filters, visible)
|
| 225 |
except ValueError as exc:
|
| 226 |
-
|
| 227 |
return grant, visible
|
| 228 |
|
| 229 |
|
|
@@ -263,14 +420,16 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
|
|
| 263 |
if field["key"] not in have)]
|
| 264 |
fields, _ = schema_overlay.for_grid(database, fields, st=session.runtime, contract=fields)
|
| 265 |
internal = grid.get("_permission_scope") or {"fields": fields, "rows": [], "row_fields": fields}
|
| 266 |
-
|
| 267 |
-
|
|
|
|
| 268 |
for member in members:
|
|
|
|
| 269 |
try:
|
| 270 |
-
perm_scope.validate_assistant_filter(
|
| 271 |
-
(perm_scope.assistant_entry(member, database) or {}).get("filter"), internal["fields"])
|
| 272 |
except ValueError as exc:
|
| 273 |
-
|
|
|
|
| 274 |
if mode in ("schema", "query_schema"):
|
| 275 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 276 |
permission_scope.update(internal)
|
|
@@ -299,15 +458,13 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
|
|
| 299 |
except HTTPException:
|
| 300 |
raise
|
| 301 |
except Exception as exc: # noqa: BLE001
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
f"scope ({type(exc).__name__}); try again shortly")
|
| 305 |
entry = cache.get(cache_key)
|
| 306 |
if (not isinstance(entry, tuple) or len(entry) < 2
|
| 307 |
or not isinstance(entry[1], (list, tuple))
|
| 308 |
or any(not isinstance(row, dict) for row in entry[1])):
|
| 309 |
-
|
| 310 |
-
"this database has no app-stored snapshot for your current scope")
|
| 311 |
grid = grid_assembly(session, scope=grid_scope, consume_corrections=False, with_rows=True)
|
| 312 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 313 |
permission_scope.update(grid["_permission_scope"])
|
|
@@ -389,12 +546,11 @@ def _assistant_table_defn(session: Session, database: str):
|
|
| 389 |
|
| 390 |
try:
|
| 391 |
definitions = session.runtime.get_projection('user_tables', drop=('rows',))
|
| 392 |
-
except Exception:
|
| 393 |
-
|
| 394 |
-
|
| 395 |
if not isinstance(definitions, dict):
|
| 396 |
-
|
| 397 |
-
"this database has no readable definition snapshot")
|
| 398 |
if database == "ut_activity":
|
| 399 |
definitions = user_tables._with_activity(definitions)
|
| 400 |
definition = definitions.get(database)
|
|
@@ -428,8 +584,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
|
|
| 428 |
# upstream fallback: it is the tenant's materialised document already held by the runtime.
|
| 429 |
table = user_tables.get(database, st=session.runtime)
|
| 430 |
if not isinstance(table, dict):
|
| 431 |
-
|
| 432 |
-
"this database changed before its stored snapshot could be read")
|
| 433 |
if not user_tables.materialises(database, st=session.runtime, defn=table):
|
| 434 |
# ⭐⭐ W47-T23 / R11 - THIS IS NO LONGER THE ANSWER FOR A MIRROR-SERVED DATABASE, and the
|
| 435 |
# seam this comment used to ask for now exists: `assistant_mirror.py`, reached through
|
|
@@ -442,9 +597,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
|
|
| 442 |
# ⛔ DO NOT "FIX" THIS BY FLIPPING `readThrough`. D-378 says these tables' classification
|
| 443 |
# is separately wrong; changing the flag to make a read pass would hide that defect and
|
| 444 |
# move ten grids onto the wrong reader.
|
| 445 |
-
|
| 446 |
-
"this database stopped storing its rows while it was being read; ask "
|
| 447 |
-
"again and it will be read from its connector mirror instead")
|
| 448 |
|
| 449 |
# A table's field contract was authorized from the rows-free definition. If it changed while
|
| 450 |
# the materialised document was fetched, fail rather than projecting a schema that was never
|
|
@@ -452,12 +605,10 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
|
|
| 452 |
actual_fields = [dict(field) for field in (table.get("fields") or [])
|
| 453 |
if isinstance(field, dict) and field.get("key")]
|
| 454 |
if actual_fields != fields:
|
| 455 |
-
|
| 456 |
-
"this database changed before its stored snapshot could be read")
|
| 457 |
raw_rows = table.get("rows") or {}
|
| 458 |
if not isinstance(raw_rows, dict):
|
| 459 |
-
|
| 460 |
-
"this database does not have a readable stored snapshot")
|
| 461 |
keys = {field["key"] for field in fields}
|
| 462 |
rows = []
|
| 463 |
for record_id, raw in raw_rows.items():
|
|
@@ -539,9 +690,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 539 |
# `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
|
| 540 |
declared = assistant_mirror.declared_fields(database, definition)
|
| 541 |
if declared is None:
|
| 542 |
-
|
| 543 |
-
"this database does not store its rows and it is not one of the "
|
| 544 |
-
"connected databases the assistant can read from the mirror")
|
| 545 |
# ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
|
| 546 |
# first sync. Passed only for an Amazon key, which is the one family it changes.
|
| 547 |
cur, refusal = (assistant_mirror.open_cursor(session, database)
|
|
@@ -554,9 +703,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 554 |
assistant_mirror.unavailable_reason(database, definition))
|
| 555 |
fields = assistant_mirror.answerable_fields(database, spec, declared)
|
| 556 |
if not fields:
|
| 557 |
-
|
| 558 |
-
"this database's connector mirror carries none of its declared columns "
|
| 559 |
-
"on this deployment, so there is nothing to answer from")
|
| 560 |
|
| 561 |
grant, visible = _assistant_preflight_request(session, database, fields, requested_fields,
|
| 562 |
filters, require_explicit_grant=False)
|
|
@@ -567,11 +714,17 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 567 |
# Pushing the raw input to SQL instead would mean the pushdown and the Python pass are two
|
| 568 |
# predicates that happen to agree, which is this repo's recorded defect class. Same input to
|
| 569 |
# both compilers, same answer by construction.
|
|
|
|
| 570 |
try:
|
| 571 |
wall_tree = perm_scope.validate_assistant_filter((grant or {}).get("filter"), fields)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 572 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 573 |
except ValueError as exc:
|
| 574 |
-
|
| 575 |
raise # unreachable; keeps the names bound for a reader
|
| 576 |
# ⚠ NO GRANT MEANS NO ROW WALL ON THIS PATH, matching `assistant_read_scope` exactly: it
|
| 577 |
# applies `assistant_apply_row_scope` only when a grant resolved, because `may_open` above is
|
|
@@ -589,10 +742,11 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 589 |
hidden.update(perm_scope.hidden_keys(member, database, fields, st=session.runtime))
|
| 590 |
tree = ((perm_scope.entry(member, database) or {}).get("filter")
|
| 591 |
if not perms.is_admin(member) else None)
|
|
|
|
| 592 |
try:
|
| 593 |
tree = perm_scope.validate_assistant_filter(tree, fields)
|
| 594 |
except ValueError as exc:
|
| 595 |
-
|
| 596 |
if tree is not None:
|
| 597 |
by_key = {field["key"]: field for field in fields}
|
| 598 |
for leaf_key in perm_scope._wall_leaf_keys(tree["nodes"]):
|
|
@@ -602,13 +756,15 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 602 |
verdict = perm_scope.wall_leaf_verdict(
|
| 603 |
database, {"colId": leaf_key}, session.runtime)
|
| 604 |
if not verdict["enforceable"]:
|
| 605 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
|
|
|
| 606 |
trees.append(tree)
|
| 607 |
visible = [field for field in visible if field.get("key") not in hidden]
|
| 608 |
try:
|
| 609 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 610 |
except ValueError as exc:
|
| 611 |
-
|
| 612 |
wall_tree = {"conj": "and", "nodes": [
|
| 613 |
{"conj": tree["conj"], "children": tree["nodes"]} for tree in trees
|
| 614 |
]} if trees else None
|
|
@@ -788,7 +944,7 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member
|
|
| 788 |
try:
|
| 789 |
perm_scope.validate_assistant_filter(filters, schema["fields"])
|
| 790 |
except ValueError as exc:
|
| 791 |
-
|
| 792 |
rows = []
|
| 793 |
if mode not in ("schema", "query_schema"):
|
| 794 |
cursor = None
|
|
@@ -804,16 +960,20 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member
|
|
| 804 |
if cursor in seen:
|
| 805 |
break
|
| 806 |
seen.add(cursor)
|
| 807 |
-
|
| 808 |
-
"This activity history is too large for an Assistant snapshot. Read it in Table by page.")
|
| 809 |
return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
|
| 810 |
|
| 811 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 812 |
def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False):
|
| 813 |
"""Separate one opaque identity equality from visible field operands."""
|
| 814 |
if database not in _ASSISTANT_BUILT_INS:
|
| 815 |
if identity_pid is not None:
|
| 816 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 817 |
return filters, identity_pid
|
| 818 |
if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"}
|
| 819 |
and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list)
|
|
@@ -824,14 +984,17 @@ def _assistant_identity_request(database, filters, identity_pid, *, trusted_filt
|
|
| 824 |
and not set(leaf) - {"id", "colId", "op", "value"}):
|
| 825 |
value = leaf["value"]
|
| 826 |
if value is None:
|
| 827 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 828 |
if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid):
|
| 829 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 830 |
identity_pid, filters = value, None
|
| 831 |
if identity_pid is not None and (type(identity_pid) not in (int, str)
|
| 832 |
or isinstance(identity_pid, str)
|
| 833 |
and (not identity_pid.strip() or len(identity_pid) > 200)):
|
| 834 |
-
_assistant_refuse(400, "assistant_bad_filter",
|
|
|
|
| 835 |
return filters, identity_pid
|
| 836 |
|
| 837 |
|
|
@@ -885,8 +1048,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 885 |
try:
|
| 886 |
access = perm_scope.min_access([session.user, *members], key, st=session.runtime)
|
| 887 |
except (perm_scope.UnknownTable, perm_scope.Unresolvable):
|
| 888 |
-
|
| 889 |
-
"The database field contract could not be read. Try again in a moment.")
|
| 890 |
if not access["readable"]:
|
| 891 |
blocked = access["blockedBy"]
|
| 892 |
names = dash_norm.plain(", ".join(blocked))
|
|
@@ -906,7 +1068,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 906 |
try:
|
| 907 |
perm_scope.validate_assistant_filter(filters, shared_fields)
|
| 908 |
except ValueError as exc:
|
| 909 |
-
|
| 910 |
if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
|
| 911 |
operands = [spec.get("field") for spec in mode[1] if isinstance(spec, dict)]
|
| 912 |
if len(mode) > 2:
|
|
@@ -970,9 +1132,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 970 |
if perm_scope.audience_declared(key, st=session.runtime, defn=_defn):
|
| 971 |
# C16 on a mirror-served table would have to run inside the SQL before counting
|
| 972 |
# and paging; filtering the returned page afterwards gives a wrong total. Refuse.
|
| 973 |
-
|
| 974 |
-
"This database limits each record to its listed recipients, "
|
| 975 |
-
"and that limit cannot be applied to this kind of read yet.")
|
| 976 |
if members:
|
| 977 |
rows, canonical_fields, source_version, label = _assistant_mirror_table(
|
| 978 |
session, key, fields, filters, defn=_defn, lent=_lent, mode=mode,
|
|
@@ -1012,7 +1172,17 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1012 |
try:
|
| 1013 |
perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields)
|
| 1014 |
except ValueError as exc:
|
| 1015 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1016 |
# ⭐ OWNER I16 — the tenant handle rides into the row wall here too, so one stored rule means
|
| 1017 |
# the same thing in the Analyst's answer as it does on the grid (see
|
| 1018 |
# `perm_scope._enrich_for_wall`).
|
|
@@ -1079,7 +1249,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
|
|
| 1079 |
try:
|
| 1080 |
requested_filter = perm_scope.validate_assistant_filter(filters, visible)
|
| 1081 |
except ValueError as exc:
|
| 1082 |
-
|
| 1083 |
if requested_filter is not None:
|
| 1084 |
visible_keys = {field["key"] for field in visible}
|
| 1085 |
evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys]
|
|
@@ -1639,17 +1809,62 @@ def _touch_active(uname):
|
|
| 1639 |
pass
|
| 1640 |
|
| 1641 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1642 |
def require_session(request: Request, response: Response) -> Session:
|
| 1643 |
"""Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is
|
| 1644 |
refreshed on use; the 30-day absolute expiry is carried over, never extended)."""
|
|
|
|
|
|
|
| 1645 |
raw = request.cookies.get(aios_session.COOKIE_NAME)
|
| 1646 |
if not raw:
|
|
|
|
|
|
|
| 1647 |
raise err(401, "no_session", "sign in to continue")
|
| 1648 |
claims = aios_session.read(raw)
|
| 1649 |
if not claims:
|
|
|
|
|
|
|
| 1650 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1651 |
user = _user_for(claims)
|
| 1652 |
if not user:
|
|
|
|
|
|
|
| 1653 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1654 |
identity = {_TENANT_HEADER: str(claims["t"]),
|
| 1655 |
_PRINCIPAL_HEADER: _principal(claims["t"], user["username"])}
|
|
@@ -1665,8 +1880,12 @@ def require_session(request: Request, response: Response) -> Session:
|
|
| 1665 |
except KeyError:
|
| 1666 |
# An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this
|
| 1667 |
# deployment, and confirming which slugs exist would answer a question we were not asked.
|
|
|
|
|
|
|
| 1668 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1669 |
-
|
|
|
|
|
|
|
| 1670 |
absolute_expiry=claims["x"])
|
| 1671 |
aios_session.set_cookie(response, request, fresh)
|
| 1672 |
# ⭐ STAMP THE TENANT THIS ANSWER WAS SERVED FOR (owner report 2026-08-09: "I can see
|
|
@@ -1687,11 +1906,13 @@ def require_session(request: Request, response: Response) -> Session:
|
|
| 1687 |
# `session.tenant` read back from something the request could influence.
|
| 1688 |
response.headers[_TENANT_HEADER] = str(claims["t"])
|
| 1689 |
response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
|
| 1690 |
-
|
| 1691 |
-
|
|
|
|
|
|
|
| 1692 |
# qa-runner has no global account record. Never let the normal activity telemetry create or
|
| 1693 |
# touch one through the default tenant-#0 users store.
|
| 1694 |
-
if (not user.get("qa_runner")
|
| 1695 |
and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
|
| 1696 |
_observe_active(claims["t"], user.get("username"))
|
| 1697 |
if request.method not in {"GET", "HEAD", "OPTIONS"}:
|
|
@@ -1707,6 +1928,8 @@ def require_changes_session(request: Request, response: Response) -> Session | N
|
|
| 1707 |
verified signed cookie is enough to return that no-data response; resolving the account record
|
| 1708 |
would itself turn those legacy requests into a permanent Neon keepalive.
|
| 1709 |
"""
|
|
|
|
|
|
|
| 1710 |
if store.backend() != "pg":
|
| 1711 |
return require_session(request, response)
|
| 1712 |
if request.headers.get("x-aios-change-check") == "focus-v1":
|
|
@@ -1748,6 +1971,9 @@ def require_session_or_bearer(request: Request, response: Response) -> Session:
|
|
| 1748 |
|
| 1749 |
`require_session` stays cookie only. A bearer session never reissues the cookie.
|
| 1750 |
"""
|
|
|
|
|
|
|
|
|
|
| 1751 |
bearer = _bearer_header(request)
|
| 1752 |
if bearer is None or request.cookies.get(aios_session.COOKIE_NAME):
|
| 1753 |
try:
|
|
|
|
| 180 |
raise err(status, code, message)
|
| 181 |
|
| 182 |
|
| 183 |
+
def _assistant_source_causes():
|
| 184 |
+
"""Return each unavailable cause's sentence and whether waiting can make the read succeed."""
|
| 185 |
+
import assistant_mirror
|
| 186 |
+
return {
|
| 187 |
+
"loading": ("The copy of this database is still loading on this server.", True),
|
| 188 |
+
"syncing": ("A sync of this database's source is running.", True),
|
| 189 |
+
"store_silent": ("The store did not answer.", True),
|
| 190 |
+
"changed": ("This database changed while it was being read.", True),
|
| 191 |
+
"not_built": ("The copy of this database for your access could not be built.", False),
|
| 192 |
+
"contract_unread": ("The field list of this database could not be read.", False),
|
| 193 |
+
"no_binding": (assistant_mirror.NO_BINDING, False),
|
| 194 |
+
"no_copy": ("This database stores no rows and has no connector copy to read them from.", False),
|
| 195 |
+
"no_columns": ("The connector copy of this database holds none of its fields on this server.", False),
|
| 196 |
+
"unreadable": ("The stored copy of this database cannot be read.", False),
|
| 197 |
+
"audience": ("This database limits each record to its listed recipients, and that limit "
|
| 198 |
+
"cannot be applied to this kind of read yet.", False),
|
| 199 |
+
"too_large": ("This activity history is too large for an Assistant snapshot. "
|
| 200 |
+
"Read it in Table by page.", False),
|
| 201 |
+
"no_connector": ("This database names no connector, so its completed sync cannot be checked.", False),
|
| 202 |
+
"no_activity_store": ("Activity storage is not set up for this workspace.", False),
|
| 203 |
+
"invalid_publication": ("The completed sync identity of this database is unreadable.", False),
|
| 204 |
+
"restore_blocked": ("The saved copy of this database could not be restored. "
|
| 205 |
+
"Ask an Administrator to repair store access and restart the server.", False),
|
| 206 |
+
"mirror_open": ("The connector copy of this database could not be opened on this server.", False),
|
| 207 |
+
"not_supported": ("Selected record reads are not available on this deployment until it is updated.",
|
| 208 |
+
False),
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _assistant_source_message(cause, verb="read", *, message=None):
|
| 213 |
+
"""The refusal text for one cause: its sentence, then a retry line only where a retry can succeed."""
|
| 214 |
+
if cause in {"amazon_restore", "amazon_sync"} and _assistant_source_cause_of(message) == cause:
|
| 215 |
+
return message
|
| 216 |
+
sentence, retry = _assistant_source_causes()[cause]
|
| 217 |
+
return f"{sentence} Try the same {verb} again shortly." if retry else sentence
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _assistant_source_cause_of(message):
|
| 221 |
+
"""The cause a refusal message names, or None for text outside the closed set."""
|
| 222 |
+
text = " ".join(str(message or "").split())
|
| 223 |
+
for cause, (sentence, _retry) in _assistant_source_causes().items():
|
| 224 |
+
if text.startswith(sentence):
|
| 225 |
+
return cause
|
| 226 |
+
if text.startswith("Amazon "):
|
| 227 |
+
import amazon_relational as amazon
|
| 228 |
+
# Recognize native instructions without another pending-state read.
|
| 229 |
+
for key in amazon.SOURCE:
|
| 230 |
+
for cause, pending in (("amazon_restore", True), ("amazon_sync", False)):
|
| 231 |
+
if text == amazon.missing_source(key, pending=pending)["missing"]:
|
| 232 |
+
return cause
|
| 233 |
+
return None
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _assistant_refuse_source(cause, status=409, code="assistant_snapshot_unavailable"):
|
| 237 |
+
"""Refuse a read whose snapshot or source is unavailable, naming its cause."""
|
| 238 |
+
_assistant_refuse(status, code, _assistant_source_message(cause))
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _assistant_log_cause(where, exc):
|
| 242 |
+
"""Log an exception's class, never its text, beside the cause the caller was told."""
|
| 243 |
+
import logging
|
| 244 |
+
logging.getLogger(__name__).warning("%s failed (%s)", where, type(exc).__name__)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _assistant_display_key(key):
|
| 248 |
+
"""A caller-supplied key or operator as message text: one line, at most 80 characters."""
|
| 249 |
+
return " ".join(str(key).split())[:80]
|
| 250 |
+
|
| 251 |
+
|
| 252 |
def assistant_unknown_field_message(requested, visible_keys):
|
| 253 |
+
"""Name the requested keys that are not readable, then suggest at most five close visible keys.
|
| 254 |
+
|
| 255 |
+
S82-T124: only keys the caller sent are echoed, and suggestions come from `visible_keys` only.
|
| 256 |
+
A hidden key and an unknown key therefore read the same, so the message is no existence oracle.
|
| 257 |
+
"""
|
| 258 |
candidates = sorted({key for key in visible_keys if isinstance(key, str) and key})
|
| 259 |
requested = [requested] if isinstance(requested, str) else requested
|
| 260 |
+
matches, missing = [], []
|
| 261 |
for key in requested if isinstance(requested, (list, tuple)) else ():
|
| 262 |
if isinstance(key, str) and key not in candidates:
|
| 263 |
+
name = _assistant_display_key(key)
|
| 264 |
+
if name and name not in missing:
|
| 265 |
+
missing.append(name)
|
| 266 |
for match in difflib.get_close_matches(key, candidates, n=5, cutoff=0.6):
|
| 267 |
if match not in matches:
|
| 268 |
matches.append(match)
|
| 269 |
+
message = ""
|
| 270 |
+
if missing:
|
| 271 |
+
message = (("No readable field is named " if len(missing) == 1
|
| 272 |
+
else "No readable fields are named ") + ", ".join(missing[:5]) + ". ")
|
| 273 |
+
message += ("Every requested field must be readable in this database. "
|
| 274 |
+
"Use a field key returned by describe_database.")
|
| 275 |
if matches:
|
| 276 |
message += " Close visible field keys: " + ", ".join(matches[:5]) + "."
|
| 277 |
+
return dash_norm.plain(message)
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _assistant_filter_leaves(tree, depth=1):
|
| 281 |
+
"""Every condition of a raw filter tree, in the order the strict validator reads them."""
|
| 282 |
+
nodes = tree if isinstance(tree, list) else (tree.get("nodes") if isinstance(tree, dict) else None)
|
| 283 |
+
out = []
|
| 284 |
+
for node in nodes if isinstance(nodes, list) else ():
|
| 285 |
+
if isinstance(node, dict) and "children" in node:
|
| 286 |
+
if depth < 3:
|
| 287 |
+
out.extend(_assistant_filter_leaves(node.get("children"), depth + 1))
|
| 288 |
+
else:
|
| 289 |
+
out.append(node)
|
| 290 |
+
return out
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def _assistant_rejected_condition(tree, fields):
|
| 294 |
+
"""The first condition the strict validator refuses on its own, or None for a shape fault."""
|
| 295 |
+
import core.perm_scope as perm_scope
|
| 296 |
+
for leaf in _assistant_filter_leaves(tree):
|
| 297 |
+
try:
|
| 298 |
+
perm_scope.validate_assistant_filter([leaf], fields)
|
| 299 |
+
except ValueError:
|
| 300 |
+
return leaf
|
| 301 |
+
except Exception: # noqa: BLE001 - a leaf the tree never reached
|
| 302 |
+
return None
|
| 303 |
+
return None
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def _assistant_bad_filter_message(reason, condition, *, visible_keys=(), wall=False):
|
| 307 |
+
"""The `assistant_bad_filter` text: the validator's reason plus the rejected field and operator.
|
| 308 |
+
|
| 309 |
+
S82-T124: every refusal names its field. A wall's field is named only when the caller may read
|
| 310 |
+
it, so the restriction an administrator set never names a hidden field.
|
| 311 |
+
"""
|
| 312 |
+
text = " ".join(str(reason or "").split()).rstrip(".") or "assistant filter cannot be read"
|
| 313 |
+
where = " in the restriction an administrator set on this database" if wall else ""
|
| 314 |
+
field = condition.get("colId") if isinstance(condition, dict) else None
|
| 315 |
+
name = _assistant_display_key(field) if isinstance(field, str) else ""
|
| 316 |
+
if condition is None:
|
| 317 |
+
text += ". The filter's structure is at fault, not one field"
|
| 318 |
+
elif not name:
|
| 319 |
+
text += f". The rejected condition{where} names no field"
|
| 320 |
+
elif wall and field not in set(visible_keys or ()):
|
| 321 |
+
text += f". The rejected condition{where} is on a field you cannot read"
|
| 322 |
+
else:
|
| 323 |
+
text += f'. The rejected condition{where} is on field "{name}"'
|
| 324 |
+
if "op" in condition:
|
| 325 |
+
op = condition.get("op")
|
| 326 |
+
text += (f' with operator "{_assistant_display_key(op)}"'
|
| 327 |
+
if isinstance(op, str) and op.strip() else " with no operator")
|
| 328 |
+
return dash_norm.plain(text + ".")
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def _assistant_refuse_filter(reason, tree, fields, *, visible_keys=None, wall=False):
|
| 332 |
+
"""Refuse a filter the strict validator rejected, naming the condition it rejected.
|
| 333 |
+
|
| 334 |
+
A caller's condition on a key outside its readable `fields` answers `assistant_unknown_field`,
|
| 335 |
+
as a field selection does, so hidden and unknown keys read alike. `pid` is the record identity
|
| 336 |
+
no filter may name, and a wall is the administrator's: both stay `assistant_bad_filter`.
|
| 337 |
+
"""
|
| 338 |
+
keys = {str(field.get("key")) for field in fields or ()
|
| 339 |
+
if isinstance(field, dict) and field.get("key")}
|
| 340 |
+
visible = keys if visible_keys is None else set(visible_keys)
|
| 341 |
+
condition = _assistant_rejected_condition(tree, fields)
|
| 342 |
+
if not wall and isinstance(condition, dict):
|
| 343 |
+
rhs = condition.get("rhs")
|
| 344 |
+
for key in (condition.get("colId"), rhs.get("colId") if isinstance(rhs, dict) else None):
|
| 345 |
+
if isinstance(key, str) and key.strip() and key not in keys and key != "pid":
|
| 346 |
+
_assistant_refuse(400, "assistant_unknown_field",
|
| 347 |
+
assistant_unknown_field_message(key, visible))
|
| 348 |
+
_assistant_refuse(400, "assistant_bad_filter",
|
| 349 |
+
_assistant_bad_filter_message(reason, condition, visible_keys=visible, wall=wall))
|
| 350 |
|
| 351 |
|
| 352 |
def _assistant_tenant_allows(session: Session, database: str) -> bool:
|
|
|
|
| 369 |
"your current permissions do not allow this database")
|
| 370 |
visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, database)
|
| 371 |
if grant is not None else list(canonical_fields or ()))
|
| 372 |
+
wall_fields = canonical_fields if permission_fields is None else permission_fields
|
| 373 |
+
visible_keys = [field.get("key") for field in visible if isinstance(field, dict)]
|
| 374 |
+
try:
|
| 375 |
+
perm_scope.validate_assistant_filter((grant or {}).get("filter"), wall_fields)
|
| 376 |
+
except ValueError as exc:
|
| 377 |
+
_assistant_refuse_filter(exc, (grant or {}).get("filter"), wall_fields,
|
| 378 |
+
visible_keys=visible_keys, wall=True)
|
| 379 |
+
_assistant_requested_fields(requested_fields, visible)
|
| 380 |
try:
|
|
|
|
|
|
|
|
|
|
| 381 |
perm_scope.validate_assistant_filter(filters, visible)
|
| 382 |
except ValueError as exc:
|
| 383 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 384 |
return grant, visible
|
| 385 |
|
| 386 |
|
|
|
|
| 420 |
if field["key"] not in have)]
|
| 421 |
fields, _ = schema_overlay.for_grid(database, fields, st=session.runtime, contract=fields)
|
| 422 |
internal = grid.get("_permission_scope") or {"fields": fields, "rows": [], "row_fields": fields}
|
| 423 |
+
_grant, visible = _assistant_preflight_request(
|
| 424 |
+
session, database, fields, requested_fields, filters,
|
| 425 |
+
require_explicit_grant=True, permission_fields=internal["fields"])
|
| 426 |
for member in members:
|
| 427 |
+
wall = (perm_scope.assistant_entry(member, database) or {}).get("filter")
|
| 428 |
try:
|
| 429 |
+
perm_scope.validate_assistant_filter(wall, internal["fields"])
|
|
|
|
| 430 |
except ValueError as exc:
|
| 431 |
+
_assistant_refuse_filter(exc, wall, internal["fields"], wall=True, visible_keys=[
|
| 432 |
+
field.get("key") for field in visible if isinstance(field, dict)])
|
| 433 |
if mode in ("schema", "query_schema"):
|
| 434 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 435 |
permission_scope.update(internal)
|
|
|
|
| 458 |
except HTTPException:
|
| 459 |
raise
|
| 460 |
except Exception as exc: # noqa: BLE001
|
| 461 |
+
_assistant_log_cause("assistant pool build", exc)
|
| 462 |
+
_assistant_refuse_source("not_built")
|
|
|
|
| 463 |
entry = cache.get(cache_key)
|
| 464 |
if (not isinstance(entry, tuple) or len(entry) < 2
|
| 465 |
or not isinstance(entry[1], (list, tuple))
|
| 466 |
or any(not isinstance(row, dict) for row in entry[1])):
|
| 467 |
+
_assistant_refuse_source("not_built")
|
|
|
|
| 468 |
grid = grid_assembly(session, scope=grid_scope, consume_corrections=False, with_rows=True)
|
| 469 |
if permission_scope is not None and "_permission_scope" in grid:
|
| 470 |
permission_scope.update(grid["_permission_scope"])
|
|
|
|
| 546 |
|
| 547 |
try:
|
| 548 |
definitions = session.runtime.get_projection('user_tables', drop=('rows',))
|
| 549 |
+
except Exception as exc:
|
| 550 |
+
_assistant_log_cause("assistant definition read", exc)
|
| 551 |
+
_assistant_refuse_source("store_silent")
|
| 552 |
if not isinstance(definitions, dict):
|
| 553 |
+
_assistant_refuse_source("unreadable")
|
|
|
|
| 554 |
if database == "ut_activity":
|
| 555 |
definitions = user_tables._with_activity(definitions)
|
| 556 |
definition = definitions.get(database)
|
|
|
|
| 584 |
# upstream fallback: it is the tenant's materialised document already held by the runtime.
|
| 585 |
table = user_tables.get(database, st=session.runtime)
|
| 586 |
if not isinstance(table, dict):
|
| 587 |
+
_assistant_refuse_source("changed")
|
|
|
|
| 588 |
if not user_tables.materialises(database, st=session.runtime, defn=table):
|
| 589 |
# ⭐⭐ W47-T23 / R11 - THIS IS NO LONGER THE ANSWER FOR A MIRROR-SERVED DATABASE, and the
|
| 590 |
# seam this comment used to ask for now exists: `assistant_mirror.py`, reached through
|
|
|
|
| 597 |
# ⛔ DO NOT "FIX" THIS BY FLIPPING `readThrough`. D-378 says these tables' classification
|
| 598 |
# is separately wrong; changing the flag to make a read pass would hide that defect and
|
| 599 |
# move ten grids onto the wrong reader.
|
| 600 |
+
_assistant_refuse_source("changed")
|
|
|
|
|
|
|
| 601 |
|
| 602 |
# A table's field contract was authorized from the rows-free definition. If it changed while
|
| 603 |
# the materialised document was fetched, fail rather than projecting a schema that was never
|
|
|
|
| 605 |
actual_fields = [dict(field) for field in (table.get("fields") or [])
|
| 606 |
if isinstance(field, dict) and field.get("key")]
|
| 607 |
if actual_fields != fields:
|
| 608 |
+
_assistant_refuse_source("changed")
|
|
|
|
| 609 |
raw_rows = table.get("rows") or {}
|
| 610 |
if not isinstance(raw_rows, dict):
|
| 611 |
+
_assistant_refuse_source("unreadable")
|
|
|
|
| 612 |
keys = {field["key"] for field in fields}
|
| 613 |
rows = []
|
| 614 |
for record_id, raw in raw_rows.items():
|
|
|
|
| 690 |
# `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
|
| 691 |
declared = assistant_mirror.declared_fields(database, definition)
|
| 692 |
if declared is None:
|
| 693 |
+
_assistant_refuse_source("no_copy")
|
|
|
|
|
|
|
| 694 |
# ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
|
| 695 |
# first sync. Passed only for an Amazon key, which is the one family it changes.
|
| 696 |
cur, refusal = (assistant_mirror.open_cursor(session, database)
|
|
|
|
| 703 |
assistant_mirror.unavailable_reason(database, definition))
|
| 704 |
fields = assistant_mirror.answerable_fields(database, spec, declared)
|
| 705 |
if not fields:
|
| 706 |
+
_assistant_refuse_source("no_columns")
|
|
|
|
|
|
|
| 707 |
|
| 708 |
grant, visible = _assistant_preflight_request(session, database, fields, requested_fields,
|
| 709 |
filters, require_explicit_grant=False)
|
|
|
|
| 714 |
# Pushing the raw input to SQL instead would mean the pushdown and the Python pass are two
|
| 715 |
# predicates that happen to agree, which is this repo's recorded defect class. Same input to
|
| 716 |
# both compilers, same answer by construction.
|
| 717 |
+
visible_keys = [field.get("key") for field in visible if isinstance(field, dict)]
|
| 718 |
try:
|
| 719 |
wall_tree = perm_scope.validate_assistant_filter((grant or {}).get("filter"), fields)
|
| 720 |
+
except ValueError as exc:
|
| 721 |
+
_assistant_refuse_filter(exc, (grant or {}).get("filter"), fields,
|
| 722 |
+
visible_keys=visible_keys, wall=True)
|
| 723 |
+
raise # unreachable; keeps the names bound for a reader
|
| 724 |
+
try:
|
| 725 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 726 |
except ValueError as exc:
|
| 727 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 728 |
raise # unreachable; keeps the names bound for a reader
|
| 729 |
# ⚠ NO GRANT MEANS NO ROW WALL ON THIS PATH, matching `assistant_read_scope` exactly: it
|
| 730 |
# applies `assistant_apply_row_scope` only when a grant resolved, because `may_open` above is
|
|
|
|
| 742 |
hidden.update(perm_scope.hidden_keys(member, database, fields, st=session.runtime))
|
| 743 |
tree = ((perm_scope.entry(member, database) or {}).get("filter")
|
| 744 |
if not perms.is_admin(member) else None)
|
| 745 |
+
raw_wall = tree
|
| 746 |
try:
|
| 747 |
tree = perm_scope.validate_assistant_filter(tree, fields)
|
| 748 |
except ValueError as exc:
|
| 749 |
+
_assistant_refuse_filter(exc, raw_wall, fields, visible_keys=visible_keys, wall=True)
|
| 750 |
if tree is not None:
|
| 751 |
by_key = {field["key"]: field for field in fields}
|
| 752 |
for leaf_key in perm_scope._wall_leaf_keys(tree["nodes"]):
|
|
|
|
| 756 |
verdict = perm_scope.wall_leaf_verdict(
|
| 757 |
database, {"colId": leaf_key}, session.runtime)
|
| 758 |
if not verdict["enforceable"]:
|
| 759 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 760 |
+
verdict["reason"], {"colId": leaf_key},
|
| 761 |
+
visible_keys=visible_keys, wall=True))
|
| 762 |
trees.append(tree)
|
| 763 |
visible = [field for field in visible if field.get("key") not in hidden]
|
| 764 |
try:
|
| 765 |
filter_tree = perm_scope.validate_assistant_filter(filters, visible)
|
| 766 |
except ValueError as exc:
|
| 767 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 768 |
wall_tree = {"conj": "and", "nodes": [
|
| 769 |
{"conj": tree["conj"], "children": tree["nodes"]} for tree in trees
|
| 770 |
]} if trees else None
|
|
|
|
| 944 |
try:
|
| 945 |
perm_scope.validate_assistant_filter(filters, schema["fields"])
|
| 946 |
except ValueError as exc:
|
| 947 |
+
_assistant_refuse_filter(exc, filters, schema["fields"])
|
| 948 |
rows = []
|
| 949 |
if mode not in ("schema", "query_schema"):
|
| 950 |
cursor = None
|
|
|
|
| 960 |
if cursor in seen:
|
| 961 |
break
|
| 962 |
seen.add(cursor)
|
| 963 |
+
_assistant_refuse_source("too_large")
|
|
|
|
| 964 |
return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
|
| 965 |
|
| 966 |
|
| 967 |
+
#: The one condition an opaque identity read carries; its refusals name this field and operator.
|
| 968 |
+
_IDENTITY_CONDITION = {"colId": "pid", "op": "eq"}
|
| 969 |
+
|
| 970 |
+
|
| 971 |
def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False):
|
| 972 |
"""Separate one opaque identity equality from visible field operands."""
|
| 973 |
if database not in _ASSISTANT_BUILT_INS:
|
| 974 |
if identity_pid is not None:
|
| 975 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 976 |
+
"Opaque identity reads require a built-in database", _IDENTITY_CONDITION))
|
| 977 |
return filters, identity_pid
|
| 978 |
if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"}
|
| 979 |
and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list)
|
|
|
|
| 984 |
and not set(leaf) - {"id", "colId", "op", "value"}):
|
| 985 |
value = leaf["value"]
|
| 986 |
if value is None:
|
| 987 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 988 |
+
"Use one non-null opaque record identity", _IDENTITY_CONDITION))
|
| 989 |
if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid):
|
| 990 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 991 |
+
"Opaque identity pins must agree", _IDENTITY_CONDITION))
|
| 992 |
identity_pid, filters = value, None
|
| 993 |
if identity_pid is not None and (type(identity_pid) not in (int, str)
|
| 994 |
or isinstance(identity_pid, str)
|
| 995 |
and (not identity_pid.strip() or len(identity_pid) > 200)):
|
| 996 |
+
_assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
|
| 997 |
+
"Use one opaque integer or string record identity", _IDENTITY_CONDITION))
|
| 998 |
return filters, identity_pid
|
| 999 |
|
| 1000 |
|
|
|
|
| 1048 |
try:
|
| 1049 |
access = perm_scope.min_access([session.user, *members], key, st=session.runtime)
|
| 1050 |
except (perm_scope.UnknownTable, perm_scope.Unresolvable):
|
| 1051 |
+
_assistant_refuse_source("contract_unread")
|
|
|
|
| 1052 |
if not access["readable"]:
|
| 1053 |
blocked = access["blockedBy"]
|
| 1054 |
names = dash_norm.plain(", ".join(blocked))
|
|
|
|
| 1068 |
try:
|
| 1069 |
perm_scope.validate_assistant_filter(filters, shared_fields)
|
| 1070 |
except ValueError as exc:
|
| 1071 |
+
_assistant_refuse_filter(exc, filters, shared_fields)
|
| 1072 |
if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
|
| 1073 |
operands = [spec.get("field") for spec in mode[1] if isinstance(spec, dict)]
|
| 1074 |
if len(mode) > 2:
|
|
|
|
| 1132 |
if perm_scope.audience_declared(key, st=session.runtime, defn=_defn):
|
| 1133 |
# C16 on a mirror-served table would have to run inside the SQL before counting
|
| 1134 |
# and paging; filtering the returned page afterwards gives a wrong total. Refuse.
|
| 1135 |
+
_assistant_refuse_source("audience")
|
|
|
|
|
|
|
| 1136 |
if members:
|
| 1137 |
rows, canonical_fields, source_version, label = _assistant_mirror_table(
|
| 1138 |
session, key, fields, filters, defn=_defn, lent=_lent, mode=mode,
|
|
|
|
| 1172 |
try:
|
| 1173 |
perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields)
|
| 1174 |
except ValueError as exc:
|
| 1175 |
+
# Only on the refusal path: the caller's readable keys decide whether the wall's field
|
| 1176 |
+
# is named. An unreadable list names no field, so the refusal stays a 400.
|
| 1177 |
+
try:
|
| 1178 |
+
withheld = set(perm_scope.hidden_keys(session.user, key, canonical_fields,
|
| 1179 |
+
st=session.runtime))
|
| 1180 |
+
readable = [field.get("key") for field in perm_scope.assistant_visible_fields(
|
| 1181 |
+
canonical_fields, session.user, key) if field.get("key") not in withheld]
|
| 1182 |
+
except Exception: # noqa: BLE001 - the refusal must still answer
|
| 1183 |
+
readable = []
|
| 1184 |
+
_assistant_refuse_filter(exc, (grant or {}).get("filter"), canonical_fields,
|
| 1185 |
+
visible_keys=readable, wall=True)
|
| 1186 |
# ⭐ OWNER I16 — the tenant handle rides into the row wall here too, so one stored rule means
|
| 1187 |
# the same thing in the Analyst's answer as it does on the grid (see
|
| 1188 |
# `perm_scope._enrich_for_wall`).
|
|
|
|
| 1249 |
try:
|
| 1250 |
requested_filter = perm_scope.validate_assistant_filter(filters, visible)
|
| 1251 |
except ValueError as exc:
|
| 1252 |
+
_assistant_refuse_filter(exc, filters, visible)
|
| 1253 |
if requested_filter is not None:
|
| 1254 |
visible_keys = {field["key"] for field in visible}
|
| 1255 |
evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys]
|
|
|
|
| 1809 |
pass
|
| 1810 |
|
| 1811 |
|
| 1812 |
+
def refuse_view_as_write(request: Request):
|
| 1813 |
+
"""Refuse every view-as method except GET and HEAD before any write."""
|
| 1814 |
+
if "X-View-As" in request.headers and request.method not in {"GET", "HEAD"}:
|
| 1815 |
+
raise err(403, "view_as_read_only", "Viewing as a member is read-only.")
|
| 1816 |
+
|
| 1817 |
+
|
| 1818 |
+
def _view_as_refused():
|
| 1819 |
+
return err(403, "view_as_refused", "This member view is unavailable.")
|
| 1820 |
+
|
| 1821 |
+
|
| 1822 |
+
def _view_as_user(request, actor, tenant, rt):
|
| 1823 |
+
"""Resolve an active member of the actor's tenant and audit this request once."""
|
| 1824 |
+
if "X-View-As" not in request.headers:
|
| 1825 |
+
return actor
|
| 1826 |
+
if not perms.is_admin(actor):
|
| 1827 |
+
raise _view_as_refused()
|
| 1828 |
+
started_at = datetime.now(timezone.utc).isoformat()
|
| 1829 |
+
target = request.headers["X-View-As"].strip().lower()
|
| 1830 |
+
if not target:
|
| 1831 |
+
raise _view_as_refused()
|
| 1832 |
+
rec = copy.deepcopy(_session_registry(target).get(target))
|
| 1833 |
+
if (not rec or not rec.get("active", True) or rec.get("role", "user") != "user"
|
| 1834 |
+
or str(rec.get("tenant") or "").strip().lower() != tenant):
|
| 1835 |
+
raise _view_as_refused()
|
| 1836 |
+
member = users._public(target, rec)
|
| 1837 |
+
audit_key = (tenant, actor["username"], target)
|
| 1838 |
+
if getattr(request.state, "_view_as_audited", None) != audit_key:
|
| 1839 |
+
from core import activity_log
|
| 1840 |
+
activity_log.append(
|
| 1841 |
+
rt, kind="view_as", object={"app": "settings", "id": target},
|
| 1842 |
+
by=actor["username"], on_behalf_of=target,
|
| 1843 |
+
changes={"member": {"before": None, "after": target},
|
| 1844 |
+
"started_at": {"before": None, "after": started_at}})
|
| 1845 |
+
request.state._view_as_audited = audit_key
|
| 1846 |
+
return member
|
| 1847 |
+
|
| 1848 |
+
|
| 1849 |
def require_session(request: Request, response: Response) -> Session:
|
| 1850 |
"""Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is
|
| 1851 |
refreshed on use; the 30-day absolute expiry is carried over, never extended)."""
|
| 1852 |
+
refuse_view_as_write(request)
|
| 1853 |
+
viewing_as = "X-View-As" in request.headers
|
| 1854 |
raw = request.cookies.get(aios_session.COOKIE_NAME)
|
| 1855 |
if not raw:
|
| 1856 |
+
if viewing_as:
|
| 1857 |
+
raise _view_as_refused()
|
| 1858 |
raise err(401, "no_session", "sign in to continue")
|
| 1859 |
claims = aios_session.read(raw)
|
| 1860 |
if not claims:
|
| 1861 |
+
if viewing_as:
|
| 1862 |
+
raise _view_as_refused()
|
| 1863 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1864 |
user = _user_for(claims)
|
| 1865 |
if not user:
|
| 1866 |
+
if viewing_as:
|
| 1867 |
+
raise _view_as_refused()
|
| 1868 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1869 |
identity = {_TENANT_HEADER: str(claims["t"]),
|
| 1870 |
_PRINCIPAL_HEADER: _principal(claims["t"], user["username"])}
|
|
|
|
| 1880 |
except KeyError:
|
| 1881 |
# An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this
|
| 1882 |
# deployment, and confirming which slugs exist would answer a question we were not asked.
|
| 1883 |
+
if viewing_as:
|
| 1884 |
+
raise _view_as_refused()
|
| 1885 |
raise err(401, "invalid_session", "Your session ended. Sign in again.")
|
| 1886 |
+
actor = user
|
| 1887 |
+
user = _view_as_user(request, actor, claims["t"], rt)
|
| 1888 |
+
fresh, _ = aios_session.mint(claims["t"], actor["username"], int(actor.get("epoch") or 0),
|
| 1889 |
absolute_expiry=claims["x"])
|
| 1890 |
aios_session.set_cookie(response, request, fresh)
|
| 1891 |
# ⭐ STAMP THE TENANT THIS ANSWER WAS SERVED FOR (owner report 2026-08-09: "I can see
|
|
|
|
| 1906 |
# `session.tenant` read back from something the request could influence.
|
| 1907 |
response.headers[_TENANT_HEADER] = str(claims["t"])
|
| 1908 |
response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
|
| 1909 |
+
if viewing_as:
|
| 1910 |
+
response.headers["X-Viewing-As"] = user["username"]
|
| 1911 |
+
# Ordinary reads only observe activity. View-as reads write their audit without
|
| 1912 |
+
# changing either person's activity stamps.
|
| 1913 |
# qa-runner has no global account record. Never let the normal activity telemetry create or
|
| 1914 |
# touch one through the default tenant-#0 users store.
|
| 1915 |
+
if (not viewing_as and not user.get("qa_runner")
|
| 1916 |
and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
|
| 1917 |
_observe_active(claims["t"], user.get("username"))
|
| 1918 |
if request.method not in {"GET", "HEAD", "OPTIONS"}:
|
|
|
|
| 1928 |
verified signed cookie is enough to return that no-data response; resolving the account record
|
| 1929 |
would itself turn those legacy requests into a permanent Neon keepalive.
|
| 1930 |
"""
|
| 1931 |
+
if "X-View-As" in request.headers:
|
| 1932 |
+
return require_session(request, response)
|
| 1933 |
if store.backend() != "pg":
|
| 1934 |
return require_session(request, response)
|
| 1935 |
if request.headers.get("x-aios-change-check") == "focus-v1":
|
|
|
|
| 1971 |
|
| 1972 |
`require_session` stays cookie only. A bearer session never reissues the cookie.
|
| 1973 |
"""
|
| 1974 |
+
refuse_view_as_write(request)
|
| 1975 |
+
if "X-View-As" in request.headers:
|
| 1976 |
+
return _cookie_session(request, response)
|
| 1977 |
bearer = _bearer_header(request)
|
| 1978 |
if bearer is None or request.cookies.get(aios_session.COOKIE_NAME):
|
| 1979 |
try:
|
api/diag_aging.py
CHANGED
|
@@ -26,22 +26,17 @@ def validate(session, rows, restrictions, *, today, documents=None):
|
|
| 26 |
scoped = ScopedOdoo(O.get_odoo(), restrictions)
|
| 27 |
helpers = scoped.helpers()
|
| 28 |
module = bind_functions(ar, {"O": helpers})
|
| 29 |
-
docs = module._open_docs(today)
|
|
|
|
| 30 |
exposures = {}
|
| 31 |
for doc in docs:
|
| 32 |
pid = helpers.m2o_id(doc["partner_id"])
|
| 33 |
-
row = exposures.setdefault(pid, {"open": 0
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
row["open"] += doc["open"]
|
| 40 |
-
# The unrelated settlement check remains false and is not part of this diagnostic.
|
| 41 |
-
existing = module.validate(today, exposure={"_all_rows": list(exposures.values())},
|
| 42 |
-
behavior={"violations": 0, "n_customers": 0})[0]
|
| 43 |
-
existing = {"check": "AR aging buckets equal open documents",
|
| 44 |
-
**{key: existing[key] for key in ("a", "b", "ok")}}
|
| 45 |
canonical = module.aging(today)
|
| 46 |
checks = [existing, {"check": "AR aging matches module buckets", "a": len(rows),
|
| 47 |
"b": len(canonical), "ok": rows == [
|
|
|
|
| 26 |
scoped = ScopedOdoo(O.get_odoo(), restrictions)
|
| 27 |
helpers = scoped.helpers()
|
| 28 |
module = bind_functions(ar, {"O": helpers})
|
| 29 |
+
docs = list(module._open_docs(today))
|
| 30 |
+
# Tie in place: ar.validate reads account.move.line, which the scoped client refuses.
|
| 31 |
exposures = {}
|
| 32 |
for doc in docs:
|
| 33 |
pid = helpers.m2o_id(doc["partner_id"])
|
| 34 |
+
row = exposures.setdefault(pid, {"open": Decimal(0), "overdue": Decimal(0)})
|
| 35 |
+
row["overdue" if doc["days_overdue"] > 5 else "open"] += amount(doc["open"])
|
| 36 |
+
exposed = sum((row["open"] + row["overdue"] for row in exposures.values()), Decimal(0))
|
| 37 |
+
population = sum((amount(doc["open"]) for doc in docs), Decimal(0))
|
| 38 |
+
existing = {"check": "AR aging buckets equal open documents", "a": float(exposed),
|
| 39 |
+
"b": float(population), "ok": abs(exposed - population) < Decimal("0.01")}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
canonical = module.aging(today)
|
| 41 |
checks = [existing, {"check": "AR aging matches module buckets", "a": len(rows),
|
| 42 |
"b": len(canonical), "ok": rows == [
|
api/external_freshness.py
CHANGED
|
@@ -82,8 +82,18 @@ def _amazon_raw(table_key, source=None):
|
|
| 82 |
return None
|
| 83 |
|
| 84 |
|
| 85 |
-
def amazon_sync_state(cur):
|
| 86 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
return {str(entity): at for entity, at in cur.execute(
|
| 88 |
"SELECT entity, updated_at FROM _sync_state "
|
| 89 |
"WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
|
|
@@ -138,7 +148,7 @@ def _amazon_refreshes(session, keys):
|
|
| 138 |
cur = None
|
| 139 |
continue
|
| 140 |
if state is None:
|
| 141 |
-
state = amazon_sync_state(cur)
|
| 142 |
refreshed[key] = amazon_sync_stamp(cur, key, state=state)
|
| 143 |
except Exception:
|
| 144 |
continue
|
|
@@ -167,7 +177,7 @@ def amazon_nav(session, pages):
|
|
| 167 |
cur, refusal = assistant_mirror.open_cursor(session, rows[0]["key"])
|
| 168 |
if refusal is not None or cur is None:
|
| 169 |
return pages
|
| 170 |
-
state = amazon_sync_state(cur)
|
| 171 |
if not state:
|
| 172 |
# An unrestored or absent mirror holds no receipt at all: unknown, not never synced.
|
| 173 |
return pages
|
|
@@ -247,14 +257,17 @@ def snapshot(session, databases):
|
|
| 247 |
edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
|
| 248 |
edits = {}
|
| 249 |
if edited:
|
|
|
|
|
|
|
|
|
|
| 250 |
try:
|
| 251 |
-
|
|
|
|
| 252 |
except Exception:
|
| 253 |
-
|
| 254 |
-
for
|
| 255 |
-
if
|
| 256 |
-
key =
|
| 257 |
-
edits[key] = _latest(edits.get(key), event.get("at"))
|
| 258 |
receipts = {}
|
| 259 |
if connected:
|
| 260 |
try:
|
|
|
|
| 82 |
return None
|
| 83 |
|
| 84 |
|
| 85 |
+
def amazon_sync_state(cur, rt=None):
|
| 86 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
| 87 |
+
if rt is not None:
|
| 88 |
+
from core import connector_sync, connector_spec
|
| 89 |
+
entry = connector_sync.status(rt, "amazon_sp", fresh=True)
|
| 90 |
+
if entry is not None:
|
| 91 |
+
last_pass = entry.get("pass") or {}
|
| 92 |
+
held = last_pass.get("held") or {}
|
| 93 |
+
stamps = last_pass.get("tableSync") or {}
|
| 94 |
+
tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
|
| 95 |
+
return {"amzn_sync_mark_" + name: stamp for name in tables
|
| 96 |
+
if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
|
| 97 |
return {str(entity): at for entity, at in cur.execute(
|
| 98 |
"SELECT entity, updated_at FROM _sync_state "
|
| 99 |
"WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
|
|
|
|
| 148 |
cur = None
|
| 149 |
continue
|
| 150 |
if state is None:
|
| 151 |
+
state = amazon_sync_state(cur, session.runtime)
|
| 152 |
refreshed[key] = amazon_sync_stamp(cur, key, state=state)
|
| 153 |
except Exception:
|
| 154 |
continue
|
|
|
|
| 177 |
cur, refusal = assistant_mirror.open_cursor(session, rows[0]["key"])
|
| 178 |
if refusal is not None or cur is None:
|
| 179 |
return pages
|
| 180 |
+
state = amazon_sync_state(cur, session.runtime)
|
| 181 |
if not state:
|
| 182 |
# An unrestored or absent mirror holds no receipt at all: unknown, not never synced.
|
| 183 |
return pages
|
|
|
|
| 257 |
edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
|
| 258 |
edits = {}
|
| 259 |
if edited:
|
| 260 |
+
# D-11777: edits append only to the history rows, so one SQL aggregate gives each table's
|
| 261 |
+
# newest edit. A store with no history backend (TypeError) or an unreachable one keeps the
|
| 262 |
+
# definition's `updated` stamp below and never fails discovery.
|
| 263 |
try:
|
| 264 |
+
from core import history_rows
|
| 265 |
+
latest = history_rows.backend(rt).latest(sorted(edited))
|
| 266 |
except Exception:
|
| 267 |
+
latest = {}
|
| 268 |
+
for key, at in latest.items() if isinstance(latest, dict) else ():
|
| 269 |
+
if key in edited:
|
| 270 |
+
edits[key] = _latest(edits.get(key), at)
|
|
|
|
| 271 |
receipts = {}
|
| 272 |
if connected:
|
| 273 |
try:
|
api/loopy_catalog_tools.py
CHANGED
|
@@ -305,8 +305,10 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
|
|
| 305 |
import contextvars
|
| 306 |
import sys
|
| 307 |
import threading
|
|
|
|
| 308 |
import catalog_store
|
| 309 |
import app_templates
|
|
|
|
| 310 |
|
| 311 |
render = render or sys.modules[__name__].catalog_pdf
|
| 312 |
budget = export_budget() if budget is None else budget
|
|
@@ -329,8 +331,15 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
|
|
| 329 |
import logging
|
| 330 |
logging.getLogger(__name__).warning("Catalog export result not saved (%s)", type(exc).__name__)
|
| 331 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 332 |
context = contextvars.copy_context()
|
| 333 |
-
worker = threading.Thread(target=context.run, args=(
|
| 334 |
try:
|
| 335 |
worker.start()
|
| 336 |
except Exception:
|
|
|
|
| 305 |
import contextvars
|
| 306 |
import sys
|
| 307 |
import threading
|
| 308 |
+
import time
|
| 309 |
import catalog_store
|
| 310 |
import app_templates
|
| 311 |
+
from core import request_timing
|
| 312 |
|
| 313 |
render = render or sys.modules[__name__].catalog_pdf
|
| 314 |
budget = export_budget() if budget is None else budget
|
|
|
|
| 331 |
import logging
|
| 332 |
logging.getLogger(__name__).warning("Catalog export result not saved (%s)", type(exc).__name__)
|
| 333 |
|
| 334 |
+
def run():
|
| 335 |
+
# HTTP completion cancels its timing state, so the export needs its own bounded scope.
|
| 336 |
+
with request_timing.request_scope() as measured:
|
| 337 |
+
measured.deadline = time.monotonic() + catalog_store.EXPORT_STALE_SECONDS
|
| 338 |
+
with request_timing.thread_scope():
|
| 339 |
+
work()
|
| 340 |
+
|
| 341 |
context = contextvars.copy_context()
|
| 342 |
+
worker = threading.Thread(target=context.run, args=(run,), daemon=True, name="catalog-pdf")
|
| 343 |
try:
|
| 344 |
worker.start()
|
| 345 |
except Exception:
|
api/loopy_file_tools.py
CHANGED
|
@@ -55,7 +55,9 @@ def read_all(session, database, *, fields=None, filters=None, sorts=None, max_ro
|
|
| 55 |
or type(page.get("returned")) is not int or page["returned"] != len(rows)):
|
| 56 |
raise ValueError("The database did not return a complete readable page.")
|
| 57 |
page_keys = [field.get("key") for field in result.get("fields", []) if isinstance(field, dict)]
|
| 58 |
-
|
|
|
|
|
|
|
| 59 |
raise err(409, "query_cursor_stale", "The readable fields changed. Ask again to read the complete list.")
|
| 60 |
if "offset" in page and (type(page["offset"]) is not int or page["offset"] != len(records)):
|
| 61 |
raise ValueError("The database skipped rows. Read the complete result again. No file was stored.")
|
|
|
|
| 55 |
or type(page.get("returned")) is not int or page["returned"] != len(rows)):
|
| 56 |
raise ValueError("The database did not return a complete readable page.")
|
| 57 |
page_keys = [field.get("key") for field in result.get("fields", []) if isinstance(field, dict)]
|
| 58 |
+
# Field set and count, not order: query pages and the schema list fields in different orders (D-8381).
|
| 59 |
+
wanted = [key for key in by_key if key in selected]
|
| 60 |
+
if len(page_keys) != len(wanted) or set(page_keys) != set(wanted):
|
| 61 |
raise err(409, "query_cursor_stale", "The readable fields changed. Ask again to read the complete list.")
|
| 62 |
if "offset" in page and (type(page["offset"]) is not int or page["offset"] != len(records)):
|
| 63 |
raise ValueError("The database skipped rows. Read the complete result again. No file was stored.")
|
api/loopy_tools.py
CHANGED
|
@@ -4,7 +4,7 @@ import logging
|
|
| 4 |
|
| 5 |
MODULES = ("loopy_file_tools", "loopy_chat_link_tools", "loopy_skill_tools",
|
| 6 |
"loopy_catalog_tools", "loopy_diagnostic_tools", "loopy_worklist_tools",
|
| 7 |
-
"loopy_reminder_tools", "tool_groups.apps")
|
| 8 |
TOOLS = []
|
| 9 |
_LOG = logging.getLogger(__name__)
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
MODULES = ("loopy_file_tools", "loopy_chat_link_tools", "loopy_skill_tools",
|
| 6 |
"loopy_catalog_tools", "loopy_diagnostic_tools", "loopy_worklist_tools",
|
| 7 |
+
"loopy_reminder_tools", "tool_groups.apps", "tool_groups.live")
|
| 8 |
TOOLS = []
|
| 9 |
_LOG = logging.getLogger(__name__)
|
| 10 |
|
api/main.py
CHANGED
|
@@ -269,20 +269,32 @@ def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen):
|
|
| 269 |
|
| 270 |
@app.middleware("http")
|
| 271 |
async def _freeze_response(request: Request, call_next):
|
| 272 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
if request.method in {"GET", "HEAD", "OPTIONS"} or request.url.path == "/api/v1/query/chat/stream":
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
|
| 287 |
|
| 288 |
@app.exception_handler(data_binding.StoreWriteRefused)
|
|
@@ -412,10 +424,11 @@ def _mirror_answers(rt=None):
|
|
| 412 |
|
| 413 |
def _tenant_readiness(rt):
|
| 414 |
"""Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo."""
|
| 415 |
-
checks = {"mirror": False, "pools": False, "first_reads": False}
|
| 416 |
detail = {"mirror": "The tenant mirror check did not answer.",
|
| 417 |
"pools": "The tenant store did not answer.",
|
| 418 |
-
"first_reads": "The tenant's first read did not answer."
|
|
|
|
| 419 |
stage = "mirror"
|
| 420 |
try:
|
| 421 |
if rt.odoo_source() is not None:
|
|
@@ -432,6 +445,11 @@ def _tenant_readiness(rt):
|
|
| 432 |
# HF display reads hide outages, so readiness uses the tenant's strict reader.
|
| 433 |
rt._store()._read_strict(rt.store_key("sync_settings"))
|
| 434 |
checks["first_reads"] = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
except Exception as exc: # noqa: BLE001
|
| 436 |
detail[stage] += f" ({type(exc).__name__})"
|
| 437 |
failing = [name for name in checks if not checks[name]]
|
|
@@ -441,21 +459,52 @@ def _tenant_readiness(rt):
|
|
| 441 |
|
| 442 |
@app.get("/api/v1/ready")
|
| 443 |
def readiness_check(request: Request, response: Response):
|
| 444 |
-
"""Report tenant readiness publicly
|
|
|
|
| 445 |
session = (deps_require_session(request, response)
|
| 446 |
if request.cookies.get(aios_session.COOKIE_NAME) else None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
try:
|
| 448 |
from harness import runtime as _ready_runtime
|
| 449 |
if session is not None:
|
| 450 |
report = _tenant_readiness(session.runtime)
|
| 451 |
else:
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 455 |
try:
|
| 456 |
-
reports.append(_tenant_readiness(
|
| 457 |
except Exception:
|
| 458 |
reports.append({"checks": dict.fromkeys(names, False)})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 459 |
checks = {name: bool(reports) and all(r["checks"].get(name, False) for r in reports)
|
| 460 |
for name in names}
|
| 461 |
failing = [name for name in names if not checks[name]]
|
|
@@ -465,7 +514,9 @@ def readiness_check(request: Request, response: Response):
|
|
| 465 |
report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"],
|
| 466 |
"detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}}
|
| 467 |
body = {"ready": report["ready"], "checks": report["checks"]}
|
| 468 |
-
if
|
|
|
|
|
|
|
| 469 |
body["version"] = os.environ.get("AIOS_VERSION") or "unknown"
|
| 470 |
response.headers["Cache-Control"] = "no-store"
|
| 471 |
if report["ready"]:
|
|
@@ -704,6 +755,8 @@ app.include_router(routes_external.router)
|
|
| 704 |
import app_routes
|
| 705 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 706 |
_mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
|
|
|
|
|
|
|
| 707 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 708 |
_mount_router("routes_sku_actions")
|
| 709 |
_mount_router("notify_destinations")
|
|
@@ -977,6 +1030,131 @@ def _migrate_env_odoo(why):
|
|
| 977 |
print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}")
|
| 978 |
|
| 979 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 980 |
def _pull_meta(why, tenant_key=None):
|
| 981 |
"""Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it.
|
| 982 |
|
|
@@ -996,22 +1174,35 @@ def _pull_meta(why, tenant_key=None):
|
|
| 996 |
outcomes = {}
|
| 997 |
try:
|
| 998 |
import process_role as _meta_role
|
|
|
|
| 999 |
# Each tenant is pulled by exactly one worker: its own, else the primary.
|
| 1000 |
tenants = [tenant_key] if tenant_key else [
|
| 1001 |
key for key in runtime.known_tenants() if _meta_role.syncs_here(key)]
|
| 1002 |
-
|
| 1003 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1004 |
try:
|
| 1005 |
-
rt = runtime.get_runtime(key)
|
| 1006 |
if not _meta._connection_rows(rt, key, {"problems": []}):
|
| 1007 |
continue
|
|
|
|
|
|
|
|
|
|
| 1008 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1009 |
log=lambda *_a: None, insights_days=7)
|
| 1010 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
|
|
|
| 1011 |
for problem in rep.get("problems") or []:
|
| 1012 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}")
|
| 1013 |
except Exception as exc:
|
| 1014 |
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
|
|
|
|
|
|
|
|
|
| 1015 |
except Exception as exc:
|
| 1016 |
print(f"[aios-api] meta scheduling FAILED ({why}): {type(exc).__name__}")
|
| 1017 |
return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
|
|
@@ -1101,6 +1292,9 @@ def _pull_amazon(why, vendor=True):
|
|
| 1101 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 1102 |
f"for {tenant_key}; the connector is idle, not broken")
|
| 1103 |
return False
|
|
|
|
|
|
|
|
|
|
| 1104 |
# Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
|
| 1105 |
# bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
|
| 1106 |
# ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
|
|
@@ -1118,17 +1312,28 @@ def _pull_amazon(why, vendor=True):
|
|
| 1118 |
_rot.invalidate_sources()
|
| 1119 |
_rot._sources()
|
| 1120 |
if first_load_needed:
|
| 1121 |
-
|
| 1122 |
-
|
| 1123 |
-
|
| 1124 |
-
|
| 1125 |
if not vendor:
|
| 1126 |
print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
|
| 1127 |
return True
|
| 1128 |
from harness import mirror_store
|
| 1129 |
-
|
| 1130 |
-
|
| 1131 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1132 |
_rot.invalidate_sources()
|
| 1133 |
_rot._sources()
|
| 1134 |
for problem in rep.get("problems") or []:
|
|
@@ -1143,7 +1348,7 @@ def _pull_amazon(why, vendor=True):
|
|
| 1143 |
store=rt, quiet=_sa.amazon_standing(rep))
|
| 1144 |
except Exception: # noqa: BLE001
|
| 1145 |
pass
|
| 1146 |
-
return not bool(rep.get("
|
| 1147 |
except Exception as exc: # noqa: BLE001
|
| 1148 |
print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
|
| 1149 |
return False
|
|
@@ -1295,6 +1500,17 @@ def _sync_now_cause(cause):
|
|
| 1295 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 1296 |
import re
|
| 1297 |
if isinstance(cause, dict):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1298 |
sentence = "The connector could not read an expected value in the sync response."
|
| 1299 |
if cause.get("classifier") == "unreadable_value":
|
| 1300 |
table = cause.get("table")
|
|
@@ -1311,6 +1527,8 @@ def _sync_now_cause(cause):
|
|
| 1311 |
raw = str(cause)
|
| 1312 |
text = raw.lower()
|
| 1313 |
causes = (
|
|
|
|
|
|
|
| 1314 |
("response_too_large", ("reduce the amount of data", "response too large"),
|
| 1315 |
"The provider refused the pull because the response contains too much data."),
|
| 1316 |
("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
|
|
@@ -1389,7 +1607,7 @@ def _sync_now_failure(cause):
|
|
| 1389 |
return _sync_now_cause(cause)[1]
|
| 1390 |
|
| 1391 |
|
| 1392 |
-
def _amazon_sync_receipt(report):
|
| 1393 |
"""Project table windows and resync counts without vendor diagnostics or exception values."""
|
| 1394 |
import re
|
| 1395 |
details = []
|
|
@@ -1397,6 +1615,8 @@ def _amazon_sync_receipt(report):
|
|
| 1397 |
if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)):
|
| 1398 |
continue
|
| 1399 |
parts = []
|
|
|
|
|
|
|
| 1400 |
progress = row.get("resync")
|
| 1401 |
if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0
|
| 1402 |
for k in ("landed", "left")):
|
|
@@ -1405,8 +1625,27 @@ def _amazon_sync_receipt(report):
|
|
| 1405 |
if isinstance(window, str):
|
| 1406 |
# Window descriptions are authored by the loader, never copied from its diagnostics.
|
| 1407 |
parts.append(" ".join(window.split()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1408 |
if parts:
|
| 1409 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1410 |
return " | ".join(details)
|
| 1411 |
|
| 1412 |
|
|
@@ -1415,6 +1654,10 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
|
| 1415 |
if ok:
|
| 1416 |
return detail
|
| 1417 |
classifier, sentence = _sync_now_cause(cause)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1418 |
print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
|
| 1419 |
if isinstance(cause, BaseException):
|
| 1420 |
import traceback
|
|
@@ -1431,16 +1674,17 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
|
| 1431 |
return sentence + suffix
|
| 1432 |
|
| 1433 |
|
| 1434 |
-
def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None):
|
| 1435 |
-
"""Drain Sync now intents
|
| 1436 |
|
| 1437 |
-
|
| 1438 |
-
|
| 1439 |
`outcome`, when given, receives `{provider: "continuing" | "done" | "failed"}` so the loop can
|
| 1440 |
project only what a desk wake actually changed.
|
| 1441 |
"""
|
| 1442 |
from core import connector_sync
|
| 1443 |
-
|
|
|
|
| 1444 |
|
| 1445 |
def progress_log(provider, tenant, receipt, rt):
|
| 1446 |
"""Publish only authored progress shapes, with one bounded detail per running receipt."""
|
|
@@ -1512,11 +1756,40 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1512 |
|
| 1513 |
return log, lines
|
| 1514 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1515 |
odoo_requested = False
|
| 1516 |
# A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
|
| 1517 |
hold = during_odoo if hold_first_load is None else hold_first_load
|
| 1518 |
# Claim only work this turn executes. A later provider remains queued with an honest wait.
|
| 1519 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1520 |
refresh_remote=True,
|
| 1521 |
exclude_requested_by=("first load",) if hold else ())
|
| 1522 |
while True:
|
|
@@ -1531,8 +1804,10 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1531 |
ok, detail = False, ""
|
| 1532 |
first_load_progress, continuing = None, False
|
| 1533 |
floor_wait = None
|
|
|
|
| 1534 |
progress = ()
|
| 1535 |
amazon_detail, amazon_failure = "", None
|
|
|
|
| 1536 |
try:
|
| 1537 |
from harness import runtime, datastore, mirror_store
|
| 1538 |
rt = runtime.get_runtime(tenant)
|
|
@@ -1544,6 +1819,12 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1544 |
datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
|
| 1545 |
raise _SyncNowRefused(
|
| 1546 |
"This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1547 |
connector_sync.poll_remote_requests(rt)
|
| 1548 |
if provider == "odoo":
|
| 1549 |
import core.odoo as odoo
|
|
@@ -1574,6 +1855,8 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1574 |
problems = []
|
| 1575 |
elif provider == "amazon_sp":
|
| 1576 |
from harness import amazon_store as amazon
|
|
|
|
|
|
|
| 1577 |
_drain_amazon_connection_revocations(rt)
|
| 1578 |
if not amazon.credential(tenant, rt=rt):
|
| 1579 |
raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
|
|
@@ -1581,13 +1864,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1581 |
target_options = notification_sync_options(receipt)
|
| 1582 |
targeted = bool(target_options)
|
| 1583 |
first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
|
| 1584 |
-
options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS
|
| 1585 |
-
"refresh_loaded": True} if first_load else {}
|
| 1586 |
_log, progress = progress_log(provider, tenant, receipt, rt)
|
| 1587 |
-
|
| 1588 |
-
|
| 1589 |
-
|
| 1590 |
-
|
|
|
|
|
|
|
| 1591 |
print("[aios-api] amazon sync done (sync desk): " + ", ".join(
|
| 1592 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1593 |
+ (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
|
|
@@ -1599,11 +1883,17 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1599 |
# HOTFIX 2026-09-29: a continuation that landed nothing only met refresh floors.
|
| 1600 |
if continuing and report.get("landedTasks") == 0 and report.get("nextDueSeconds"):
|
| 1601 |
floor_wait = float(report["nextDueSeconds"])
|
| 1602 |
-
|
|
|
|
| 1603 |
tables = report.get("tables") or {}
|
| 1604 |
-
amazon_detail = _amazon_sync_receipt(report)
|
| 1605 |
amazon_failure = next((row["failure"] for row in tables.values()
|
| 1606 |
if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1607 |
if not problems and not tables and not continuing:
|
| 1608 |
raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
|
| 1609 |
if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
|
|
@@ -1621,7 +1911,15 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1621 |
raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
|
| 1622 |
else:
|
| 1623 |
raise _SyncNowRefused("This connector has no Sync now runner.")
|
| 1624 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1625 |
ok = True
|
| 1626 |
detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
|
| 1627 |
f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
|
|
@@ -1652,9 +1950,22 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
|
|
| 1652 |
first_load=first_load_progress, request_id=receipt.get("id"))
|
| 1653 |
else:
|
| 1654 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1655 |
if outcome is not None:
|
| 1656 |
-
outcome[provider] =
|
| 1657 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1658 |
_project_desk({provider: "done"}, tenant)
|
| 1659 |
if continuing and ok:
|
| 1660 |
# The continuation takes this run's desk slot: publish the finish to its web hand-off first.
|
|
@@ -1706,7 +2017,7 @@ def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations
|
|
| 1706 |
if state["present"] and (state["changed"] or force_relations):
|
| 1707 |
try:
|
| 1708 |
import automation_engine as _engine
|
| 1709 |
-
_engine.refresh_relations(rt, log=
|
| 1710 |
except Exception as e: # noqa: BLE001
|
| 1711 |
print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}")
|
| 1712 |
return state
|
|
@@ -1959,7 +2270,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1959 |
# refusal silently cancel a relational pass that had already succeeded.
|
| 1960 |
try:
|
| 1961 |
import automation_engine as _engine
|
| 1962 |
-
_engine.refresh_relations(_rt, log=
|
| 1963 |
except Exception as e: # noqa: BLE001
|
| 1964 |
rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
|
| 1965 |
print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
|
|
@@ -2141,9 +2452,36 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 2141 |
.get("databases") or {})
|
| 2142 |
except Exception: # noqa: BLE001
|
| 2143 |
db_health = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2144 |
|
| 2145 |
def _recovered(item):
|
| 2146 |
-
# D-13512: a prior failure leaves once its database ran clean
|
| 2147 |
key = item.get("key")
|
| 2148 |
if key in database_failures:
|
| 2149 |
return False
|
|
@@ -2152,19 +2490,11 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 2152 |
stage = str(item.get("stage") or "materialization")
|
| 2153 |
row = ((db_health.get(key) or {}).get("stages") or {}).get(stage) or {}
|
| 2154 |
ran, failed_at = _rel._parse_iso(row.get("at")), _rel._parse_iso(item.get("at"))
|
| 2155 |
-
|
| 2156 |
-
return True
|
| 2157 |
-
# Only a missing binding is cured by a binding that reads: a count mismatch reads rows too.
|
| 2158 |
-
if (not _rel.is_read_through_key(key)
|
| 2159 |
-
or "no readable mirror binding" not in str(item.get("cause") or "")):
|
| 2160 |
-
return False
|
| 2161 |
-
try:
|
| 2162 |
-
from routes_odoo_tables import population as _population
|
| 2163 |
-
return (_population(key, rt=_rt) or 0) > 0
|
| 2164 |
-
except Exception: # noqa: BLE001
|
| 2165 |
-
return False
|
| 2166 |
|
| 2167 |
kept = [item for item in prior if isinstance(item, dict) and not _recovered(item)]
|
|
|
|
|
|
|
| 2168 |
if failed_databases is not None and table_keys is not None:
|
| 2169 |
# D-2679: a changed-tables pass keeps the failures of databases it did not run.
|
| 2170 |
failed_databases = [
|
|
@@ -2974,11 +3304,79 @@ def _sheet_warm_after_pass(tenant, good, generation_before, rebuilt):
|
|
| 2974 |
print(f"[sheet-warm] request failed: {type(exc).__name__}: {exc}")
|
| 2975 |
|
| 2976 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2977 |
def _store_resync_loop():
|
| 2978 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 2979 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 2980 |
refreshed pool re-reads the freshly synced store."""
|
| 2981 |
-
|
|
|
|
| 2982 |
return
|
| 2983 |
import time as _t
|
| 2984 |
from core import connector_sync
|
|
@@ -3014,21 +3412,25 @@ def _store_resync_loop():
|
|
| 3014 |
last_full_pass_at = _t.time()
|
| 3015 |
# The least time between two desk-only passes of an Amazon first-load continuation.
|
| 3016 |
desk_min_spacing_seconds = 30.0
|
|
|
|
| 3017 |
# H73: the start of the last daily-slot pass, and whether the boot restore already ran.
|
| 3018 |
last_slot_pass_at = _memory.get("slotPassAt")
|
| 3019 |
slot_restored = False
|
| 3020 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 3021 |
slot_resume_checked = False
|
| 3022 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
|
|
|
| 3023 |
_slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
|
| 3024 |
_slot_interrupted_clear = globals().get("_resync_slot_interrupted_clear") or (lambda *_a, **_k: False)
|
| 3025 |
-
#
|
| 3026 |
_backfill_read = globals().get("_odoo_backfill_checkpoints")
|
| 3027 |
backfill_next_at = None
|
| 3028 |
backfill_continuations = 0
|
| 3029 |
backfill_spacing_seconds = 60.0
|
| 3030 |
-
backfill_cap =
|
| 3031 |
while True:
|
|
|
|
|
|
|
| 3032 |
# ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
| 3033 |
#
|
| 3034 |
# ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 —
|
|
@@ -3101,7 +3503,7 @@ def _store_resync_loop():
|
|
| 3101 |
_slot = (globals().get("_resolve_resync_slot") or (lambda: None))()
|
| 3102 |
|
| 3103 |
def _near_slot():
|
| 3104 |
-
#
|
| 3105 |
if _slot is None:
|
| 3106 |
return False
|
| 3107 |
import odoo_relational as _rel_near
|
|
@@ -3114,8 +3516,9 @@ def _store_resync_loop():
|
|
| 3114 |
# full pass too, so passes start a day apart however long the pass ran (a daily pass
|
| 3115 |
# that also refreshes every Amazon table can run an hour, and a wait counted from its
|
| 3116 |
# END drifted later every day and past the 26-hour stale line).
|
| 3117 |
-
_woken = bool(connector_sync.sync_wake_event()
|
| 3118 |
max(0.0, _every - (_t.time() - last_full_pass_at)),
|
|
|
|
| 3119 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
|
| 3120 |
connector_sync.sync_wake_event().clear()
|
| 3121 |
_slot_due = False
|
|
@@ -3126,9 +3529,10 @@ def _store_resync_loop():
|
|
| 3126 |
if passes or slot_restored:
|
| 3127 |
# Sleep to one second past the next slot; Sync now still interrupts the wait.
|
| 3128 |
_slot_now = _t.time()
|
| 3129 |
-
_woken = bool(connector_sync.sync_wake_event()
|
| 3130 |
0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
| 3131 |
else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
|
|
|
|
| 3132 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
|
| 3133 |
connector_sync.sync_wake_event().clear()
|
| 3134 |
_slot_now = _t.time()
|
|
@@ -3161,6 +3565,11 @@ def _store_resync_loop():
|
|
| 3161 |
f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
|
| 3162 |
else:
|
| 3163 |
RESYNC_STATE.pop("nextPassAt", None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3164 |
# ⛔⛔ HOTFIX 2026-09-29 — A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
|
| 3165 |
# capped syncing at once a day, and a deploy is not a reason to sync again. So the first
|
| 3166 |
# pass after a boot that lands within one cadence of the last scheduled pass only catches
|
|
@@ -3181,7 +3590,7 @@ def _store_resync_loop():
|
|
| 3181 |
if _woken and (_t.time() - last_full_pass_at < _every if _slot is None else not _slot_due):
|
| 3182 |
_desk = {}
|
| 3183 |
_desk_started = _t.time()
|
| 3184 |
-
if not _run_sync_requests(_desk, hold_first_load=
|
| 3185 |
_project_desk(_desk, _cad["tenant"])
|
| 3186 |
if "continuing" in _desk.values():
|
| 3187 |
# A floor between continuations, so a chunk that returns at once cannot spin.
|
|
@@ -3192,23 +3601,27 @@ def _store_resync_loop():
|
|
| 3192 |
_odoo_requested = None
|
| 3193 |
# H1: a queued backfill pass that is due, unless the slot, the cadence or a Sync now runs.
|
| 3194 |
_continuation = (backfill_next_at is not None and _t.time() >= backfill_next_at - 1.0
|
|
|
|
| 3195 |
and _odoo_requested is None and not _slot_due
|
| 3196 |
and (_slot is not None or _t.time() - last_full_pass_at < _every))
|
| 3197 |
if _slot is not None and not _slot_due and _odoo_requested is None and not _continuation:
|
| 3198 |
# H73: outside the slot no Odoo call is made. Serve the stored mirror, drain the desk.
|
| 3199 |
_desk = {}
|
| 3200 |
-
if _run_sync_requests(_desk, hold_first_load=
|
| 3201 |
_odoo_requested = True
|
| 3202 |
else:
|
| 3203 |
_project_desk(_desk, _cad["tenant"])
|
| 3204 |
if not slot_restored:
|
| 3205 |
-
|
|
|
|
| 3206 |
# The restored mirror warms the workbooks once; Sheet warm reads only the mirror.
|
| 3207 |
_sheet_warm_after_pass(_cad["tenant"], True, None, True)
|
| 3208 |
slot_restored = True
|
| 3209 |
print(f"[aios-api] store resync: no Odoo pass outside the daily slot; the next one "
|
| 3210 |
f"runs at {RESYNC_STATE['nextPassAt']} ({_slot['at']} {_slot['tz']})")
|
| 3211 |
continue
|
|
|
|
|
|
|
| 3212 |
passes += 1
|
| 3213 |
backfill_next_at = None
|
| 3214 |
if not _continuation:
|
|
@@ -3220,10 +3633,20 @@ def _store_resync_loop():
|
|
| 3220 |
# H73: stamp the slot BEFORE any Odoo read. H2: `slotDoneAt` marks its end, any outcome.
|
| 3221 |
last_slot_pass_at = last_full_pass_at
|
| 3222 |
_slot_write(last_full_pass_at)
|
| 3223 |
-
|
| 3224 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3225 |
_warm_generation = _sheet_warm_generation()
|
| 3226 |
_backfill_before = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3227 |
try:
|
| 3228 |
selected_entities = _configured_mirror_entities(_cad["tenant"])
|
| 3229 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
|
|
@@ -3261,6 +3684,10 @@ def _store_resync_loop():
|
|
| 3261 |
print(f"[aios-api] store resync failed ({_err['kind']}, "
|
| 3262 |
f"{RESYNC_STATE['consecutiveFailures']} in a row): "
|
| 3263 |
f"{_err['type']}: {_err['message']}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3264 |
if _publication_held:
|
| 3265 |
print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; "
|
| 3266 |
"the next admitted pass resumes the unfinished Odoo cohort")
|
|
@@ -3270,8 +3697,8 @@ def _store_resync_loop():
|
|
| 3270 |
_backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3271 |
_moved = ([row for row in _backfill_after if row not in _backfill_before]
|
| 3272 |
if _err is None and _backfill_after and _backfill_before is not None else [])
|
| 3273 |
-
if _moved and
|
| 3274 |
-
print(f"[aios-api] odoo backfill stops for today:
|
| 3275 |
f"the next slot pass or Sync now resumes it")
|
| 3276 |
elif _moved:
|
| 3277 |
backfill_continuations += 1
|
|
@@ -3279,7 +3706,7 @@ def _store_resync_loop():
|
|
| 3279 |
_row = max(_moved, key=lambda row: row[4] or 0)
|
| 3280 |
print(f"[aios-api] odoo backfill continues: {str(_row[0]).split('.', 1)[0]} at "
|
| 3281 |
f"{_row[4] or 0:,} rows; next bounded pass queued "
|
| 3282 |
-
f"({backfill_continuations}
|
| 3283 |
continue
|
| 3284 |
if _boot_catch_up and not _odoo_requested:
|
| 3285 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
|
@@ -3441,9 +3868,10 @@ def _store_resync_loop():
|
|
| 3441 |
except Exception as e: # noqa: BLE001
|
| 3442 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 3443 |
f"{type(e).__name__}: {e}")
|
| 3444 |
-
|
|
|
|
| 3445 |
_pull_amazon("boot", False)
|
| 3446 |
-
|
| 3447 |
# The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
|
| 3448 |
RESYNC_STATE["slotPass"] = _slot_claimed
|
| 3449 |
try:
|
|
@@ -3457,7 +3885,8 @@ def _store_resync_loop():
|
|
| 3457 |
try:
|
| 3458 |
from harness import runtime as _runtime
|
| 3459 |
_rt_resync = _runtime.get_runtime(_cad["tenant"])
|
| 3460 |
-
_amzn_state =
|
|
|
|
| 3461 |
last_amzn_fingerprint = (_amzn_state["fingerprint"]
|
| 3462 |
if _amzn_state.get("known") else None)
|
| 3463 |
except Exception as e: # noqa: BLE001
|
|
@@ -3486,7 +3915,7 @@ def _store_resync_loop():
|
|
| 3486 |
_slot_write(_t.time(), "slotDoneAt")
|
| 3487 |
|
| 3488 |
|
| 3489 |
-
def _boot_app_migrations(why):
|
| 3490 |
"""Apply tenant App retirement migrations independently of mirror sync and reads."""
|
| 3491 |
if _freeze_holds("app migrations"):
|
| 3492 |
return
|
|
@@ -3495,7 +3924,8 @@ def _boot_app_migrations(why):
|
|
| 3495 |
import timeseries_retirement
|
| 3496 |
import tenant_scan
|
| 3497 |
from harness import runtime as _runtime
|
| 3498 |
-
tenants = list(tenant_scan.live_tenants(_runtime)
|
|
|
|
| 3499 |
except Exception as exc:
|
| 3500 |
print(f"[aios-api] app migrations ({why}) skipped: {type(exc).__name__}: {exc}")
|
| 3501 |
return
|
|
@@ -3512,6 +3942,28 @@ def _boot_app_migrations(why):
|
|
| 3512 |
print(f"[aios-api] Returns split ({why}) {slug}: {split['outcome']}")
|
| 3513 |
except Exception as exc:
|
| 3514 |
print(f"[aios-api] Returns split ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3515 |
try:
|
| 3516 |
result = application_registry.ensure_route_application(rt, slug)
|
| 3517 |
reason = f": {result['reason']}" if result.get("reason") else ""
|
|
@@ -3529,6 +3981,97 @@ def _boot_app_migrations(why):
|
|
| 3529 |
print(f"[aios-api] Command Center App ({why}) {slug}: skipped: {type(exc).__name__}")
|
| 3530 |
|
| 3531 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3532 |
def _prewarm():
|
| 3533 |
if _freeze_holds("prewarm"):
|
| 3534 |
return
|
|
@@ -3601,17 +4144,35 @@ def _seed_mirror_if_absent():
|
|
| 3601 |
from harness import runtime as _runtime
|
| 3602 |
MIRROR_SEED["attempted"] = True
|
| 3603 |
try:
|
| 3604 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3605 |
try:
|
| 3606 |
-
rt = _runtime.get_runtime(tenant_key)
|
| 3607 |
result = _ds.restore_odoo_mirror(tenant_key, rt)
|
| 3608 |
except Exception as exc:
|
| 3609 |
-
result =
|
| 3610 |
-
f"Odoo mirror restore failed for {tenant_key} ({type(exc).__name__})"]}
|
| 3611 |
-
_ds.ODOO_MIRROR_STATUS[str(_ds.path_for(tenant_key).resolve())] = result
|
| 3612 |
if result["state"] == "refused":
|
| 3613 |
MIRROR_SEED["cause"] = "; ".join(result["problems"])
|
| 3614 |
MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3615 |
if _ds.ensure_seed():
|
| 3616 |
MIRROR_SEED["seeded"] = True
|
| 3617 |
print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
|
|
@@ -3715,9 +4276,30 @@ def _await_mirror_restore():
|
|
| 3715 |
thread.start()
|
| 3716 |
|
| 3717 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3718 |
@app.on_event("startup")
|
| 3719 |
def _start_process_role():
|
| 3720 |
-
"""Web role:
|
| 3721 |
if data_binding.gate_process():
|
| 3722 |
return
|
| 3723 |
import process_role as _process_role
|
|
@@ -3730,12 +4312,22 @@ def _restore_sync_request_desks():
|
|
| 3730 |
import process_role as _process_role
|
| 3731 |
# Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
|
| 3732 |
requeue = _process_role.may_write()
|
| 3733 |
-
|
|
|
|
|
|
|
|
|
|
| 3734 |
try:
|
| 3735 |
-
connector_sync.restore_sync_requests(
|
| 3736 |
except Exception as exc:
|
| 3737 |
print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
|
| 3738 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3739 |
|
| 3740 |
def _boot_mirrors():
|
| 3741 |
import process_role as _role_boot
|
|
@@ -3773,6 +4365,53 @@ _READ_WARM_ATTEMPTS = 40
|
|
| 3773 |
_READ_WARM_RETRY_SECONDS = 15
|
| 3774 |
|
| 3775 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3776 |
def _probe_read_pools(rt):
|
| 3777 |
"""Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
|
| 3778 |
from harness import datastore as _probe_ds
|
|
@@ -3781,11 +4420,18 @@ def _probe_read_pools(rt):
|
|
| 3781 |
physical = _probe_ds.serving_file(path)[0] if _probe_ds.versioned_mode() else path
|
| 3782 |
if not physical.exists():
|
| 3783 |
return "The mirror file is missing."
|
| 3784 |
-
|
| 3785 |
-
|
| 3786 |
-
|
| 3787 |
-
|
| 3788 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3789 |
if rt.available() is False:
|
| 3790 |
return "The tenant store did not answer."
|
| 3791 |
return ""
|
|
@@ -3800,6 +4446,7 @@ def _warm_read_pools(attempts=None, retry_seconds=None):
|
|
| 3800 |
import process_role as _warm_role
|
| 3801 |
attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
|
| 3802 |
retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
|
|
|
|
| 3803 |
for attempt in range(1, attempts + 1):
|
| 3804 |
t0 = _t.time()
|
| 3805 |
stage = "pools"
|
|
@@ -3812,13 +4459,23 @@ def _warm_read_pools(attempts=None, retry_seconds=None):
|
|
| 3812 |
raise RuntimeError(problem)
|
| 3813 |
_warm_role.mark_ready("pools", True)
|
| 3814 |
stage = "first_reads"
|
| 3815 |
-
|
| 3816 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3817 |
_warm_role.mark_ready("first_reads", True)
|
| 3818 |
routes_customers.warm_bu_pools(rt)
|
| 3819 |
print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
|
| 3820 |
return True
|
| 3821 |
except Exception as exc: # noqa: BLE001
|
|
|
|
|
|
|
|
|
|
| 3822 |
if not _warm_role._READINESS.get(stage, {}).get("ok"):
|
| 3823 |
_warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
|
| 3824 |
f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
|
|
@@ -3833,14 +4490,16 @@ def _restore_declared_mirrors(why):
|
|
| 3833 |
"""Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
|
| 3834 |
from core import connector_spec
|
| 3835 |
from harness import mirror_store, runtime
|
|
|
|
|
|
|
| 3836 |
errors = RESYNC_STATE.setdefault("mirrorRestoreErrors", {})
|
| 3837 |
-
|
|
|
|
| 3838 |
for connector in tuple(connector_spec.MIRRORS):
|
| 3839 |
if connector in {"odoo", "amazon"}:
|
| 3840 |
continue # Their existing adapters also enforce seed and deletion policies.
|
| 3841 |
identity = f"{tenant_key}/{connector}"
|
| 3842 |
try:
|
| 3843 |
-
rt = runtime.get_runtime(tenant_key)
|
| 3844 |
result = mirror_store.restore_mirror(connector, tenant_key, rt)
|
| 3845 |
if result["problems"]:
|
| 3846 |
errors[identity] = {"state": result["state"], "problems": list(result["problems"])}
|
|
@@ -3849,9 +4508,25 @@ def _restore_declared_mirrors(why):
|
|
| 3849 |
print(f"[aios-api] {connector} mirror {tenant_key} {result['state']} ({why})")
|
| 3850 |
except Exception as exc:
|
| 3851 |
errors[identity] = {"state": "refused", "problems": [type(exc).__name__]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3852 |
return not errors
|
| 3853 |
|
| 3854 |
|
|
|
|
|
|
|
|
|
|
| 3855 |
try:
|
| 3856 |
from harness import datastore as _ds_boot
|
| 3857 |
if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
|
|
@@ -3875,6 +4550,7 @@ if (not data_binding.gate_process()
|
|
| 3875 |
name="qa-sandbox-schema").start()
|
| 3876 |
|
| 3877 |
if (not data_binding.gate_process()
|
|
|
|
| 3878 |
and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 3879 |
and (bool(os.environ.get("SPACE_ID")) or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
| 3880 |
and not _freeze_holds("app migrations")):
|
|
@@ -3888,9 +4564,13 @@ if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
|
|
| 3888 |
if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
|
| 3889 |
import marketing_stream_loop
|
| 3890 |
import ads_schedule_loop
|
|
|
|
|
|
|
| 3891 |
import routes_returns_refunds
|
| 3892 |
marketing_stream_loop.start()
|
| 3893 |
ads_schedule_loop.start()
|
|
|
|
|
|
|
| 3894 |
routes_returns_refunds.start_auto_approve_worker()
|
| 3895 |
|
| 3896 |
|
|
@@ -3898,8 +4578,10 @@ if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
|
|
| 3898 |
def _stop_amazon_app_workers():
|
| 3899 |
import marketing_stream_loop
|
| 3900 |
import ads_schedule_loop
|
|
|
|
| 3901 |
marketing_stream_loop.stop()
|
| 3902 |
ads_schedule_loop.stop()
|
|
|
|
| 3903 |
|
| 3904 |
|
| 3905 |
if (not data_binding.gate_process()
|
|
@@ -3939,9 +4621,15 @@ if (not data_binding.gate_process()
|
|
| 3939 |
# The other structural cost remains outside that skip policy:
|
| 3940 |
# * the whole-document store write itself. A partial-JSONB write is a wave.
|
| 3941 |
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 3942 |
-
|
| 3943 |
-
|
| 3944 |
-
|
| 3945 |
-
|
| 3946 |
-
|
| 3947 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
|
| 270 |
@app.middleware("http")
|
| 271 |
async def _freeze_response(request: Request, call_next):
|
| 272 |
+
"""Enforce read-only member views and preserve maintenance refusals and verified response identity."""
|
| 273 |
+
import deps
|
| 274 |
+
try:
|
| 275 |
+
deps.refuse_view_as_write(request)
|
| 276 |
+
except StarletteHTTPException as exc:
|
| 277 |
+
return _error_shape(request, exc)
|
| 278 |
if request.method in {"GET", "HEAD", "OPTIONS"} or request.url.path == "/api/v1/query/chat/stream":
|
| 279 |
+
response = await call_next(request)
|
| 280 |
+
else:
|
| 281 |
+
with freeze_notice.refusal_scope() as refused:
|
| 282 |
+
try:
|
| 283 |
+
response = await call_next(request)
|
| 284 |
+
except Exception:
|
| 285 |
+
if not refused[0]:
|
| 286 |
+
raise
|
| 287 |
+
return _data_input_frozen(request, freeze_notice.DataInputFrozen())
|
| 288 |
+
if refused[0]:
|
| 289 |
+
# Replace the response, including any cookie minted from unpersisted auth state.
|
| 290 |
+
return _data_input_frozen(request, freeze_notice.DataInputFrozen())
|
| 291 |
+
identity = getattr(request.state, "_view_as_audited", None)
|
| 292 |
+
if identity is not None:
|
| 293 |
+
tenant, _actor, member = identity
|
| 294 |
+
response.headers["X-Viewing-As"] = member
|
| 295 |
+
response.headers[deps._TENANT_HEADER] = str(tenant)
|
| 296 |
+
response.headers[deps._PRINCIPAL_HEADER] = deps._principal(tenant, member)
|
| 297 |
+
return response
|
| 298 |
|
| 299 |
|
| 300 |
@app.exception_handler(data_binding.StoreWriteRefused)
|
|
|
|
| 424 |
|
| 425 |
def _tenant_readiness(rt):
|
| 426 |
"""Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo."""
|
| 427 |
+
checks = {"mirror": False, "pools": False, "first_reads": False, "tenant_keys": False}
|
| 428 |
detail = {"mirror": "The tenant mirror check did not answer.",
|
| 429 |
"pools": "The tenant store did not answer.",
|
| 430 |
+
"first_reads": "The tenant's first read did not answer.",
|
| 431 |
+
"tenant_keys": "The tenant data keys did not answer."}
|
| 432 |
stage = "mirror"
|
| 433 |
try:
|
| 434 |
if rt.odoo_source() is not None:
|
|
|
|
| 445 |
# HF display reads hide outages, so readiness uses the tenant's strict reader.
|
| 446 |
rt._store()._read_strict(rt.store_key("sync_settings"))
|
| 447 |
checks["first_reads"] = True
|
| 448 |
+
stage = "tenant_keys"
|
| 449 |
+
from core import tenant_keys as _tenant_keys
|
| 450 |
+
checks["tenant_keys"], _keys_detail = _tenant_keys.readiness_check([rt.key])
|
| 451 |
+
if not checks["tenant_keys"]:
|
| 452 |
+
detail["tenant_keys"] = _keys_detail
|
| 453 |
except Exception as exc: # noqa: BLE001
|
| 454 |
detail[stage] += f" ({type(exc).__name__})"
|
| 455 |
failing = [name for name in checks if not checks[name]]
|
|
|
|
| 459 |
|
| 460 |
@app.get("/api/v1/ready")
|
| 461 |
def readiness_check(request: Request, response: Response):
|
| 462 |
+
"""Report tenant readiness publicly; only a verified session or the deploy nonce sees the version."""
|
| 463 |
+
import hmac as _ready_hmac
|
| 464 |
session = (deps_require_session(request, response)
|
| 465 |
if request.cookies.get(aios_session.COOKIE_NAME) else None)
|
| 466 |
+
# A deploy probe proves itself with the deploy nonce; it sees the version, never a tenant's data.
|
| 467 |
+
nonce = os.environ.get("AIOS_DEPLOY_NONCE") or ""
|
| 468 |
+
probe = bool(nonce) and _ready_hmac.compare_digest(
|
| 469 |
+
request.headers.get("x-aios-deploy-nonce") or "", nonce)
|
| 470 |
+
skipped = {}
|
| 471 |
try:
|
| 472 |
from harness import runtime as _ready_runtime
|
| 473 |
if session is not None:
|
| 474 |
report = _tenant_readiness(session.runtime)
|
| 475 |
else:
|
| 476 |
+
import tenant_scan as _ready_scan
|
| 477 |
+
names = ("mirror", "pools", "first_reads", "tenant_keys")
|
| 478 |
+
reports, unmirrored, resolved = [], [], set()
|
| 479 |
+
|
| 480 |
+
def failed(slug, _exc):
|
| 481 |
+
# An active tenant whose runtime fails is judged, and fails closed.
|
| 482 |
+
resolved.add(slug)
|
| 483 |
+
reports.append({"checks": dict.fromkeys(names, False)})
|
| 484 |
+
|
| 485 |
+
# R70: only active tenants with a declared mirror are judged; the others are named.
|
| 486 |
+
for key, tenant_rt in _ready_scan.live_tenants(_ready_runtime, on_error=failed):
|
| 487 |
+
resolved.add(key)
|
| 488 |
+
try:
|
| 489 |
+
mirrored = tenant_rt.odoo_source() is not None
|
| 490 |
+
except Exception:
|
| 491 |
+
mirrored = True
|
| 492 |
+
if not mirrored:
|
| 493 |
+
unmirrored.append((key, tenant_rt))
|
| 494 |
+
skipped[key] = "no mirror"
|
| 495 |
+
continue
|
| 496 |
try:
|
| 497 |
+
reports.append(_tenant_readiness(tenant_rt))
|
| 498 |
except Exception:
|
| 499 |
reports.append({"checks": dict.fromkeys(names, False)})
|
| 500 |
+
for key in _ready_runtime.known_tenants():
|
| 501 |
+
if key not in resolved:
|
| 502 |
+
skipped[key] = "inactive"
|
| 503 |
+
if not reports:
|
| 504 |
+
# No tenant declares a mirror, so the active tenants' own stores answer for the process.
|
| 505 |
+
for key, tenant_rt in unmirrored:
|
| 506 |
+
skipped.pop(key, None)
|
| 507 |
+
reports.append(_tenant_readiness(tenant_rt))
|
| 508 |
checks = {name: bool(reports) and all(r["checks"].get(name, False) for r in reports)
|
| 509 |
for name in names}
|
| 510 |
failing = [name for name in names if not checks[name]]
|
|
|
|
| 514 |
report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"],
|
| 515 |
"detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}}
|
| 516 |
body = {"ready": report["ready"], "checks": report["checks"]}
|
| 517 |
+
if skipped:
|
| 518 |
+
body["skipped"] = skipped
|
| 519 |
+
if session is not None or probe:
|
| 520 |
body["version"] = os.environ.get("AIOS_VERSION") or "unknown"
|
| 521 |
response.headers["Cache-Control"] = "no-store"
|
| 522 |
if report["ready"]:
|
|
|
|
| 755 |
import app_routes
|
| 756 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 757 |
_mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
|
| 758 |
+
_mount_router("routes_amazon_projects")
|
| 759 |
+
_mount_router("routes_competitors")
|
| 760 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 761 |
_mount_router("routes_sku_actions")
|
| 762 |
_mount_router("notify_destinations")
|
|
|
|
| 1030 |
print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}")
|
| 1031 |
|
| 1032 |
|
| 1033 |
+
def _publish_connector_status(rt, provider, outcome=None, *, detail="", connection_id=None, report=None):
|
| 1034 |
+
"""C-status: write a pass end and the mirror's completed sync to the tenant's status document.
|
| 1035 |
+
|
| 1036 |
+
`outcome=None` writes only the completed sync, so a restored mirror seeds the store. Odoo's
|
| 1037 |
+
good sync is the pass end itself. Never raises: a status write cannot fail a pass.
|
| 1038 |
+
"""
|
| 1039 |
+
try:
|
| 1040 |
+
from core import connector_sync as _cs
|
| 1041 |
+
good = None
|
| 1042 |
+
if provider == "odoo":
|
| 1043 |
+
good = _cs._utc_stamp() if outcome == "done" else None
|
| 1044 |
+
else:
|
| 1045 |
+
import routes_connectors as _rc
|
| 1046 |
+
stamp, known = _rc._mirror_completed_sync(rt, provider)
|
| 1047 |
+
good = stamp if known else None
|
| 1048 |
+
if outcome is None:
|
| 1049 |
+
return _cs.record_good_sync(rt, provider, good, connection_id=connection_id) if good else False
|
| 1050 |
+
extra = {}
|
| 1051 |
+
if (provider == "odoo" or provider == "amazon_sp" and RESYNC_STATE.get("slotPass")) and RESYNC_STATE.get("slotDate"):
|
| 1052 |
+
extra["slotDate"] = RESYNC_STATE["slotDate"]
|
| 1053 |
+
if provider == "amazon_sp" and isinstance(report, dict):
|
| 1054 |
+
from harness import amazon_store as _az
|
| 1055 |
+
holds = _az._MIRROR_STORE._publication_holds(report, ()) or {}
|
| 1056 |
+
extra["notOffered"] = len(holds.get("standing") or ())
|
| 1057 |
+
extra["notOfferedTables"] = sorted(holds.get("standing") or ())
|
| 1058 |
+
extra["held"] = {name: {"reason": "Amazon does not offer this report to this seller."
|
| 1059 |
+
if name in (holds.get("standing") or ())
|
| 1060 |
+
else "This table did not finish. Its last published rows are kept."}
|
| 1061 |
+
for name in holds.get("held") or {}}
|
| 1062 |
+
previous = _cs.status(rt, provider, connection_id, fresh=True) or {}
|
| 1063 |
+
stamps = dict((previous.get("pass") or {}).get("tableSync") or {})
|
| 1064 |
+
for name, row in (report.get("tables") or {}).items():
|
| 1065 |
+
if name not in extra["held"] and isinstance(row, dict) and row.get("state", "done") == "done":
|
| 1066 |
+
stamps[name] = good or _cs._utc_stamp()
|
| 1067 |
+
extra["tableSync"] = stamps
|
| 1068 |
+
if extra["held"]:
|
| 1069 |
+
from harness.runtime import mirror_cursor
|
| 1070 |
+
cur = None
|
| 1071 |
+
try:
|
| 1072 |
+
cur = mirror_cursor(rt, require_ready=False)
|
| 1073 |
+
for name, held in extra["held"].items():
|
| 1074 |
+
table = _az.table_name(name)
|
| 1075 |
+
if _az._MIRROR_STORE._valid_table(table):
|
| 1076 |
+
try:
|
| 1077 |
+
held["rows"] = int(cur.execute(f'SELECT count(*) FROM "{table}"').fetchone()[0])
|
| 1078 |
+
except Exception:
|
| 1079 |
+
held["rows"] = None
|
| 1080 |
+
except Exception:
|
| 1081 |
+
for held in extra["held"].values():
|
| 1082 |
+
held["rows"] = None
|
| 1083 |
+
finally:
|
| 1084 |
+
if cur is not None:
|
| 1085 |
+
cur.close()
|
| 1086 |
+
return _cs.record_pass_end(rt, provider, outcome, detail=detail, last_good_sync_at=good,
|
| 1087 |
+
connection_id=connection_id, extra=extra)
|
| 1088 |
+
except Exception as exc: # noqa: BLE001
|
| 1089 |
+
print(f"[aios-api] connector status not written for {provider} ({type(exc).__name__})", flush=True)
|
| 1090 |
+
return False
|
| 1091 |
+
|
| 1092 |
+
|
| 1093 |
+
#: The last resync snapshot written to the store, so an unchanged loop state is not rewritten.
|
| 1094 |
+
_RESYNC_PUBLISHED = [None]
|
| 1095 |
+
|
| 1096 |
+
|
| 1097 |
+
def _publish_resync_state(tenant_key, *, started=False, ended=False, outcome=None):
|
| 1098 |
+
"""C-status: copy RESYNC_STATE, and the Odoo pass it ran, to the tenant's status document."""
|
| 1099 |
+
try:
|
| 1100 |
+
tenant_key = str(tenant_key or "")
|
| 1101 |
+
if not tenant_key:
|
| 1102 |
+
return False
|
| 1103 |
+
from core import connector_sync as _cs
|
| 1104 |
+
from harness import runtime as _status_runtime
|
| 1105 |
+
rt = _status_runtime.get_runtime(tenant_key)
|
| 1106 |
+
cid = str(RESYNC_STATE.get("connectionId") or "default")
|
| 1107 |
+
if started:
|
| 1108 |
+
_cs.record_pass_start(rt, "odoo", connection_id=cid)
|
| 1109 |
+
if ended:
|
| 1110 |
+
error = RESYNC_STATE.get("lastError")
|
| 1111 |
+
_publish_connector_status(rt, "odoo", outcome, connection_id=cid,
|
| 1112 |
+
detail=str(error.get("message") or "") if isinstance(error, dict) else "")
|
| 1113 |
+
import json as _json
|
| 1114 |
+
snapshot = _json.dumps(RESYNC_STATE, sort_keys=True, default=str)
|
| 1115 |
+
if snapshot != _RESYNC_PUBLISHED[0] and _cs.record_resync_state(
|
| 1116 |
+
rt, "odoo", RESYNC_STATE, connection_id=cid):
|
| 1117 |
+
_RESYNC_PUBLISHED[0] = snapshot
|
| 1118 |
+
return True
|
| 1119 |
+
except Exception as exc: # noqa: BLE001
|
| 1120 |
+
print(f"[aios-api] resync status not written ({type(exc).__name__})", flush=True)
|
| 1121 |
+
return False
|
| 1122 |
+
|
| 1123 |
+
|
| 1124 |
+
def _connector_publication_status(connector, tenant_key, manifest, _rows, _paths):
|
| 1125 |
+
"""Close a running pass at publication, before Amazon queues its App callbacks."""
|
| 1126 |
+
if not process_role.may_write() or not manifest.get("snapshot_id"):
|
| 1127 |
+
return
|
| 1128 |
+
provider = {"odoo": "odoo", "amazon": "amazon_sp", "meta": "meta_ads"}.get(connector)
|
| 1129 |
+
if provider is None:
|
| 1130 |
+
return
|
| 1131 |
+
from core import connector_sync
|
| 1132 |
+
from harness import runtime
|
| 1133 |
+
rt = runtime.get_runtime(tenant_key)
|
| 1134 |
+
cid = None
|
| 1135 |
+
if provider == "odoo":
|
| 1136 |
+
import odoo_relational
|
| 1137 |
+
cid = str(odoo_relational.default_connection_id(rt) or "default")
|
| 1138 |
+
entry = connector_sync.status(rt, provider, cid, fresh=True) or {}
|
| 1139 |
+
last_pass = entry.get("pass") or {}
|
| 1140 |
+
if last_pass.get("outcome") != "running":
|
| 1141 |
+
return
|
| 1142 |
+
from datetime import datetime
|
| 1143 |
+
started = connector_sync._utc_stamp(last_pass["startedAt"]) if last_pass.get("startedAt") else None
|
| 1144 |
+
at = connector_sync._utc_stamp(manifest["completed_at"]) if manifest.get("completed_at") else None
|
| 1145 |
+
if not started or not at or datetime.fromisoformat(at) < datetime.fromisoformat(started):
|
| 1146 |
+
return
|
| 1147 |
+
extra = {"publishedAt": at}
|
| 1148 |
+
if RESYNC_STATE.get("slotDate") and (provider == "odoo" or RESYNC_STATE.get("slotPass")):
|
| 1149 |
+
extra["slotDate"] = RESYNC_STATE["slotDate"]
|
| 1150 |
+
connector_sync.record_pass_end(rt, provider, "done", at=at, last_good_sync_at=at,
|
| 1151 |
+
connection_id=cid, extra=extra)
|
| 1152 |
+
|
| 1153 |
+
|
| 1154 |
+
from harness import mirror_store as _status_mirrors
|
| 1155 |
+
_status_mirrors.register_publish_listener(_connector_publication_status)
|
| 1156 |
+
|
| 1157 |
+
|
| 1158 |
def _pull_meta(why, tenant_key=None):
|
| 1159 |
"""Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it.
|
| 1160 |
|
|
|
|
| 1174 |
outcomes = {}
|
| 1175 |
try:
|
| 1176 |
import process_role as _meta_role
|
| 1177 |
+
import tenant_scan as _meta_scan
|
| 1178 |
# Each tenant is pulled by exactly one worker: its own, else the primary.
|
| 1179 |
tenants = [tenant_key] if tenant_key else [
|
| 1180 |
key for key in runtime.known_tenants() if _meta_role.syncs_here(key)]
|
| 1181 |
+
outcomes.update(dict.fromkeys(tenants, False))
|
| 1182 |
+
|
| 1183 |
+
def runtime_failed(key, exc):
|
| 1184 |
+
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
| 1185 |
+
|
| 1186 |
+
# S82-T18: an inactive tenant is skipped with one named line, never a KeyError per pass.
|
| 1187 |
+
for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
|
| 1188 |
+
on_error=runtime_failed):
|
| 1189 |
try:
|
|
|
|
| 1190 |
if not _meta._connection_rows(rt, key, {"problems": []}):
|
| 1191 |
continue
|
| 1192 |
+
from core import connector_sync as _meta_status
|
| 1193 |
+
_meta_status.record_pass_start(rt, "meta_ads")
|
| 1194 |
+
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1195 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1196 |
log=lambda *_a: None, insights_days=7)
|
| 1197 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1198 |
+
_status(rt, "meta_ads", "done" if outcomes[key] else "failed")
|
| 1199 |
for problem in rep.get("problems") or []:
|
| 1200 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}")
|
| 1201 |
except Exception as exc:
|
| 1202 |
print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
|
| 1203 |
+
_failed = globals().get("_publish_connector_status")
|
| 1204 |
+
if _failed:
|
| 1205 |
+
_failed(rt, "meta_ads", "failed", detail="The Meta Ads pull failed.")
|
| 1206 |
except Exception as exc:
|
| 1207 |
print(f"[aios-api] meta scheduling FAILED ({why}): {type(exc).__name__}")
|
| 1208 |
return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
|
|
|
|
| 1292 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 1293 |
f"for {tenant_key}; the connector is idle, not broken")
|
| 1294 |
return False
|
| 1295 |
+
if _amazon.first_load_job_live(rt):
|
| 1296 |
+
print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
|
| 1297 |
+
return False
|
| 1298 |
# Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
|
| 1299 |
# bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
|
| 1300 |
# ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
|
|
|
|
| 1312 |
_rot.invalidate_sources()
|
| 1313 |
_rot._sources()
|
| 1314 |
if first_load_needed:
|
| 1315 |
+
print(f"[aios-api] amazon sync ({why}): history belongs to the independent first-load job")
|
| 1316 |
+
# C-status: the restored mirror seeds the store's last good sync; an older stamp never moves it back.
|
| 1317 |
+
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1318 |
+
_status(rt, "amazon_sp")
|
| 1319 |
if not vendor:
|
| 1320 |
print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
|
| 1321 |
return True
|
| 1322 |
from harness import mirror_store
|
| 1323 |
+
from core import connector_sync as _amazon_status
|
| 1324 |
+
_amazon_status.record_pass_start(rt, "amazon_sp")
|
| 1325 |
+
try:
|
| 1326 |
+
with _amazon_role.connector_writer(rt, "amazon"):
|
| 1327 |
+
rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 1328 |
+
slot_pass=bool(RESYNC_STATE.get("slotPass")),
|
| 1329 |
+
log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
|
| 1330 |
+
except Exception:
|
| 1331 |
+
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1332 |
+
raise
|
| 1333 |
+
_holds = _amazon._MIRROR_STORE._publication_holds(rep, ()) or {}
|
| 1334 |
+
_problems = _holds.get("unsettled", rep.get("problems") or [])
|
| 1335 |
+
_status(rt, "amazon_sp", "failed" if _problems or rep.get("audit_problems") else
|
| 1336 |
+
"continuing" if rep.get("state") == "partial" else "done", report=rep)
|
| 1337 |
_rot.invalidate_sources()
|
| 1338 |
_rot._sources()
|
| 1339 |
for problem in rep.get("problems") or []:
|
|
|
|
| 1348 |
store=rt, quiet=_sa.amazon_standing(rep))
|
| 1349 |
except Exception: # noqa: BLE001
|
| 1350 |
pass
|
| 1351 |
+
return not bool(_problems or rep.get("audit_problems"))
|
| 1352 |
except Exception as exc: # noqa: BLE001
|
| 1353 |
print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
|
| 1354 |
return False
|
|
|
|
| 1500 |
"""Classify actual provider failures without returning vendor text or exception reprs."""
|
| 1501 |
import re
|
| 1502 |
if isinstance(cause, dict):
|
| 1503 |
+
if cause.get("amazonStatus") == "NOT_OFFERED" or cause.get("classifier") == "not_offered":
|
| 1504 |
+
return "not_offered", "Amazon does not offer this report to this seller. Its last published rows are kept."
|
| 1505 |
+
# S82-T04 zero-row class: a pass that moved no rows because nothing was due names when it reopens.
|
| 1506 |
+
if cause.get("classifier") == "nothing_due":
|
| 1507 |
+
import datetime as _dt
|
| 1508 |
+
at = cause.get("nextAt")
|
| 1509 |
+
sentence = "Amazon had nothing due in this pass, so no rows moved. Each table waits for its next report window."
|
| 1510 |
+
if isinstance(at, (int, float)) and not isinstance(at, bool) and 0 < at < 4102444800:
|
| 1511 |
+
when = _dt.datetime.fromtimestamp(at, _dt.timezone.utc)
|
| 1512 |
+
sentence += f" The next window opens at {when:%H:%M} UTC on {when.day} {when:%b}."
|
| 1513 |
+
return "nothing_due", sentence
|
| 1514 |
sentence = "The connector could not read an expected value in the sync response."
|
| 1515 |
if cause.get("classifier") == "unreadable_value":
|
| 1516 |
table = cause.get("table")
|
|
|
|
| 1527 |
raw = str(cause)
|
| 1528 |
text = raw.lower()
|
| 1529 |
causes = (
|
| 1530 |
+
("not_offered", ("not_offered", "not offered", "does not offer this report"),
|
| 1531 |
+
"Amazon does not offer this report to this seller. Its last published rows are kept."),
|
| 1532 |
("response_too_large", ("reduce the amount of data", "response too large"),
|
| 1533 |
"The provider refused the pull because the response contains too much data."),
|
| 1534 |
("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
|
|
|
|
| 1607 |
return _sync_now_cause(cause)[1]
|
| 1608 |
|
| 1609 |
|
| 1610 |
+
def _amazon_sync_receipt(report, rt=None):
|
| 1611 |
"""Project table windows and resync counts without vendor diagnostics or exception values."""
|
| 1612 |
import re
|
| 1613 |
details = []
|
|
|
|
| 1615 |
if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)):
|
| 1616 |
continue
|
| 1617 |
parts = []
|
| 1618 |
+
if row.get("state") == "refused" and row.get("amazonStatus") == "NOT_OFFERED":
|
| 1619 |
+
parts.append("report not offered; last published rows kept")
|
| 1620 |
progress = row.get("resync")
|
| 1621 |
if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0
|
| 1622 |
for k in ("landed", "left")):
|
|
|
|
| 1625 |
if isinstance(window, str):
|
| 1626 |
# Window descriptions are authored by the loader, never copied from its diagnostics.
|
| 1627 |
parts.append(" ".join(window.split()))
|
| 1628 |
+
operation, resume_at = row.get("operation"), row.get("resume_at")
|
| 1629 |
+
if isinstance(operation, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", operation):
|
| 1630 |
+
parts.append(f"operation {operation}")
|
| 1631 |
+
if isinstance(resume_at, str):
|
| 1632 |
+
from core.connector_sync import _utc_stamp
|
| 1633 |
+
stamp = _utc_stamp(resume_at)
|
| 1634 |
+
if stamp:
|
| 1635 |
+
parts.append(f"resume_at {stamp}")
|
| 1636 |
if parts:
|
| 1637 |
+
line = f"{table}: " + "; ".join(parts)
|
| 1638 |
+
if operation or resume_at:
|
| 1639 |
+
details.insert(0, line)
|
| 1640 |
+
else:
|
| 1641 |
+
details.append(line)
|
| 1642 |
+
if rt is not None and not any("operation " in line or "resume_at " in line for line in details):
|
| 1643 |
+
from harness import amazon_store
|
| 1644 |
+
job = amazon_store.first_load_job_summary(rt)
|
| 1645 |
+
if job:
|
| 1646 |
+
tail = _amazon_sync_receipt({"tables": {"first_load": job}})
|
| 1647 |
+
if tail:
|
| 1648 |
+
details.append(tail)
|
| 1649 |
return " | ".join(details)
|
| 1650 |
|
| 1651 |
|
|
|
|
| 1654 |
if ok:
|
| 1655 |
return detail
|
| 1656 |
classifier, sentence = _sync_now_cause(cause)
|
| 1657 |
+
if classifier == "nothing_due":
|
| 1658 |
+
# A wait with a named cause is not a failure: no FAILED line, no failure location.
|
| 1659 |
+
print(f"[aios-api] sync WAITS provider={provider} tenant={tenant} classifier={classifier}")
|
| 1660 |
+
return sentence
|
| 1661 |
print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
|
| 1662 |
if isinstance(cause, BaseException):
|
| 1663 |
import traceback
|
|
|
|
| 1674 |
return sentence + suffix
|
| 1675 |
|
| 1676 |
|
| 1677 |
+
def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, amazon_only=False):
|
| 1678 |
+
"""Drain admitted Sync now intents before their resident projection.
|
| 1679 |
|
| 1680 |
+
HTTP handlers queue intents. The slot loop owns Odoo and Meta, and its detached turn owns Amazon.
|
| 1681 |
+
A failed request cannot discard another tenant's queued work.
|
| 1682 |
`outcome`, when given, receives `{provider: "continuing" | "done" | "failed"}` so the loop can
|
| 1683 |
project only what a desk wake actually changed.
|
| 1684 |
"""
|
| 1685 |
from core import connector_sync
|
| 1686 |
+
if amazon_only or globals().get("_AMAZON_DESK", {}).get("job") is None:
|
| 1687 |
+
_drain_pending_amazon_revocations()
|
| 1688 |
|
| 1689 |
def progress_log(provider, tenant, receipt, rt):
|
| 1690 |
"""Publish only authored progress shapes, with one bounded detail per running receipt."""
|
|
|
|
| 1756 |
|
| 1757 |
return log, lines
|
| 1758 |
|
| 1759 |
+
def nothing_due_at(report, problems, failure, tenant):
|
| 1760 |
+
"""S82-T04: the epoch the next Amazon window opens if a pass moved no rows and every table waited.
|
| 1761 |
+
|
| 1762 |
+
An unclassified problem on such a pass is a wait, not a failure. A named failure class keeps its FAILED.
|
| 1763 |
+
Nested on purpose: several gates compile this runner with a fixed set of main.py names.
|
| 1764 |
+
"""
|
| 1765 |
+
import time
|
| 1766 |
+
tables = report.get("tables") or {}
|
| 1767 |
+
rows = [row for row in tables.values() if isinstance(row, dict)]
|
| 1768 |
+
if (not rows or len(rows) != len(tables) or report.get("landedTasks")
|
| 1769 |
+
or any(row.get("state") != "deferred" or row.get("rows") for row in rows)):
|
| 1770 |
+
return None
|
| 1771 |
+
classify = globals().get("_sync_now_cause")
|
| 1772 |
+
if problems and classify and classify(failure or problems[0])[0] != "unclassified":
|
| 1773 |
+
return None
|
| 1774 |
+
now, idle = time.time(), 1800.0
|
| 1775 |
+
wait, held = report.get("nextDueSeconds"), connector_sync.sync_not_before("amazon_sp", tenant)
|
| 1776 |
+
if isinstance(wait, (int, float)) and not isinstance(wait, bool) and wait > 0:
|
| 1777 |
+
at = now + float(wait)
|
| 1778 |
+
elif held and held > now:
|
| 1779 |
+
at = float(held)
|
| 1780 |
+
else:
|
| 1781 |
+
at = now + idle # No floor is known: wait one desk cadence, never requeue at once.
|
| 1782 |
+
return min(max(at, now + 60.0), now + 86400.0)
|
| 1783 |
+
|
| 1784 |
odoo_requested = False
|
| 1785 |
# A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
|
| 1786 |
hold = during_odoo if hold_first_load is None else hold_first_load
|
| 1787 |
# Claim only work this turn executes. A later provider remains queued with an honest wait.
|
| 1788 |
+
excluded = ({"odoo", "meta_ads"} if amazon_only else
|
| 1789 |
+
({"odoo"} if during_odoo else set()))
|
| 1790 |
+
if not amazon_only and globals().get("_AMAZON_DESK", {}).get("managed"):
|
| 1791 |
+
excluded.add("amazon_sp")
|
| 1792 |
+
pending = connector_sync.take_sync_requests(limit=1, exclude_providers=excluded,
|
| 1793 |
refresh_remote=True,
|
| 1794 |
exclude_requested_by=("first load",) if hold else ())
|
| 1795 |
while True:
|
|
|
|
| 1804 |
ok, detail = False, ""
|
| 1805 |
first_load_progress, continuing = None, False
|
| 1806 |
floor_wait = None
|
| 1807 |
+
nothing_due, first_load = None, False
|
| 1808 |
progress = ()
|
| 1809 |
amazon_detail, amazon_failure = "", None
|
| 1810 |
+
status_rt, status_cid = None, None
|
| 1811 |
try:
|
| 1812 |
from harness import runtime, datastore, mirror_store
|
| 1813 |
rt = runtime.get_runtime(tenant)
|
|
|
|
| 1819 |
datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
|
| 1820 |
raise _SyncNowRefused(
|
| 1821 |
"This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
|
| 1822 |
+
# C-status: the pass starts on the resident writer; its end is written after the finish below.
|
| 1823 |
+
status_rt = rt
|
| 1824 |
+
if provider == "odoo":
|
| 1825 |
+
import routes_connectors as _status_routes
|
| 1826 |
+
status_cid = _status_routes._odoo_connection_slot(rt, None)
|
| 1827 |
+
connector_sync.record_pass_start(rt, provider, connection_id=status_cid)
|
| 1828 |
connector_sync.poll_remote_requests(rt)
|
| 1829 |
if provider == "odoo":
|
| 1830 |
import core.odoo as odoo
|
|
|
|
| 1855 |
problems = []
|
| 1856 |
elif provider == "amazon_sp":
|
| 1857 |
from harness import amazon_store as amazon
|
| 1858 |
+
if amazon.first_load_job_live(rt):
|
| 1859 |
+
raise _SyncNowRefused("The first-load job holds the Amazon writer lease.")
|
| 1860 |
_drain_amazon_connection_revocations(rt)
|
| 1861 |
if not amazon.credential(tenant, rt=rt):
|
| 1862 |
raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
|
|
|
|
| 1864 |
target_options = notification_sync_options(receipt)
|
| 1865 |
targeted = bool(target_options)
|
| 1866 |
first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
|
| 1867 |
+
options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {}
|
|
|
|
| 1868 |
_log, progress = progress_log(provider, tenant, receipt, rt)
|
| 1869 |
+
import process_role as _desk_role
|
| 1870 |
+
with _desk_role.connector_writer(rt, "amazon"):
|
| 1871 |
+
report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
|
| 1872 |
+
scheduled=False if targeted else receipt.get("requestedBy") == "first load",
|
| 1873 |
+
**target_options, **options,
|
| 1874 |
+
log=_log)
|
| 1875 |
print("[aios-api] amazon sync done (sync desk): " + ", ".join(
|
| 1876 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1877 |
+ (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
|
|
|
|
| 1883 |
# HOTFIX 2026-09-29: a continuation that landed nothing only met refresh floors.
|
| 1884 |
if continuing and report.get("landedTasks") == 0 and report.get("nextDueSeconds"):
|
| 1885 |
floor_wait = float(report["nextDueSeconds"])
|
| 1886 |
+
_holds = amazon._MIRROR_STORE._publication_holds(report, ()) or {}
|
| 1887 |
+
problems = list(_holds.get("unsettled", report.get("problems") or [])) + list(report.get("audit_problems") or [])
|
| 1888 |
tables = report.get("tables") or {}
|
| 1889 |
+
amazon_detail = _amazon_sync_receipt(report, rt)
|
| 1890 |
amazon_failure = next((row["failure"] for row in tables.values()
|
| 1891 |
if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None)
|
| 1892 |
+
# The floor timer below already serves a clean 0-row continuation that names its floor.
|
| 1893 |
+
nothing_due = (None if targeted or (continuing and floor_wait and not problems)
|
| 1894 |
+
else nothing_due_at(report, problems, amazon_failure, tenant))
|
| 1895 |
+
if nothing_due:
|
| 1896 |
+
continuing, floor_wait = False, None
|
| 1897 |
if not problems and not tables and not continuing:
|
| 1898 |
raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
|
| 1899 |
if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
|
|
|
|
| 1911 |
raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
|
| 1912 |
else:
|
| 1913 |
raise _SyncNowRefused("This connector has no Sync now runner.")
|
| 1914 |
+
if nothing_due:
|
| 1915 |
+
# S82-T04: nothing was due. Finish with the named cause, then wait for the floor.
|
| 1916 |
+
if problems:
|
| 1917 |
+
print(f"[aios-api] amazon sync: {len(problems)} unclassified problem(s) on a pass "
|
| 1918 |
+
f"with nothing due; see the connector sync report", flush=True)
|
| 1919 |
+
ok, problems, amazon_detail = True, [], ""
|
| 1920 |
+
detail = _sync_now_detail(provider, tenant, ok=False,
|
| 1921 |
+
cause={"classifier": "nothing_due", "nextAt": nothing_due})
|
| 1922 |
+
elif continuing:
|
| 1923 |
ok = True
|
| 1924 |
detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
|
| 1925 |
f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
|
|
|
|
| 1950 |
first_load=first_load_progress, request_id=receipt.get("id"))
|
| 1951 |
else:
|
| 1952 |
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
|
| 1953 |
+
pass_outcome = ("waiting" if nothing_due and ok else "continuing" if (continuing and ok)
|
| 1954 |
+
else ("done" if ok else "failed"))
|
| 1955 |
+
_status_write = globals().get("_publish_connector_status")
|
| 1956 |
+
if status_rt is not None and _status_write:
|
| 1957 |
+
_status_write(status_rt, provider, pass_outcome, detail=detail, connection_id=status_cid,
|
| 1958 |
+
**({"report": report} if provider == "amazon_sp" else {}))
|
| 1959 |
if outcome is not None:
|
| 1960 |
+
outcome[provider] = pass_outcome
|
| 1961 |
+
if nothing_due and ok and first_load:
|
| 1962 |
+
# The next continuation waits on the desk for the floor; requeueing at once spins 0-row passes.
|
| 1963 |
+
connector_sync.request_sync("amazon_sp", tenant, requested_by="first load",
|
| 1964 |
+
not_before=nothing_due, wait_cause=detail)
|
| 1965 |
+
import time as _time
|
| 1966 |
+
print(f"[aios-api] amazon first load had nothing due; it waits "
|
| 1967 |
+
f"{int(nothing_due - _time.time())}s for the next report window", flush=True)
|
| 1968 |
+
if during_odoo and ok and not continuing and not nothing_due:
|
| 1969 |
_project_desk({provider: "done"}, tenant)
|
| 1970 |
if continuing and ok:
|
| 1971 |
# The continuation takes this run's desk slot: publish the finish to its web hand-off first.
|
|
|
|
| 2017 |
if state["present"] and (state["changed"] or force_relations):
|
| 2018 |
try:
|
| 2019 |
import automation_engine as _engine
|
| 2020 |
+
_engine.refresh_relations(rt, log=print)
|
| 2021 |
except Exception as e: # noqa: BLE001
|
| 2022 |
print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}")
|
| 2023 |
return state
|
|
|
|
| 2270 |
# refusal silently cancel a relational pass that had already succeeded.
|
| 2271 |
try:
|
| 2272 |
import automation_engine as _engine
|
| 2273 |
+
_engine.refresh_relations(_rt, log=print)
|
| 2274 |
except Exception as e: # noqa: BLE001
|
| 2275 |
rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
|
| 2276 |
print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
|
|
|
|
| 2452 |
.get("databases") or {})
|
| 2453 |
except Exception: # noqa: BLE001
|
| 2454 |
db_health = {}
|
| 2455 |
+
# S84-T11: a missing binding is cured only when its own default-mirror binding reads.
|
| 2456 |
+
recovered_database_keys = set()
|
| 2457 |
+
for item in prior:
|
| 2458 |
+
if not isinstance(item, dict):
|
| 2459 |
+
continue
|
| 2460 |
+
key = item.get("key")
|
| 2461 |
+
if (key in database_keys or key in database_failures
|
| 2462 |
+
or item.get("code") != "materialization_failed"
|
| 2463 |
+
or item.get("stage") != "materialization"
|
| 2464 |
+
or item.get("cause") !=
|
| 2465 |
+
"Refused: the database has no readable mirror binding"
|
| 2466 |
+
or not _rel.is_read_through_key(key)):
|
| 2467 |
+
continue
|
| 2468 |
+
try:
|
| 2469 |
+
import routes_odoo_tables as _rot
|
| 2470 |
+
_base, owner = _rel.split_table_key(key)
|
| 2471 |
+
if owner is not None or not _ut.get_definition(key, st=_rt):
|
| 2472 |
+
continue
|
| 2473 |
+
if not any(_key == key and bucket in _rel.enabled_buckets(_rt)
|
| 2474 |
+
for bucket, _key, _label, _fields in _rel.TABLES):
|
| 2475 |
+
continue
|
| 2476 |
+
with _rt.mirror_handle().ro_cursor() as cur:
|
| 2477 |
+
nrows = _rot.population(key, cur=cur, rt=_rt)
|
| 2478 |
+
if isinstance(nrows, int) and nrows >= 0:
|
| 2479 |
+
recovered_database_keys.add(key)
|
| 2480 |
+
except Exception: # noqa: BLE001
|
| 2481 |
+
pass # A missing, filling or unreadable mirror cannot prove recovery.
|
| 2482 |
|
| 2483 |
def _recovered(item):
|
| 2484 |
+
# D-13512: a prior failure leaves once its database ran clean in a later pass.
|
| 2485 |
key = item.get("key")
|
| 2486 |
if key in database_failures:
|
| 2487 |
return False
|
|
|
|
| 2490 |
stage = str(item.get("stage") or "materialization")
|
| 2491 |
row = ((db_health.get(key) or {}).get("stages") or {}).get(stage) or {}
|
| 2492 |
ran, failed_at = _rel._parse_iso(row.get("at")), _rel._parse_iso(item.get("at"))
|
| 2493 |
+
return bool(row.get("state") == "ready" and ran and failed_at and ran > failed_at)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2494 |
|
| 2495 |
kept = [item for item in prior if isinstance(item, dict) and not _recovered(item)]
|
| 2496 |
+
kept = [item for item in kept if isinstance(item, dict)
|
| 2497 |
+
and item.get("key") not in recovered_database_keys]
|
| 2498 |
if failed_databases is not None and table_keys is not None:
|
| 2499 |
# D-2679: a changed-tables pass keeps the failures of databases it did not run.
|
| 2500 |
failed_databases = [
|
|
|
|
| 3304 |
print(f"[sheet-warm] request failed: {type(exc).__name__}: {exc}")
|
| 3305 |
|
| 3306 |
|
| 3307 |
+
def _sync_idle_wait(event, seconds):
|
| 3308 |
+
"""The resync loop's wait. A `sync_worker.py --once` job exits here when it is idle (S82-T10)."""
|
| 3309 |
+
if globals().get("_AMAZON_DESK", {}).get("job") is not None:
|
| 3310 |
+
# An unclaimed or finishing desk turn still owns work, even between receipt states.
|
| 3311 |
+
return event.wait(seconds)
|
| 3312 |
+
import sync_runtime
|
| 3313 |
+
return sync_runtime.idle_wait(event, seconds)
|
| 3314 |
+
|
| 3315 |
+
|
| 3316 |
+
def _amazon_intraday_pass(tenant_key, previous_fingerprint):
|
| 3317 |
+
"""Run due Amazon tables and their projection without advancing the Odoo pass clock."""
|
| 3318 |
+
if globals().get("_AMAZON_DESK", {}).get("job") is not None:
|
| 3319 |
+
return previous_fingerprint
|
| 3320 |
+
try:
|
| 3321 |
+
from harness import amazon_store, runtime
|
| 3322 |
+
rt = runtime.get_runtime(tenant_key)
|
| 3323 |
+
if not amazon_store.credential(tenant_key, rt=rt) or amazon_store.first_load_job_live(rt):
|
| 3324 |
+
return previous_fingerprint
|
| 3325 |
+
plan = amazon_store.scheduled_sync_plan(tenant_key, rt=rt)
|
| 3326 |
+
if not plan["tables"]:
|
| 3327 |
+
return previous_fingerprint
|
| 3328 |
+
print("[aios-api] amazon intraday pass due: " + ", ".join(plan["tables"]), flush=True)
|
| 3329 |
+
RESYNC_STATE["slotPass"] = False
|
| 3330 |
+
_pull_amazon("intraday")
|
| 3331 |
+
state = _rebuild_amazon_relational("intraday", rt, previous_fingerprint)
|
| 3332 |
+
return state["fingerprint"] if state.get("known") else None
|
| 3333 |
+
except Exception as exc:
|
| 3334 |
+
print(f"[aios-api] amazon intraday pass failed ({type(exc).__name__}); "
|
| 3335 |
+
"the next cadence check retries it", flush=True)
|
| 3336 |
+
return None
|
| 3337 |
+
|
| 3338 |
+
|
| 3339 |
+
def _amazon_desk_pass(tenant, *, project=True):
|
| 3340 |
+
"""Run one Amazon desk claim off the slot clock and collect its projection after publication."""
|
| 3341 |
+
import process_role
|
| 3342 |
+
import time
|
| 3343 |
+
if not process_role.may_write():
|
| 3344 |
+
return False
|
| 3345 |
+
from core import connector_sync
|
| 3346 |
+
state = globals().setdefault("_AMAZON_DESK", {"managed": True, "job": None, "outcome": {}})
|
| 3347 |
+
job = state["job"]
|
| 3348 |
+
if job is not None:
|
| 3349 |
+
if job.is_alive() or not project:
|
| 3350 |
+
return True
|
| 3351 |
+
job.join()
|
| 3352 |
+
state["job"] = None
|
| 3353 |
+
_project_desk(state["outcome"], tenant)
|
| 3354 |
+
receipt = connector_sync.sync_request("amazon_sp", tenant)
|
| 3355 |
+
if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
|
| 3356 |
+
or max(receipt.get("notBefore") or 0,
|
| 3357 |
+
connector_sync.sync_not_before("amazon_sp", tenant) or 0) > time.time()):
|
| 3358 |
+
return False
|
| 3359 |
+
outcome = state["outcome"] = {}
|
| 3360 |
+
|
| 3361 |
+
def drain():
|
| 3362 |
+
try:
|
| 3363 |
+
_run_sync_requests(outcome, hold_first_load=True, amazon_only=True)
|
| 3364 |
+
except Exception as exc:
|
| 3365 |
+
print(f"[aios-api] amazon desk turn failed ({type(exc).__name__})", flush=True)
|
| 3366 |
+
finally:
|
| 3367 |
+
connector_sync.sync_wake_event().set()
|
| 3368 |
+
|
| 3369 |
+
# This turn inherits store-resync's heavy admission, including tenant-worker scope.
|
| 3370 |
+
state["job"] = process_role.start("store-resync", "heavy", drain)
|
| 3371 |
+
return state["job"] is not None
|
| 3372 |
+
|
| 3373 |
+
|
| 3374 |
def _store_resync_loop():
|
| 3375 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 3376 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 3377 |
refreshed pool re-reads the freshly synced store."""
|
| 3378 |
+
import process_role as _loop_role
|
| 3379 |
+
if not _loop_role.may_write() or _freeze_holds("store resync"):
|
| 3380 |
return
|
| 3381 |
import time as _t
|
| 3382 |
from core import connector_sync
|
|
|
|
| 3412 |
last_full_pass_at = _t.time()
|
| 3413 |
# The least time between two desk-only passes of an Amazon first-load continuation.
|
| 3414 |
desk_min_spacing_seconds = 30.0
|
| 3415 |
+
amazon_check_seconds = 60.0
|
| 3416 |
# H73: the start of the last daily-slot pass, and whether the boot restore already ran.
|
| 3417 |
last_slot_pass_at = _memory.get("slotPassAt")
|
| 3418 |
slot_restored = False
|
| 3419 |
# H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
|
| 3420 |
slot_resume_checked = False
|
| 3421 |
_slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
|
| 3422 |
+
_idle_wait = globals().get("_sync_idle_wait") or (lambda event, seconds: event.wait(seconds))
|
| 3423 |
_slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
|
| 3424 |
_slot_interrupted_clear = globals().get("_resync_slot_interrupted_clear") or (lambda *_a, **_k: False)
|
| 3425 |
+
# Column fills resume only before this slot's load window ends.
|
| 3426 |
_backfill_read = globals().get("_odoo_backfill_checkpoints")
|
| 3427 |
backfill_next_at = None
|
| 3428 |
backfill_continuations = 0
|
| 3429 |
backfill_spacing_seconds = 60.0
|
| 3430 |
+
backfill_cap = None
|
| 3431 |
while True:
|
| 3432 |
+
if backfill_next_at is not None and backfill_cap is not None and _t.time() >= backfill_cap:
|
| 3433 |
+
backfill_next_at = None
|
| 3434 |
# ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
| 3435 |
#
|
| 3436 |
# ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 —
|
|
|
|
| 3503 |
_slot = (globals().get("_resolve_resync_slot") or (lambda: None))()
|
| 3504 |
|
| 3505 |
def _near_slot():
|
| 3506 |
+
# Completed desk projections wait inside the existing first-load slot quiet window.
|
| 3507 |
if _slot is None:
|
| 3508 |
return False
|
| 3509 |
import odoo_relational as _rel_near
|
|
|
|
| 3516 |
# full pass too, so passes start a day apart however long the pass ran (a daily pass
|
| 3517 |
# that also refreshes every Amazon table can run an hour, and a wait counted from its
|
| 3518 |
# END drifted later every day and past the 26-hour stale line).
|
| 3519 |
+
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3520 |
max(0.0, _every - (_t.time() - last_full_pass_at)),
|
| 3521 |
+
amazon_check_seconds,
|
| 3522 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
|
| 3523 |
connector_sync.sync_wake_event().clear()
|
| 3524 |
_slot_due = False
|
|
|
|
| 3529 |
if passes or slot_restored:
|
| 3530 |
# Sleep to one second past the next slot; Sync now still interrupts the wait.
|
| 3531 |
_slot_now = _t.time()
|
| 3532 |
+
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
|
| 3533 |
0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
|
| 3534 |
else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
|
| 3535 |
+
amazon_check_seconds,
|
| 3536 |
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
|
| 3537 |
connector_sync.sync_wake_event().clear()
|
| 3538 |
_slot_now = _t.time()
|
|
|
|
| 3565 |
f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
|
| 3566 |
else:
|
| 3567 |
RESYNC_STATE.pop("nextPassAt", None)
|
| 3568 |
+
_cadence_due = not passes or _t.time() - last_full_pass_at >= _every
|
| 3569 |
+
_amazon_desk_busy = (_amazon_desk_pass(_cad["tenant"], project=not (_slot_due or _near_slot()))
|
| 3570 |
+
if _slot is not None or globals().get("_AMAZON_DESK") else False)
|
| 3571 |
+
if (not _slot_due if _slot is not None else not _cadence_due):
|
| 3572 |
+
last_amzn_fingerprint = _amazon_intraday_pass(_cad["tenant"], last_amzn_fingerprint)
|
| 3573 |
# ⛔⛔ HOTFIX 2026-09-29 — A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
|
| 3574 |
# capped syncing at once a day, and a deploy is not a reason to sync again. So the first
|
| 3575 |
# pass after a boot that lands within one cadence of the last scheduled pass only catches
|
|
|
|
| 3590 |
if _woken and (_t.time() - last_full_pass_at < _every if _slot is None else not _slot_due):
|
| 3591 |
_desk = {}
|
| 3592 |
_desk_started = _t.time()
|
| 3593 |
+
if not _run_sync_requests(_desk, hold_first_load=True):
|
| 3594 |
_project_desk(_desk, _cad["tenant"])
|
| 3595 |
if "continuing" in _desk.values():
|
| 3596 |
# A floor between continuations, so a chunk that returns at once cannot spin.
|
|
|
|
| 3601 |
_odoo_requested = None
|
| 3602 |
# H1: a queued backfill pass that is due, unless the slot, the cadence or a Sync now runs.
|
| 3603 |
_continuation = (backfill_next_at is not None and _t.time() >= backfill_next_at - 1.0
|
| 3604 |
+
and backfill_cap is not None and _t.time() < backfill_cap
|
| 3605 |
and _odoo_requested is None and not _slot_due
|
| 3606 |
and (_slot is not None or _t.time() - last_full_pass_at < _every))
|
| 3607 |
if _slot is not None and not _slot_due and _odoo_requested is None and not _continuation:
|
| 3608 |
# H73: outside the slot no Odoo call is made. Serve the stored mirror, drain the desk.
|
| 3609 |
_desk = {}
|
| 3610 |
+
if _run_sync_requests(_desk, hold_first_load=True):
|
| 3611 |
_odoo_requested = True
|
| 3612 |
else:
|
| 3613 |
_project_desk(_desk, _cad["tenant"])
|
| 3614 |
if not slot_restored:
|
| 3615 |
+
if not _amazon_desk_busy:
|
| 3616 |
+
_pull_amazon("boot", False)
|
| 3617 |
# The restored mirror warms the workbooks once; Sheet warm reads only the mirror.
|
| 3618 |
_sheet_warm_after_pass(_cad["tenant"], True, None, True)
|
| 3619 |
slot_restored = True
|
| 3620 |
print(f"[aios-api] store resync: no Odoo pass outside the daily slot; the next one "
|
| 3621 |
f"runs at {RESYNC_STATE['nextPassAt']} ({_slot['at']} {_slot['tz']})")
|
| 3622 |
continue
|
| 3623 |
+
if (_slot is None and not _cadence_due and _odoo_requested is None and not _continuation):
|
| 3624 |
+
continue
|
| 3625 |
passes += 1
|
| 3626 |
backfill_next_at = None
|
| 3627 |
if not _continuation:
|
|
|
|
| 3633 |
# H73: stamp the slot BEFORE any Odoo read. H2: `slotDoneAt` marks its end, any outcome.
|
| 3634 |
last_slot_pass_at = last_full_pass_at
|
| 3635 |
_slot_write(last_full_pass_at)
|
| 3636 |
+
from harness import datastore as _fill_ds
|
| 3637 |
+
if not _continuation:
|
| 3638 |
+
_slot_epoch = (_rel_slot.slot_before(last_full_pass_at, _slot)
|
| 3639 |
+
if _slot is not None else last_full_pass_at)
|
| 3640 |
+
backfill_cap = _slot_epoch + _fill_ds.COL_BACKFILL_WINDOW_SECONDS
|
| 3641 |
+
RESYNC_STATE["slotDate"] = _t.strftime("%Y-%m-%d", _t.gmtime(_slot_epoch))
|
| 3642 |
+
_fill_ds.COL_BACKFILL_STOP_AT = backfill_cap
|
| 3643 |
+
_odoo_requested = bool(_odoo_requested)
|
| 3644 |
_warm_generation = _sheet_warm_generation()
|
| 3645 |
_backfill_before = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3646 |
+
# C-status: the loop's state and its Odoo pass reach the store at each pass start and end.
|
| 3647 |
+
_resync_status = globals().get("_publish_resync_state")
|
| 3648 |
+
if _resync_status:
|
| 3649 |
+
_resync_status(_cad["tenant"], started=True)
|
| 3650 |
try:
|
| 3651 |
selected_entities = _configured_mirror_entities(_cad["tenant"])
|
| 3652 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
|
|
|
|
| 3684 |
print(f"[aios-api] store resync failed ({_err['kind']}, "
|
| 3685 |
f"{RESYNC_STATE['consecutiveFailures']} in a row): "
|
| 3686 |
f"{_err['type']}: {_err['message']}")
|
| 3687 |
+
if _resync_status:
|
| 3688 |
+
_resync_status(_cad["tenant"], ended=True,
|
| 3689 |
+
outcome=("failed" if _err is not None else "waiting" if _manual
|
| 3690 |
+
else "held" if _publication_held else "done"))
|
| 3691 |
if _publication_held:
|
| 3692 |
print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; "
|
| 3693 |
"the next admitted pass resumes the unfinished Odoo cohort")
|
|
|
|
| 3697 |
_backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
|
| 3698 |
_moved = ([row for row in _backfill_after if row not in _backfill_before]
|
| 3699 |
if _err is None and _backfill_after and _backfill_before is not None else [])
|
| 3700 |
+
if _moved and _t.time() + backfill_spacing_seconds >= backfill_cap:
|
| 3701 |
+
print(f"[aios-api] odoo backfill stops for today: the load window ended; "
|
| 3702 |
f"the next slot pass or Sync now resumes it")
|
| 3703 |
elif _moved:
|
| 3704 |
backfill_continuations += 1
|
|
|
|
| 3706 |
_row = max(_moved, key=lambda row: row[4] or 0)
|
| 3707 |
print(f"[aios-api] odoo backfill continues: {str(_row[0]).split('.', 1)[0]} at "
|
| 3708 |
f"{_row[4] or 0:,} rows; next bounded pass queued "
|
| 3709 |
+
f"({backfill_continuations} continuations today)")
|
| 3710 |
continue
|
| 3711 |
if _boot_catch_up and not _odoo_requested:
|
| 3712 |
# Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
|
|
|
|
| 3868 |
except Exception as e: # noqa: BLE001
|
| 3869 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 3870 |
f"{type(e).__name__}: {e}")
|
| 3871 |
+
# The independent desk retains Amazon's pull and projection while Odoo completes its pass.
|
| 3872 |
+
if not _amazon_desk_busy and _resumed.get("amazon"):
|
| 3873 |
_pull_amazon("boot", False)
|
| 3874 |
+
elif not _amazon_desk_busy:
|
| 3875 |
# The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
|
| 3876 |
RESYNC_STATE["slotPass"] = _slot_claimed
|
| 3877 |
try:
|
|
|
|
| 3885 |
try:
|
| 3886 |
from harness import runtime as _runtime
|
| 3887 |
_rt_resync = _runtime.get_runtime(_cad["tenant"])
|
| 3888 |
+
_amzn_state = ({} if _amazon_desk_busy else
|
| 3889 |
+
_rebuild_amazon_relational("resync", _rt_resync, _amzn_previous))
|
| 3890 |
last_amzn_fingerprint = (_amzn_state["fingerprint"]
|
| 3891 |
if _amzn_state.get("known") else None)
|
| 3892 |
except Exception as e: # noqa: BLE001
|
|
|
|
| 3915 |
_slot_write(_t.time(), "slotDoneAt")
|
| 3916 |
|
| 3917 |
|
| 3918 |
+
def _boot_app_migrations(why, tenant=None):
|
| 3919 |
"""Apply tenant App retirement migrations independently of mirror sync and reads."""
|
| 3920 |
if _freeze_holds("app migrations"):
|
| 3921 |
return
|
|
|
|
| 3924 |
import timeseries_retirement
|
| 3925 |
import tenant_scan
|
| 3926 |
from harness import runtime as _runtime
|
| 3927 |
+
tenants = list(tenant_scan.live_tenants(_runtime, tenants=[tenant]) if tenant is not None
|
| 3928 |
+
else tenant_scan.live_tenants(_runtime))
|
| 3929 |
except Exception as exc:
|
| 3930 |
print(f"[aios-api] app migrations ({why}) skipped: {type(exc).__name__}: {exc}")
|
| 3931 |
return
|
|
|
|
| 3942 |
print(f"[aios-api] Returns split ({why}) {slug}: {split['outcome']}")
|
| 3943 |
except Exception as exc:
|
| 3944 |
print(f"[aios-api] Returns split ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
| 3945 |
+
try:
|
| 3946 |
+
result = application_registry.remove_template_doubles(rt, slug)
|
| 3947 |
+
reason = f": {result['reason']}" if result.get("reason") else ""
|
| 3948 |
+
print(f"[aios-api] one template copy ({why}) {slug}: {result['outcome']}{reason}")
|
| 3949 |
+
except Exception as exc:
|
| 3950 |
+
print(f"[aios-api] one template copy ({why}) {slug}: {type(exc).__name__}")
|
| 3951 |
+
try:
|
| 3952 |
+
from modules import amazon_profit as _amazon_profit
|
| 3953 |
+
moved = _amazon_profit.migrate_projects(rt)
|
| 3954 |
+
print(f"[aios-api] Projects move ({why}) {slug}: copied {moved['copied']} of {moved['rows']}")
|
| 3955 |
+
except Exception as exc:
|
| 3956 |
+
print(f"[aios-api] Projects move ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
| 3957 |
+
try:
|
| 3958 |
+
import core.user_tables as _ut
|
| 3959 |
+
from modules import amazon_profit as _amazon_profit
|
| 3960 |
+
import routes_amazon_projects as _projects_routes
|
| 3961 |
+
if _ut.get_one(_amazon_profit.TABLE_KEY, rt) is not None:
|
| 3962 |
+
_amazon_profit.ensure_table(rt, _amazon_profit.WRITER)
|
| 3963 |
+
painted = _projects_routes.paint(rt)
|
| 3964 |
+
print(f"[aios-api] Projects cells ({why}) {slug}: {sum(len(ids) for ids in painted.values())} rows")
|
| 3965 |
+
except Exception as exc:
|
| 3966 |
+
print(f"[aios-api] Projects cells ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
|
| 3967 |
try:
|
| 3968 |
result = application_registry.ensure_route_application(rt, slug)
|
| 3969 |
reason = f": {result['reason']}" if result.get("reason") else ""
|
|
|
|
| 3981 |
print(f"[aios-api] Command Center App ({why}) {slug}: skipped: {type(exc).__name__}")
|
| 3982 |
|
| 3983 |
|
| 3984 |
+
def _rows_layouts():
|
| 3985 |
+
"""The rows layouts the boot step migrates, each `(name, ready(rt, slug), migrate(rt, slug))`.
|
| 3986 |
+
|
| 3987 |
+
Every module marker reader needs an entry here: verify_account_rows fails a new one without it.
|
| 3988 |
+
"""
|
| 3989 |
+
from core import account_rows, history_rows, sheets
|
| 3990 |
+
|
| 3991 |
+
def users_ready(_rt, slug):
|
| 3992 |
+
state = account_rows.verify(slug)
|
| 3993 |
+
return state["mode"] == "rows" or not state["records"]
|
| 3994 |
+
|
| 3995 |
+
def history_migrate(_rt, slug):
|
| 3996 |
+
# R58: history_rows refuses a production store unless it is isolated staging. Never widen it.
|
| 3997 |
+
result = history_rows.migrate(slug)
|
| 3998 |
+
return dict(result, records=(result.get("copied") or {}).get("events", 0))
|
| 3999 |
+
|
| 4000 |
+
return [*[(bucket,
|
| 4001 |
+
lambda _rt, slug, b=bucket: account_rows.record_rows_ready(slug, b),
|
| 4002 |
+
lambda _rt, slug, b=bucket: account_rows.records_action("migrate", slug, b))
|
| 4003 |
+
for bucket in account_rows.RECORD_BUCKETS],
|
| 4004 |
+
("users", users_ready, lambda _rt, slug: account_rows.migrate(slug)),
|
| 4005 |
+
("value_history", lambda _rt, slug: history_rows.rows_ready(slug), history_migrate),
|
| 4006 |
+
# Bounded per boot by sheets.BOOT_MIGRATION_LIMIT and BOOT_MIGRATION_SECONDS.
|
| 4007 |
+
("workbooks", lambda _rt, slug: sheets.rows_ready(slug),
|
| 4008 |
+
lambda _rt, slug: sheets.migrate_rows_batch(slug))]
|
| 4009 |
+
|
| 4010 |
+
|
| 4011 |
+
def _boot_rows_migrations(why):
|
| 4012 |
+
"""Migrate each live tenant's rows layouts before their writers run. A migrated layout is a no-op."""
|
| 4013 |
+
if _freeze_holds("rows migrations"):
|
| 4014 |
+
return
|
| 4015 |
+
import sync_alerts
|
| 4016 |
+
|
| 4017 |
+
def report(tenant, name, outcome):
|
| 4018 |
+
print(f"[aios-api] rows migration ({why}) {tenant} {name}: {outcome}")
|
| 4019 |
+
|
| 4020 |
+
def failure(exc):
|
| 4021 |
+
# Scrubbed, so a connection error never prints a credential.
|
| 4022 |
+
return f"failed: {type(exc).__name__}: {sync_alerts.scrub(exc)}"
|
| 4023 |
+
|
| 4024 |
+
def migrated(result):
|
| 4025 |
+
# A bounded layout names, by count, what it left for the next boot and what it could not copy.
|
| 4026 |
+
extra = [f"remaining={result['remaining']} for the next boot"] if result.get("remaining") else []
|
| 4027 |
+
extra += [f"failed {code}={count}" for code, count in sorted((result.get("failed") or {}).items())]
|
| 4028 |
+
if not (result.get("changed") or extra):
|
| 4029 |
+
return "already in rows mode"
|
| 4030 |
+
verb = "migrated" if result.get("changed") else "not migrated"
|
| 4031 |
+
return f"{verb} (" + ", ".join([f"records={result.get('records', 0)}", *extra]) + ")"
|
| 4032 |
+
|
| 4033 |
+
try:
|
| 4034 |
+
import tenant_scan
|
| 4035 |
+
from core import account_rows
|
| 4036 |
+
from harness import runtime as _runtime
|
| 4037 |
+
layouts = _rows_layouts()
|
| 4038 |
+
unresolved = []
|
| 4039 |
+
tenants = list(tenant_scan.live_tenants(
|
| 4040 |
+
_runtime, name="rows-migrations", on_error=lambda slug, exc: unresolved.append((slug, exc))))
|
| 4041 |
+
except Exception as exc:
|
| 4042 |
+
print(f"[aios-api] rows migrations ({why}) skipped: {failure(exc)}")
|
| 4043 |
+
return
|
| 4044 |
+
for tenant, exc in unresolved:
|
| 4045 |
+
for name, _ready, _migrate in layouts:
|
| 4046 |
+
report(tenant, name, failure(exc))
|
| 4047 |
+
for tenant, rt in tenants:
|
| 4048 |
+
try:
|
| 4049 |
+
backend = account_rows.records_backend(rt)
|
| 4050 |
+
except Exception as exc:
|
| 4051 |
+
for name, _ready, _migrate in layouts:
|
| 4052 |
+
report(tenant, name, failure(exc))
|
| 4053 |
+
continue
|
| 4054 |
+
for name, ready, migrate in layouts:
|
| 4055 |
+
# One tenant or layout failing never stops the others.
|
| 4056 |
+
try:
|
| 4057 |
+
if backend is None:
|
| 4058 |
+
report(tenant, name, "skipped, not a PostgreSQL rows store")
|
| 4059 |
+
elif ready(rt, backend[1]):
|
| 4060 |
+
report(tenant, name, "already in rows mode")
|
| 4061 |
+
else:
|
| 4062 |
+
report(tenant, name, migrated(migrate(rt, backend[1])))
|
| 4063 |
+
except Exception as exc:
|
| 4064 |
+
report(tenant, name, failure(exc))
|
| 4065 |
+
|
| 4066 |
+
|
| 4067 |
+
def _boot_rows_before_writers():
|
| 4068 |
+
"""Run the rows migration synchronously at import, before this web or worker process starts a writer."""
|
| 4069 |
+
flag = os.environ.get("AIOS_BOOT_ROWS_MIGRATIONS")
|
| 4070 |
+
if data_binding.gate_process() or flag == "0" or not (os.environ.get("SPACE_ID") or flag == "1"):
|
| 4071 |
+
return
|
| 4072 |
+
_boot_rows_migrations("boot")
|
| 4073 |
+
|
| 4074 |
+
|
| 4075 |
def _prewarm():
|
| 4076 |
if _freeze_holds("prewarm"):
|
| 4077 |
return
|
|
|
|
| 4144 |
from harness import runtime as _runtime
|
| 4145 |
MIRROR_SEED["attempted"] = True
|
| 4146 |
try:
|
| 4147 |
+
import tenant_loops
|
| 4148 |
+
import tenant_scan
|
| 4149 |
+
|
| 4150 |
+
def refused(tenant_key, exc):
|
| 4151 |
+
result = {"state": "refused", "problems": [
|
| 4152 |
+
f"Odoo mirror restore failed for {tenant_key} ({type(exc).__name__})"]}
|
| 4153 |
+
_ds.ODOO_MIRROR_STATUS[str(_ds.path_for(tenant_key).resolve())] = result
|
| 4154 |
+
return result
|
| 4155 |
+
|
| 4156 |
+
def restore(tenant_key, rt):
|
| 4157 |
try:
|
|
|
|
| 4158 |
result = _ds.restore_odoo_mirror(tenant_key, rt)
|
| 4159 |
except Exception as exc:
|
| 4160 |
+
result = refused(tenant_key, exc)
|
|
|
|
|
|
|
| 4161 |
if result["state"] == "refused":
|
| 4162 |
MIRROR_SEED["cause"] = "; ".join(result["problems"])
|
| 4163 |
MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
|
| 4164 |
+
|
| 4165 |
+
def runtime_failed(tenant_key, exc):
|
| 4166 |
+
MIRROR_SEED["cause"] = "; ".join(refused(tenant_key, exc)["problems"])
|
| 4167 |
+
MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
|
| 4168 |
+
|
| 4169 |
+
# S82-T18: each tenant restores in its own job; an inactive one is skipped with one named line.
|
| 4170 |
+
jobs = tenant_loops.for_each_tenant(
|
| 4171 |
+
restore, name="odoo-mirror-restore", tenants=_runtime.known_tenants(),
|
| 4172 |
+
get_runtime=tenant_scan.runtime_lookup(_runtime, "odoo-mirror-restore",
|
| 4173 |
+
on_error=runtime_failed), kind="light")
|
| 4174 |
+
for job in jobs.values():
|
| 4175 |
+
job.join() # The legacy seed below decides on the restored manifests.
|
| 4176 |
if _ds.ensure_seed():
|
| 4177 |
MIRROR_SEED["seeded"] = True
|
| 4178 |
print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
|
|
|
|
| 4276 |
thread.start()
|
| 4277 |
|
| 4278 |
|
| 4279 |
+
@app.on_event("startup")
|
| 4280 |
+
def _reconcile_project_management_defaults():
|
| 4281 |
+
"""Reconcile bounded authored metadata at startup before web requests can read legacy defaults."""
|
| 4282 |
+
if (data_binding.gate_process() or process_role.role() not in ("web", "all")
|
| 4283 |
+
or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "0"
|
| 4284 |
+
or not (os.environ.get("SPACE_ID") or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
| 4285 |
+
or _freeze_holds("Project management defaults")):
|
| 4286 |
+
return
|
| 4287 |
+
import application_registry
|
| 4288 |
+
import tenant_scan
|
| 4289 |
+
from harness import runtime
|
| 4290 |
+
# This startup barrier changes metadata only. Heavy migrations keep their worker owner.
|
| 4291 |
+
for slug, rt in tenant_scan.live_tenants(runtime):
|
| 4292 |
+
try:
|
| 4293 |
+
result = application_registry.reconcile_project_management_defaults(rt, slug)
|
| 4294 |
+
print(f"[aios-api] Project management defaults {slug}: "
|
| 4295 |
+
f"names={result['renamed']} fields={result['reordered']}")
|
| 4296 |
+
except Exception as exc:
|
| 4297 |
+
print(f"[aios-api] Project management defaults {slug}: {type(exc).__name__}")
|
| 4298 |
+
|
| 4299 |
+
|
| 4300 |
@app.on_event("startup")
|
| 4301 |
def _start_process_role():
|
| 4302 |
+
"""Web role: tick the sync runtime, forward Sync now, install the worker's snapshots."""
|
| 4303 |
if data_binding.gate_process():
|
| 4304 |
return
|
| 4305 |
import process_role as _process_role
|
|
|
|
| 4312 |
import process_role as _process_role
|
| 4313 |
# Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
|
| 4314 |
requeue = _process_role.may_write()
|
| 4315 |
+
import tenant_loops
|
| 4316 |
+
import tenant_scan
|
| 4317 |
+
|
| 4318 |
+
def restore(tenant_key, rt):
|
| 4319 |
try:
|
| 4320 |
+
connector_sync.restore_sync_requests(rt, requeue=requeue)
|
| 4321 |
except Exception as exc:
|
| 4322 |
print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
|
| 4323 |
|
| 4324 |
+
# S82-T18: an inactive tenant is skipped with one named line; one failure leaves the others restored.
|
| 4325 |
+
jobs = tenant_loops.for_each_tenant(
|
| 4326 |
+
restore, name="sync-request-restore", tenants=runtime.known_tenants(),
|
| 4327 |
+
get_runtime=tenant_scan.runtime_lookup(runtime, "sync-request-restore"), kind="light")
|
| 4328 |
+
for job in jobs.values():
|
| 4329 |
+
job.join() # The desks are restored before the boot goes on.
|
| 4330 |
+
|
| 4331 |
|
| 4332 |
def _boot_mirrors():
|
| 4333 |
import process_role as _role_boot
|
|
|
|
| 4365 |
_READ_WARM_RETRY_SECONDS = 15
|
| 4366 |
|
| 4367 |
|
| 4368 |
+
def _read_pool_state(rt):
|
| 4369 |
+
"""The D-12159 settling line: which connection the read warm holds, and whether it is shared."""
|
| 4370 |
+
from harness import datastore as _state_ds
|
| 4371 |
+
try:
|
| 4372 |
+
handle = rt.mirror_handle() if callable(getattr(rt, "mirror_handle", None)) else None
|
| 4373 |
+
shared = _state_ds._INSTANCE.get("con")
|
| 4374 |
+
return (f"instance={id(shared) if shared is not None else None} "
|
| 4375 |
+
f"handle_con_none={None if handle is None else handle._con is None} "
|
| 4376 |
+
f"handle_path_is_db_path={None if handle is None else handle.path == _state_ds.DB_PATH.resolve()}")
|
| 4377 |
+
except Exception as exc: # noqa: BLE001
|
| 4378 |
+
return f"unreadable ({type(exc).__name__})"
|
| 4379 |
+
|
| 4380 |
+
|
| 4381 |
+
def _connection_closed(con):
|
| 4382 |
+
try:
|
| 4383 |
+
con.cursor().close()
|
| 4384 |
+
return False
|
| 4385 |
+
except Exception: # noqa: BLE001
|
| 4386 |
+
return True
|
| 4387 |
+
|
| 4388 |
+
|
| 4389 |
+
def _reopen_read_pool(rt):
|
| 4390 |
+
"""Drop this tenant's closed mirror read connection so the next cursor opens a new one. True if dropped."""
|
| 4391 |
+
from harness import datastore as _reopen_ds
|
| 4392 |
+
try:
|
| 4393 |
+
handle = rt.mirror_handle() if callable(getattr(rt, "mirror_handle", None)) else None
|
| 4394 |
+
if handle is None or handle.path == _reopen_ds.DB_PATH.resolve():
|
| 4395 |
+
with _reopen_ds._INSTANCE_LOCK:
|
| 4396 |
+
con = _reopen_ds._INSTANCE.get("con")
|
| 4397 |
+
if con is None or not _connection_closed(con):
|
| 4398 |
+
return False
|
| 4399 |
+
_reopen_ds._INSTANCE["con"] = None
|
| 4400 |
+
_reopen_ds._close_instance(con)
|
| 4401 |
+
else:
|
| 4402 |
+
with _reopen_ds._HANDLES_LOCK:
|
| 4403 |
+
con = handle._con
|
| 4404 |
+
if con is None or not _connection_closed(con):
|
| 4405 |
+
return False
|
| 4406 |
+
handle._con = None
|
| 4407 |
+
_reopen_ds._close_instance(con)
|
| 4408 |
+
except Exception as exc: # noqa: BLE001
|
| 4409 |
+
print(f"[aios-api] read pool re-open refused ({type(exc).__name__})")
|
| 4410 |
+
return False
|
| 4411 |
+
print("[aios-api] read pool connection was closed; the next read opens a new one")
|
| 4412 |
+
return True
|
| 4413 |
+
|
| 4414 |
+
|
| 4415 |
def _probe_read_pools(rt):
|
| 4416 |
"""Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
|
| 4417 |
from harness import datastore as _probe_ds
|
|
|
|
| 4420 |
physical = _probe_ds.serving_file(path)[0] if _probe_ds.versioned_mode() else path
|
| 4421 |
if not physical.exists():
|
| 4422 |
return "The mirror file is missing."
|
| 4423 |
+
for retry in (False, True):
|
| 4424 |
+
try:
|
| 4425 |
+
cur = handle.ro_cursor() if handle is not None else _probe_ds.ro_cursor()
|
| 4426 |
+
try:
|
| 4427 |
+
cur.execute("SELECT 1").fetchone()
|
| 4428 |
+
finally:
|
| 4429 |
+
cur.close()
|
| 4430 |
+
break
|
| 4431 |
+
except Exception: # noqa: BLE001
|
| 4432 |
+
# D-12159: a closed pool connection stays closed until something drops it.
|
| 4433 |
+
if retry or not _reopen_read_pool(rt):
|
| 4434 |
+
raise
|
| 4435 |
if rt.available() is False:
|
| 4436 |
return "The tenant store did not answer."
|
| 4437 |
return ""
|
|
|
|
| 4446 |
import process_role as _warm_role
|
| 4447 |
attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
|
| 4448 |
retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
|
| 4449 |
+
rt, diagnosed = None, False
|
| 4450 |
for attempt in range(1, attempts + 1):
|
| 4451 |
t0 = _t.time()
|
| 4452 |
stage = "pools"
|
|
|
|
| 4459 |
raise RuntimeError(problem)
|
| 4460 |
_warm_role.mark_ready("pools", True)
|
| 4461 |
stage = "first_reads"
|
| 4462 |
+
for retry in (False, True):
|
| 4463 |
+
try:
|
| 4464 |
+
routes_customers.warm_default(rt)
|
| 4465 |
+
routes_products._pool_for(rt, None)
|
| 4466 |
+
break
|
| 4467 |
+
except Exception: # noqa: BLE001
|
| 4468 |
+
# D-12159: a pool connection closed mid-warm is dropped, then the reads run once more.
|
| 4469 |
+
if retry or not _reopen_read_pool(rt):
|
| 4470 |
+
raise
|
| 4471 |
_warm_role.mark_ready("first_reads", True)
|
| 4472 |
routes_customers.warm_bu_pools(rt)
|
| 4473 |
print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
|
| 4474 |
return True
|
| 4475 |
except Exception as exc: # noqa: BLE001
|
| 4476 |
+
if not diagnosed and rt is not None:
|
| 4477 |
+
diagnosed = True
|
| 4478 |
+
print(f"[aios-api] read warm pool state at the first failure: {_read_pool_state(rt)}")
|
| 4479 |
if not _warm_role._READINESS.get(stage, {}).get("ok"):
|
| 4480 |
_warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
|
| 4481 |
f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
|
|
|
|
| 4490 |
"""Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
|
| 4491 |
from core import connector_spec
|
| 4492 |
from harness import mirror_store, runtime
|
| 4493 |
+
import tenant_loops
|
| 4494 |
+
import tenant_scan
|
| 4495 |
errors = RESYNC_STATE.setdefault("mirrorRestoreErrors", {})
|
| 4496 |
+
|
| 4497 |
+
def restore(tenant_key, rt):
|
| 4498 |
for connector in tuple(connector_spec.MIRRORS):
|
| 4499 |
if connector in {"odoo", "amazon"}:
|
| 4500 |
continue # Their existing adapters also enforce seed and deletion policies.
|
| 4501 |
identity = f"{tenant_key}/{connector}"
|
| 4502 |
try:
|
|
|
|
| 4503 |
result = mirror_store.restore_mirror(connector, tenant_key, rt)
|
| 4504 |
if result["problems"]:
|
| 4505 |
errors[identity] = {"state": result["state"], "problems": list(result["problems"])}
|
|
|
|
| 4508 |
print(f"[aios-api] {connector} mirror {tenant_key} {result['state']} ({why})")
|
| 4509 |
except Exception as exc:
|
| 4510 |
errors[identity] = {"state": "refused", "problems": [type(exc).__name__]}
|
| 4511 |
+
|
| 4512 |
+
def runtime_failed(tenant_key, exc):
|
| 4513 |
+
for connector in tuple(connector_spec.MIRRORS):
|
| 4514 |
+
if connector not in {"odoo", "amazon"}:
|
| 4515 |
+
errors[f"{tenant_key}/{connector}"] = {"state": "refused", "problems": [type(exc).__name__]}
|
| 4516 |
+
|
| 4517 |
+
# S82-T18: each tenant restores in its own job; an inactive one is skipped with one named line.
|
| 4518 |
+
jobs = tenant_loops.for_each_tenant(
|
| 4519 |
+
restore, name="declared-mirror-restore", tenants=runtime.known_tenants(),
|
| 4520 |
+
get_runtime=tenant_scan.runtime_lookup(runtime, "declared-mirror-restore",
|
| 4521 |
+
on_error=runtime_failed), kind="light")
|
| 4522 |
+
for job in jobs.values():
|
| 4523 |
+
job.join() # The caller reads the errors of a finished restore.
|
| 4524 |
return not errors
|
| 4525 |
|
| 4526 |
|
| 4527 |
+
# D-12554: the worker imports main and runs no startup hook, so the migration runs here, before any thread.
|
| 4528 |
+
_boot_rows_before_writers()
|
| 4529 |
+
|
| 4530 |
try:
|
| 4531 |
from harness import datastore as _ds_boot
|
| 4532 |
if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
|
|
|
|
| 4550 |
name="qa-sandbox-schema").start()
|
| 4551 |
|
| 4552 |
if (not data_binding.gate_process()
|
| 4553 |
+
and process_role.role() != "worker"
|
| 4554 |
and os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 4555 |
and (bool(os.environ.get("SPACE_ID")) or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
| 4556 |
and not _freeze_holds("app migrations")):
|
|
|
|
| 4564 |
if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
|
| 4565 |
import marketing_stream_loop
|
| 4566 |
import ads_schedule_loop
|
| 4567 |
+
import amazon_quotes_loop
|
| 4568 |
+
from harness import amazon_store
|
| 4569 |
import routes_returns_refunds
|
| 4570 |
marketing_stream_loop.start()
|
| 4571 |
ads_schedule_loop.start()
|
| 4572 |
+
amazon_quotes_loop.start()
|
| 4573 |
+
process_role.start("amazon-after-sync", "heavy", amazon_store.run_after_sync_loop)
|
| 4574 |
routes_returns_refunds.start_auto_approve_worker()
|
| 4575 |
|
| 4576 |
|
|
|
|
| 4578 |
def _stop_amazon_app_workers():
|
| 4579 |
import marketing_stream_loop
|
| 4580 |
import ads_schedule_loop
|
| 4581 |
+
import amazon_quotes_loop
|
| 4582 |
marketing_stream_loop.stop()
|
| 4583 |
ads_schedule_loop.stop()
|
| 4584 |
+
amazon_quotes_loop.stop()
|
| 4585 |
|
| 4586 |
|
| 4587 |
if (not data_binding.gate_process()
|
|
|
|
| 4621 |
# The other structural cost remains outside that skip policy:
|
| 4622 |
# * the whole-document store write itself. A partial-JSONB write is a wave.
|
| 4623 |
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 4624 |
+
def _start_store_resync():
|
| 4625 |
+
# The worker calls this after its boot migrations finish: a per-pass idle exit must wait.
|
| 4626 |
+
if (not data_binding.gate_process()
|
| 4627 |
+
and os.environ.get("AIOS_STORE_RESYNC") == "1" and not _freeze_holds("store resync")):
|
| 4628 |
+
import process_role as _role_resync
|
| 4629 |
+
_role_resync.thread(target=_store_resync_loop, daemon=True, kind="heavy", name="store-resync").start()
|
| 4630 |
+
import sheet_warm as _sheet_warm
|
| 4631 |
+
_sheet_warm.start()
|
| 4632 |
+
|
| 4633 |
+
|
| 4634 |
+
if process_role.role() != "worker":
|
| 4635 |
+
_start_store_resync()
|
api/notify_destinations.py
CHANGED
|
@@ -1,12 +1,14 @@
|
|
| 1 |
"""Authenticated preference routes and destination-bound notification delivery."""
|
| 2 |
|
| 3 |
import re
|
|
|
|
| 4 |
|
| 5 |
from fastapi import APIRouter, Body, Depends, Query
|
| 6 |
|
| 7 |
from core import notify_destinations as rows
|
| 8 |
from deps import Session, err, require_session
|
| 9 |
import email_send
|
|
|
|
| 10 |
|
| 11 |
NotificationItem = rows.NotificationItem
|
| 12 |
NotificationPayload = rows.NotificationPayload
|
|
@@ -26,6 +28,12 @@ _REASONS = {
|
|
| 26 |
'notification_source_required': 'This notification needs source records so the Slack agent can check its read access.',
|
| 27 |
'transport_error': 'Slack delivery was not confirmed. Check Slack before retrying.',
|
| 28 |
'slack_readiness_unavailable': 'Slack readiness could not be checked. Check its connection and permissions.',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
}
|
| 30 |
|
| 31 |
|
|
@@ -43,6 +51,56 @@ def _own_session(session):
|
|
| 43 |
raise err(401, 'notification_user_required', 'Sign in to edit notification destinations.')
|
| 44 |
|
| 45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
@router.get('/destinations')
|
| 47 |
def get_destinations(session: Session = Depends(require_session)):
|
| 48 |
_own_session(session)
|
|
@@ -51,6 +109,7 @@ def get_destinations(session: Session = Depends(require_session)):
|
|
| 51 |
except Exception:
|
| 52 |
raise err(503, 'notification_store_unavailable', 'Notification destinations are unavailable for this workspace.')
|
| 53 |
mail_ready = email_send.availability(session.runtime)
|
|
|
|
| 54 |
projected = []
|
| 55 |
for preference in destinations:
|
| 56 |
readiness = {
|
|
@@ -60,8 +119,14 @@ def get_destinations(session: Session = Depends(require_session)):
|
|
| 60 |
}
|
| 61 |
if preference['email']:
|
| 62 |
readiness['email:' + preference['email']] = dict(mail_ready)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
projected.append({**preference, 'label': rows.KIND_LABELS[preference['kind']], 'readiness': readiness})
|
| 64 |
-
return {'destinations': projected, 'email': mail_ready
|
|
|
|
|
|
|
| 65 |
|
| 66 |
|
| 67 |
def _save(kind, body, session, *, replace):
|
|
@@ -70,9 +135,13 @@ def _save(kind, body, session, *, replace):
|
|
| 70 |
rows.kind_id(kind)
|
| 71 |
patch = rows.clean_patch(body, replace=replace)
|
| 72 |
except (ValueError, TypeError):
|
| 73 |
-
raise err(400, 'notification_destination_invalid', 'Choose a known notification kind
|
|
|
|
|
|
|
| 74 |
try:
|
| 75 |
return rows.edit(session.runtime, session.uname, kind, patch)
|
|
|
|
|
|
|
| 76 |
except Exception:
|
| 77 |
raise err(503, 'notification_store_unavailable', 'Notification destinations could not be saved for this workspace.')
|
| 78 |
|
|
@@ -184,6 +253,36 @@ def _dispatch_slack(rt, preference, channel, kind, items, *, dispatcher=None):
|
|
| 184 |
return {'state': 'sent', 'reason': None, 'message': ''}
|
| 185 |
|
| 186 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None):
|
| 188 |
"""Deliver trusted recipient-safe items to saved picks, optionally targeting one user."""
|
| 189 |
kind, items = rows.kind_id(kind), rows.clean_items(items)
|
|
@@ -195,6 +294,9 @@ def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None)
|
|
| 195 |
destinations = ['slack:' + channel for channel in preference['slackChannels']]
|
| 196 |
if preference['email']:
|
| 197 |
destinations.append('email:' + preference['email'])
|
|
|
|
|
|
|
|
|
|
| 198 |
for destination in destinations:
|
| 199 |
current = rows.get(rt, preference['user'], kind)
|
| 200 |
if not rows.selected(current, destination):
|
|
@@ -202,12 +304,18 @@ def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None)
|
|
| 202 |
try:
|
| 203 |
if destination.startswith('slack:'):
|
| 204 |
outcome = _dispatch_slack(rt, current, destination[6:], kind, items, dispatcher=dispatcher)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
else:
|
| 206 |
result = email_send.send_notification(rt, destination[6:], kind, items, transport=mail_transport)
|
| 207 |
outcome = ({'state': 'sent', 'reason': None, 'message': ''} if result.get('sent') is True
|
| 208 |
else _refused(result.get('reason'), message=result.get('message')))
|
| 209 |
except Exception:
|
| 210 |
-
|
|
|
|
|
|
|
| 211 |
# Persistence failures propagate. An unrecorded refusal must never look successful.
|
| 212 |
recorded = rows.record_delivery(rt, preference['user'], kind, destination, outcome)
|
| 213 |
results.append({'user': preference['user'], 'destination': destination,
|
|
|
|
| 1 |
"""Authenticated preference routes and destination-bound notification delivery."""
|
| 2 |
|
| 3 |
import re
|
| 4 |
+
from datetime import datetime, timezone
|
| 5 |
|
| 6 |
from fastapi import APIRouter, Body, Depends, Query
|
| 7 |
|
| 8 |
from core import notify_destinations as rows
|
| 9 |
from deps import Session, err, require_session
|
| 10 |
import email_send
|
| 11 |
+
import dash_norm
|
| 12 |
|
| 13 |
NotificationItem = rows.NotificationItem
|
| 14 |
NotificationPayload = rows.NotificationPayload
|
|
|
|
| 28 |
'notification_source_required': 'This notification needs source records so the Slack agent can check its read access.',
|
| 29 |
'transport_error': 'Slack delivery was not confirmed. Check Slack before retrying.',
|
| 30 |
'slack_readiness_unavailable': 'Slack readiness could not be checked. Check its connection and permissions.',
|
| 31 |
+
'in_app_unavailable': 'In-app notifications are unavailable for this workspace.',
|
| 32 |
+
'in_app_delivery_unconfirmed': 'The in-app notification could not be confirmed.',
|
| 33 |
+
'webhook_not_configured': 'Choose one of your saved alert webhooks in connector settings.',
|
| 34 |
+
'webhook_readiness_unavailable': 'Webhook readiness could not be checked. Check its saved configuration.',
|
| 35 |
+
'webhook_delivery_unconfirmed': 'Webhook delivery was not confirmed. Check the destination before retrying.',
|
| 36 |
+
'webhook_refused': 'The configured webhook refused this notification.',
|
| 37 |
}
|
| 38 |
|
| 39 |
|
|
|
|
| 51 |
raise err(401, 'notification_user_required', 'Sign in to edit notification destinations.')
|
| 52 |
|
| 53 |
|
| 54 |
+
def _in_app_readiness(rt):
|
| 55 |
+
try:
|
| 56 |
+
import assistant_log
|
| 57 |
+
ready = (callable(getattr(assistant_log, 'append', None))
|
| 58 |
+
and callable(getattr(rt, 'get', None)) and callable(getattr(rt, 'update', None)))
|
| 59 |
+
available = getattr(rt, 'available', None)
|
| 60 |
+
if callable(available):
|
| 61 |
+
ready = ready and bool(available())
|
| 62 |
+
except Exception:
|
| 63 |
+
ready = False
|
| 64 |
+
reason = None if ready else 'in_app_unavailable'
|
| 65 |
+
return {'configured': ready, 'available': ready, 'reason': reason,
|
| 66 |
+
'message': '' if ready else _REASONS[reason]}
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _webhook_readiness(rt, user, identity):
|
| 70 |
+
try:
|
| 71 |
+
rows.webhook_fields(rt, user, identity)
|
| 72 |
+
import automation_engine
|
| 73 |
+
reason = None if callable(getattr(automation_engine, 'fetch_json', None)) else 'webhook_readiness_unavailable'
|
| 74 |
+
except ValueError:
|
| 75 |
+
reason = 'webhook_not_configured'
|
| 76 |
+
except Exception:
|
| 77 |
+
reason = 'webhook_readiness_unavailable'
|
| 78 |
+
return {'available': not reason, 'reason': reason,
|
| 79 |
+
'message': '' if reason is None else _REASONS[reason]}
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _configured_webhooks(rt, user):
|
| 83 |
+
"""Expose only safe metadata for this user's configured saved webhook references."""
|
| 84 |
+
from core import keychain
|
| 85 |
+
destinations = []
|
| 86 |
+
try:
|
| 87 |
+
if not callable(getattr(rt, 'get', None)):
|
| 88 |
+
return {'configured': False, 'destinations': [], 'reason': 'webhook_not_configured'}
|
| 89 |
+
for entry in keychain.list_entries(rt, 'generic'):
|
| 90 |
+
identity = entry.get('id')
|
| 91 |
+
if entry.get('createdBy') != user or not isinstance(identity, str) or not rows._WEBHOOK.fullmatch(identity):
|
| 92 |
+
continue
|
| 93 |
+
readiness = _webhook_readiness(rt, user, identity)
|
| 94 |
+
if readiness['available']:
|
| 95 |
+
label = entry.get('label')
|
| 96 |
+
destinations.append({'id': identity, 'label': dash_norm.plain(label[:200]) if isinstance(label, str) else identity,
|
| 97 |
+
'destination': 'webhook:' + identity, **readiness})
|
| 98 |
+
except Exception:
|
| 99 |
+
return {'configured': False, 'destinations': [], 'reason': 'webhook_readiness_unavailable'}
|
| 100 |
+
return {'configured': bool(destinations), 'destinations': destinations,
|
| 101 |
+
'reason': None if destinations else 'webhook_not_configured'}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
@router.get('/destinations')
|
| 105 |
def get_destinations(session: Session = Depends(require_session)):
|
| 106 |
_own_session(session)
|
|
|
|
| 109 |
except Exception:
|
| 110 |
raise err(503, 'notification_store_unavailable', 'Notification destinations are unavailable for this workspace.')
|
| 111 |
mail_ready = email_send.availability(session.runtime)
|
| 112 |
+
in_app = _in_app_readiness(session.runtime)
|
| 113 |
projected = []
|
| 114 |
for preference in destinations:
|
| 115 |
readiness = {
|
|
|
|
| 119 |
}
|
| 120 |
if preference['email']:
|
| 121 |
readiness['email:' + preference['email']] = dict(mail_ready)
|
| 122 |
+
if preference.get('inApp'):
|
| 123 |
+
readiness['in-app:' + session.uname] = dict(in_app)
|
| 124 |
+
for identity in preference.get('webhooks', []):
|
| 125 |
+
readiness['webhook:' + identity] = _webhook_readiness(session.runtime, session.uname, identity)
|
| 126 |
projected.append({**preference, 'label': rows.KIND_LABELS[preference['kind']], 'readiness': readiness})
|
| 127 |
+
return {'destinations': projected, 'email': mail_ready,
|
| 128 |
+
'inApp': {**in_app, 'destination': 'in-app:' + session.uname},
|
| 129 |
+
'webhooks': _configured_webhooks(session.runtime, session.uname)}
|
| 130 |
|
| 131 |
|
| 132 |
def _save(kind, body, session, *, replace):
|
|
|
|
| 135 |
rows.kind_id(kind)
|
| 136 |
patch = rows.clean_patch(body, replace=replace)
|
| 137 |
except (ValueError, TypeError):
|
| 138 |
+
raise err(400, 'notification_destination_invalid', 'Choose a known notification kind and valid saved destinations.')
|
| 139 |
+
if patch.get('inApp') and not _in_app_readiness(session.runtime)['available']:
|
| 140 |
+
raise err(400, 'notification_destination_invalid', _REASONS['in_app_unavailable'])
|
| 141 |
try:
|
| 142 |
return rows.edit(session.runtime, session.uname, kind, patch)
|
| 143 |
+
except ValueError:
|
| 144 |
+
raise err(400, 'notification_destination_invalid', 'Choose one of your configured webhook destinations.')
|
| 145 |
except Exception:
|
| 146 |
raise err(503, 'notification_store_unavailable', 'Notification destinations could not be saved for this workspace.')
|
| 147 |
|
|
|
|
| 253 |
return {'state': 'sent', 'reason': None, 'message': ''}
|
| 254 |
|
| 255 |
|
| 256 |
+
def _dispatch_in_app(rt, preference, kind, items):
|
| 257 |
+
ready = _in_app_readiness(rt)
|
| 258 |
+
if not ready['available']:
|
| 259 |
+
return _refused(ready['reason'])
|
| 260 |
+
import assistant_log
|
| 261 |
+
items = [{**item, 'title': dash_norm.plain(item['title']), 'body': dash_norm.plain(item['body'])} for item in items]
|
| 262 |
+
body = '\n\n'.join(item['title'] + '\n' + item['body'] for item in items)
|
| 263 |
+
turn = {'role': 'system', 'door': 'wake', 'kind': 'notice', 'text': body,
|
| 264 |
+
'at': datetime.now(timezone.utc).isoformat(),
|
| 265 |
+
'payload': {'title': rows.KIND_LABELS[kind], 'body': body, 'items': items,
|
| 266 |
+
'notificationKind': kind, 'sender': 'Loopable'}}
|
| 267 |
+
receipt = assistant_log.append(rt, preference['user'], turn)
|
| 268 |
+
return ({'state': 'sent', 'reason': None, 'message': ''}
|
| 269 |
+
if isinstance(receipt, str) and receipt else _refused('in_app_delivery_unconfirmed'))
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _dispatch_webhook(rt, preference, identity, kind, items):
|
| 273 |
+
ready = _webhook_readiness(rt, preference['user'], identity)
|
| 274 |
+
if not ready['available']:
|
| 275 |
+
return _refused(ready['reason'])
|
| 276 |
+
fields = rows.webhook_fields(rt, preference['user'], identity)
|
| 277 |
+
import automation_engine
|
| 278 |
+
items = [{**item, 'title': dash_norm.plain(item['title']), 'body': dash_norm.plain(item['body'])} for item in items]
|
| 279 |
+
payload = {'version': rows.PAYLOAD_VERSION, 'tenant': rt.key, 'user': preference['user'],
|
| 280 |
+
'kind': kind, 'items': items}
|
| 281 |
+
status, _body = automation_engine.fetch_json(fields['url'], payload, timeout=20, max_kb=64)
|
| 282 |
+
return ({'state': 'sent', 'reason': None, 'message': ''}
|
| 283 |
+
if 200 <= status < 300 else _refused('webhook_refused'))
|
| 284 |
+
|
| 285 |
+
|
| 286 |
def deliver(rt, kind, items, *, user=None, dispatcher=None, mail_transport=None):
|
| 287 |
"""Deliver trusted recipient-safe items to saved picks, optionally targeting one user."""
|
| 288 |
kind, items = rows.kind_id(kind), rows.clean_items(items)
|
|
|
|
| 294 |
destinations = ['slack:' + channel for channel in preference['slackChannels']]
|
| 295 |
if preference['email']:
|
| 296 |
destinations.append('email:' + preference['email'])
|
| 297 |
+
if preference.get('inApp'):
|
| 298 |
+
destinations.append('in-app:' + preference['user'])
|
| 299 |
+
destinations.extend('webhook:' + identity for identity in preference.get('webhooks', []))
|
| 300 |
for destination in destinations:
|
| 301 |
current = rows.get(rt, preference['user'], kind)
|
| 302 |
if not rows.selected(current, destination):
|
|
|
|
| 304 |
try:
|
| 305 |
if destination.startswith('slack:'):
|
| 306 |
outcome = _dispatch_slack(rt, current, destination[6:], kind, items, dispatcher=dispatcher)
|
| 307 |
+
elif destination.startswith('in-app:'):
|
| 308 |
+
outcome = _dispatch_in_app(rt, current, kind, items)
|
| 309 |
+
elif destination.startswith('webhook:'):
|
| 310 |
+
outcome = _dispatch_webhook(rt, current, destination[8:], kind, items)
|
| 311 |
else:
|
| 312 |
result = email_send.send_notification(rt, destination[6:], kind, items, transport=mail_transport)
|
| 313 |
outcome = ({'state': 'sent', 'reason': None, 'message': ''} if result.get('sent') is True
|
| 314 |
else _refused(result.get('reason'), message=result.get('message')))
|
| 315 |
except Exception:
|
| 316 |
+
reason = {'slack': 'transport_error', 'email': 'email_delivery_unconfirmed',
|
| 317 |
+
'in-app': 'in_app_delivery_unconfirmed', 'webhook': 'webhook_delivery_unconfirmed'}
|
| 318 |
+
outcome = _refused(reason[destination.split(':', 1)[0]])
|
| 319 |
# Persistence failures propagate. An unrecorded refusal must never look successful.
|
| 320 |
recorded = rows.record_delivery(rt, preference['user'], kind, destination, outcome)
|
| 321 |
results.append({'user': preference['user'], 'destination': destination,
|
api/odoo_relational.py
CHANGED
|
@@ -49,7 +49,9 @@ the oracle's predicate as an explicit `payment_state` condition pair.
|
|
| 49 |
"""
|
| 50 |
import datetime as _dt
|
| 51 |
import json as _json
|
|
|
|
| 52 |
import time as _time
|
|
|
|
| 53 |
|
| 54 |
#: ⛔ ROUTE ENTITLEMENT, NOT THE SPAWN GATE — `refresh()` stopped reading this at W53-T39 / R3,
|
| 55 |
#: which admits on CONFIGURED CREDENTIALS (`available()`) so a second Odoo tenant needs no edit.
|
|
@@ -101,7 +103,7 @@ LANDED_COST_LINES_KEY = "ut_odoo_landed_cost_lines"
|
|
| 101 |
from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PURCHASE_LINES_KEY,
|
| 102 |
PICKINGS_KEY, LANDED_PICKINGS_FIELD, landed_picking_field,
|
| 103 |
landed_picking_sql, require_landed_cost_mirror,
|
| 104 |
-
picking_fields, picking_source_spec,
|
| 105 |
order_fields as purchase_order_fields,
|
| 106 |
line_fields as _purchase_line_fields,
|
| 107 |
source_specs as purchase_source_specs,
|
|
@@ -110,6 +112,77 @@ from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PUR
|
|
| 110 |
#: S78-T14: the databases whose row id is a `stock.picking` id. Their read wall admits a delivery photo.
|
| 111 |
DELIVERY_KEYS = (SHIPMENTS_KEY, PICKINGS_KEY)
|
| 112 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 114 |
JOIN_KEY = "partner_id"
|
| 115 |
#: The product-grain equivalent.
|
|
@@ -159,6 +232,13 @@ def _preset(field, flow="odoo_relational"):
|
|
| 159 |
grid renders it grey and the preset walls refuse a rename or a delete."""
|
| 160 |
field = dict(field)
|
| 161 |
field["automation"] = {"flowId": flow, "preset": True}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
return field
|
| 163 |
|
| 164 |
|
|
@@ -700,7 +780,7 @@ def merge_audit_cols(sources):
|
|
| 700 |
if optional:
|
| 701 |
merged["optional"] = tuple(optional)
|
| 702 |
out[key] = merged
|
| 703 |
-
return out
|
| 704 |
|
| 705 |
|
| 706 |
#: Materialised grids read their audited columns beside their reader's own SQL.
|
|
@@ -773,6 +853,8 @@ def audit_mirror_additions():
|
|
| 773 |
# Customer tags migrate before their order projections and presets are mounted.
|
| 774 |
if "category_id" not in out["res_partner"]:
|
| 775 |
out["res_partner"].append("category_id")
|
|
|
|
|
|
|
| 776 |
return out
|
| 777 |
|
| 778 |
|
|
@@ -814,7 +896,7 @@ def shipment_fields():
|
|
| 814 |
"default": True, "description": "Elapsed days from the sale order date to the actual "
|
| 815 |
"shipment completion. Blank when either date is missing."},
|
| 816 |
_refreshed_field(),
|
| 817 |
-
)] + audit_fields(SHIPMENTS_KEY)
|
| 818 |
|
| 819 |
|
| 820 |
def agent_fields():
|
|
@@ -1124,6 +1206,14 @@ def invoice_fields():
|
|
| 1124 |
)] + audit_fields(INVOICES_KEY)
|
| 1125 |
|
| 1126 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1127 |
def order_fields():
|
| 1128 |
"""One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope."""
|
| 1129 |
return [_preset(f) for f in (
|
|
@@ -1151,8 +1241,8 @@ def order_fields():
|
|
| 1151 |
"description": "Odoo's fulfilment status: pending, started, partial, or full."},
|
| 1152 |
{"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay",
|
| 1153 |
"default": True, "agg": "sum"},
|
| 1154 |
-
{"key": "team", "label": "
|
| 1155 |
-
"default": True
|
| 1156 |
{"key": "pricelist", "label": "Pricelist", "type": "text", "source": "overlay",
|
| 1157 |
"default": True, "description": "The pricelist on this Odoo sale order."},
|
| 1158 |
{"key": "state", "label": "State", "type": "select", "source": "overlay",
|
|
@@ -1235,8 +1325,8 @@ def order_line_fields():
|
|
| 1235 |
"default": True},
|
| 1236 |
{"key": "customer_type", "label": "Customer type", "type": "text", "source": "odoo",
|
| 1237 |
"default": True, "description": "The customer's Odoo tags. Blank when no tag is set."},
|
| 1238 |
-
{"key": "team", "label": "
|
| 1239 |
-
"default": True,
|
| 1240 |
"description": "The sales team on this line's Odoo order."},
|
| 1241 |
{"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
|
| 1242 |
"default": False},
|
|
@@ -1428,6 +1518,9 @@ def landed_cost_line_fields():
|
|
| 1428 |
"format": {"decimals": 2}},
|
| 1429 |
{"key": "date", "label": "Date", "type": "date", "default": True},
|
| 1430 |
{"key": "state", "label": "State", "type": "text", "default": True},
|
|
|
|
|
|
|
|
|
|
| 1431 |
{"key": "write_date", "label": "Updated", "type": "date", "default": False},
|
| 1432 |
]] + system_link_fields(LANDED_COST_LINES_KEY)
|
| 1433 |
|
|
@@ -1532,7 +1625,8 @@ def customer_fields():
|
|
| 1532 |
{"key": "street2", "label": "Street 2", "type": "text", "source": "overlay",
|
| 1533 |
"default": False},
|
| 1534 |
{"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
|
| 1535 |
-
{"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True
|
|
|
|
| 1536 |
{"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False},
|
| 1537 |
{"key": "country", "label": "Country", "type": "text", "source": "overlay",
|
| 1538 |
"default": False},
|
|
@@ -2130,7 +2224,9 @@ def _as_date(value):
|
|
| 2130 |
STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort."""
|
| 2131 |
if not value:
|
| 2132 |
return ""
|
| 2133 |
-
|
|
|
|
|
|
|
| 2134 |
|
| 2135 |
|
| 2136 |
def _in_scope(pid, excluded):
|
|
@@ -4420,7 +4516,8 @@ def landed_cost_line_source_spec():
|
|
| 4420 |
"quantity": ("va.quantity", _stock_number), "former_cost": ("va.former_cost", _stock_number),
|
| 4421 |
"additional_landed_cost": ("va.additional_landed_cost", _stock_number),
|
| 4422 |
"final_cost": ("va.final_cost", _stock_number), "date": ("lc.date", _as_date),
|
| 4423 |
-
"state": ("lc.state", _stock_text), "
|
|
|
|
| 4424 |
},
|
| 4425 |
}
|
| 4426 |
|
|
@@ -4438,11 +4535,11 @@ def require_landed_cost_lines_mirror(cur):
|
|
| 4438 |
if not fields <= columns(cur, entity):
|
| 4439 |
raise Refused("The landed cost lines mirror is incomplete. Sync Odoo before reading it.")
|
| 4440 |
try:
|
| 4441 |
-
|
| 4442 |
except Exception:
|
| 4443 |
-
|
| 4444 |
spec = datastore.ENTITIES.get(entity)
|
| 4445 |
-
if (not
|
| 4446 |
or (spec.get("backfill_fields")
|
| 4447 |
and not datastore._columns_backfilled(cur, entity, spec))):
|
| 4448 |
raise Refused("The landed cost lines mirror is catching up. Try again shortly.")
|
|
@@ -4591,7 +4688,7 @@ def validate_customer_invoice_cadence(client, cur, today=None):
|
|
| 4591 |
observed = dict(cur.execute(
|
| 4592 |
f"SELECT {partner}, {sem._measure_sql(metric, store['alias'], today=today)} "
|
| 4593 |
f"FROM {store['table']} {store['alias']} {store.get('join', '')} "
|
| 4594 |
-
f"WHERE {sem.render_scope(store['scope_sql'])} GROUP BY {partner}").fetchall())
|
| 4595 |
pids = [int(row[JOIN_KEY]) for row in read_customers(cur)]
|
| 4596 |
# The oracle stays on invoice_date when a mutant redirects the served binding to orders.
|
| 4597 |
independent = sem.customer_rollup_oracle('customer_days_since_invoice', client,
|
|
@@ -4623,7 +4720,7 @@ def validate_customer_rollups(client, cur):
|
|
| 4623 |
value = cur.execute(
|
| 4624 |
f"SELECT {sem._measure_sql(metric, store['alias'])} FROM {store['table']} "
|
| 4625 |
f"{store['alias']} {store.get('join', '')} "
|
| 4626 |
-
f"WHERE {sem.render_scope(store['scope_sql'])}").fetchone()[0]
|
| 4627 |
independent = sem.customer_rollup_oracle(bag['measure'], client)
|
| 4628 |
ok = (str(value or '')[:10] == str(independent or '')[:10]
|
| 4629 |
if metric.get('format') == 'date' else
|
|
@@ -5848,33 +5945,58 @@ def _slot_on(day, slot, zone):
|
|
| 5848 |
return _dt.datetime(day.year, day.month, day.day, hour, minute, tzinfo=zone).timestamp()
|
| 5849 |
|
| 5850 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5851 |
def slot_before(now, slot):
|
| 5852 |
-
"""The latest occurrence of the slot at or before epoch `now`."""
|
| 5853 |
-
zone = sync_zone(slot["tz"])
|
| 5854 |
-
day = _dt.datetime.fromtimestamp(float(now), zone).date()
|
| 5855 |
for back in range(4):
|
| 5856 |
-
at = _slot_on(day - _dt.timedelta(days=back), slot, zone)
|
| 5857 |
if at <= float(now):
|
| 5858 |
return at
|
| 5859 |
raise ValueError("no slot occurrence found")
|
| 5860 |
|
| 5861 |
|
| 5862 |
def slot_after(now, slot):
|
| 5863 |
-
"""The first occurrence of the slot strictly after epoch `now`."""
|
| 5864 |
-
zone = sync_zone(slot["tz"])
|
| 5865 |
-
day = _dt.datetime.fromtimestamp(float(now), zone).date()
|
| 5866 |
for ahead in range(4):
|
| 5867 |
-
at = _slot_on(day + _dt.timedelta(days=ahead), slot, zone)
|
| 5868 |
if at > float(now):
|
| 5869 |
return at
|
| 5870 |
raise ValueError("no slot occurrence found")
|
| 5871 |
|
| 5872 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5873 |
#: A first-load continuation does not start this close to the next slot, so it cannot delay the slot.
|
| 5874 |
FIRST_LOAD_SLOT_QUIET_SECONDS = 2 * 3600
|
| 5875 |
|
| 5876 |
-
#:
|
| 5877 |
SLOT_LATE_SECONDS = 90 * 60
|
|
|
|
| 5878 |
|
| 5879 |
|
| 5880 |
def slot_near(now, slot, seconds=FIRST_LOAD_SLOT_QUIET_SECONDS):
|
|
@@ -6020,8 +6142,7 @@ def record_sync_outcome(rt, ok, at=None, failed_databases=None):
|
|
| 6020 |
return str(defn.get("label") or key) if isinstance(defn, dict) else key
|
| 6021 |
|
| 6022 |
def _plain(text, limit):
|
| 6023 |
-
|
| 6024 |
-
return " ".join(str(text or "").replace("—", ",").replace("–", ",").split())[:limit]
|
| 6025 |
|
| 6026 |
failed = []
|
| 6027 |
for item in failed_databases or ():
|
|
@@ -6234,6 +6355,82 @@ def _effective_sync_seconds(preset):
|
|
| 6234 |
return max(secs, env)
|
| 6235 |
|
| 6236 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6237 |
def sync_freshness(rt, now=None):
|
| 6238 |
"""C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
|
| 6239 |
fail. PURE over the store reads.
|
|
@@ -6242,6 +6439,7 @@ def sync_freshness(rt, now=None):
|
|
| 6242 |
`STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
|
| 6243 |
(frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
|
| 6244 |
the last run's own reason, its code prefix removed, when that run ended in error.
|
|
|
|
| 6245 |
"""
|
| 6246 |
now = now or _utc_now()
|
| 6247 |
if isinstance(now, str):
|
|
@@ -6261,10 +6459,15 @@ def sync_freshness(rt, now=None):
|
|
| 6261 |
except Exception: # noqa: BLE001
|
| 6262 |
status = None
|
| 6263 |
status = status if isinstance(status, dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6264 |
phase = str(status.get("phase") or "unknown")
|
| 6265 |
-
last_good = raw.get("lastGoodSyncAt") or None
|
| 6266 |
# Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
|
| 6267 |
-
if not last_good and phase == "ok":
|
| 6268 |
last_good = status.get("lastRun") or None
|
| 6269 |
every = _effective_sync_seconds(cfg["syncEvery"])
|
| 6270 |
allowed = None if every is None else every + STALE_GRACE_SECONDS
|
|
@@ -6272,18 +6475,6 @@ def sync_freshness(rt, now=None):
|
|
| 6272 |
if allowed is not None and not cfg["frozen"]:
|
| 6273 |
good_at = _parse_iso(last_good)
|
| 6274 |
stale = good_at is None or (now - good_at).total_seconds() > allowed
|
| 6275 |
-
# A daily slot that passed SLOT_LATE_SECONDS ago with no good sync since is late, not fresh.
|
| 6276 |
-
late_slot = None
|
| 6277 |
-
slot = sync_slot(cfg)
|
| 6278 |
-
if slot is not None and not cfg["frozen"] and not stale:
|
| 6279 |
-
try:
|
| 6280 |
-
due = slot_before(now.timestamp(), slot)
|
| 6281 |
-
except Exception: # noqa: BLE001
|
| 6282 |
-
due = None
|
| 6283 |
-
good_at = _parse_iso(last_good)
|
| 6284 |
-
if (due is not None and now.timestamp() - due > SLOT_LATE_SECONDS
|
| 6285 |
-
and (good_at is None or good_at.timestamp() < due)):
|
| 6286 |
-
late_slot = slot_iso(due)
|
| 6287 |
failure = None
|
| 6288 |
errors = status.get("errors") if isinstance(status.get("errors"), list) else []
|
| 6289 |
if phase == "error" and errors:
|
|
@@ -6293,20 +6484,29 @@ def sync_freshness(rt, now=None):
|
|
| 6293 |
code, message = "sync_failed", text
|
| 6294 |
failure = {"code": code.strip()[:80], "message": message.strip()[:300],
|
| 6295 |
"at": status.get("lastRun") or None}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6296 |
# D-2679: a database that failed while its siblings synced is reported by itself.
|
| 6297 |
failed_databases = []
|
| 6298 |
for item in raw.get("failedDatabases") if isinstance(raw.get("failedDatabases"), list) else ():
|
| 6299 |
if isinstance(item, dict) and item.get("key"):
|
| 6300 |
failed_databases.append({
|
| 6301 |
-
name: (" ".join(
|
| 6302 |
-
.replace("–", ",").split())[:300] or None)
|
| 6303 |
for name in ("key", "label", "code", "message", "stage", "at")
|
| 6304 |
+ (("cause",) if item.get("cause") else ())})
|
| 6305 |
return {"stale": bool(stale), "lastGoodSyncAt": last_good,
|
| 6306 |
-
|
| 6307 |
"failedDatabases": failed_databases,
|
|
|
|
| 6308 |
"schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
|
| 6309 |
-
"lastRunAt":
|
| 6310 |
"frozen": cfg["frozen"], "failure": failure,
|
| 6311 |
"syncAt": cfg["syncAt"], "syncTz": cfg["syncTz"],
|
| 6312 |
"nextSyncAt": None if cfg["frozen"] else next_sync_at(cfg, now.timestamp())}
|
|
@@ -6628,7 +6828,7 @@ def shipment_source_spec():
|
|
| 6628 |
"state": ("sp.state", _stock_text),
|
| 6629 |
"scheduled_date": (date_sql("sp.scheduled_date"), _as_date),
|
| 6630 |
"date_done": (date_sql("sp.date_done"), _as_date),
|
| 6631 |
-
"order_date": (
|
| 6632 |
"delivery_days": ("epoch(TRY_CAST(sp.date_done AS TIMESTAMPTZ) - "
|
| 6633 |
"TRY_CAST(so.date_order AS TIMESTAMPTZ)) / 86400.0", number),
|
| 6634 |
"refreshed": ("(SELECT substr(updated_at, 1, 10) FROM _sync_state "
|
|
@@ -6965,6 +7165,11 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
|
|
| 6965 |
# ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a
|
| 6966 |
# column a user added to a locked database is never rewritten.
|
| 6967 |
stored = have[fkey]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6968 |
# ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp,
|
| 6969 |
# same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the
|
| 6970 |
# loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on
|
|
@@ -6973,7 +7178,7 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
|
|
| 6973 |
continue
|
| 6974 |
if (stored.get("automation") or {}).get("preset"):
|
| 6975 |
for prop in ("label", "type", "options", "choicesFrom", "link", "rollup", "description",
|
| 6976 |
-
"agg", "pinned", "default"):
|
| 6977 |
# A subsequent row-only refresh must not replace authoritative selections with
|
| 6978 |
# an older compiled vocabulary while its metadata catalogue is unavailable.
|
| 6979 |
if (stored.get("choicesFrom") == "odoo" and field.get("choicesFrom") != "odoo"
|
|
@@ -7126,76 +7331,233 @@ def _read_model_rows(client, catalogue_row, *, on_page=None):
|
|
| 7126 |
|
| 7127 |
def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, today,
|
| 7128 |
on_progress):
|
| 7129 |
-
"""Publish a
|
|
|
|
|
|
|
|
|
|
| 7130 |
ut = _ut()
|
| 7131 |
key, model = choice["key"], choice["model"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7132 |
fields, stamp = _model_field_specs(catalogue_row), today or _iso_today()
|
| 7133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7134 |
try:
|
| 7135 |
-
|
| 7136 |
-
|
| 7137 |
-
|
| 7138 |
-
|
| 7139 |
-
|
| 7140 |
-
|
| 7141 |
-
|
| 7142 |
-
|
| 7143 |
-
|
| 7144 |
-
|
| 7145 |
-
|
| 7146 |
-
|
| 7147 |
-
|
| 7148 |
-
|
| 7149 |
-
|
| 7150 |
-
|
| 7151 |
-
|
| 7152 |
-
|
| 7153 |
-
|
| 7154 |
-
|
| 7155 |
-
|
| 7156 |
-
|
| 7157 |
-
|
| 7158 |
-
|
| 7159 |
-
|
| 7160 |
-
|
| 7161 |
-
|
| 7162 |
-
|
| 7163 |
-
|
| 7164 |
-
|
| 7165 |
-
|
| 7166 |
-
|
| 7167 |
-
|
| 7168 |
-
|
| 7169 |
-
|
| 7170 |
-
|
| 7171 |
-
shares.repair_published_database_grants(rt, table_keys=(key,))
|
| 7172 |
-
# ⛔ HOTFIX 2026-09-24: the store door stamps a prefix that crossed the byte budget
|
| 7173 |
-
# and stores none of its rows. Stop reading Odoo here rather than pull the rest of
|
| 7174 |
-
# the model only to have every further publish discarded.
|
| 7175 |
-
if ut.oversize_of(key, defn=(held.get("doc") or {}).get(key) or {}):
|
| 7176 |
-
raise Refused(str((ut.limit_report(key, st=rt) or {}).get("cause")
|
| 7177 |
-
or "this database is too large to store"))
|
| 7178 |
-
ut.register_connected(key)
|
| 7179 |
-
published_rows = written[key]["rows"]
|
| 7180 |
-
pages_since_publish, last_publish = 0, _time.monotonic()
|
| 7181 |
-
on_progress(published_rows, total)
|
| 7182 |
-
|
| 7183 |
-
values = _read_model_rows(client, catalogue_row, on_page=on_page)
|
| 7184 |
-
if total is not None and len(values) < total:
|
| 7185 |
-
raise Refused("Odoo returned fewer rows than its count. "
|
| 7186 |
-
"Retry sync to load the remaining rows.")
|
| 7187 |
return {"tables": written, "errors": []}
|
| 7188 |
|
| 7189 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7190 |
def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
| 7191 |
username="automation", today=None, table_keys=None, on_progress=None):
|
| 7192 |
-
"""
|
| 7193 |
|
| 7194 |
Disabled models are absent from this pass rather than represented by an empty row set. That
|
| 7195 |
distinction is the preservation guarantee: no updater touches their definition, rows, fields,
|
| 7196 |
grants, ownership, or tenant rename, and re-enabling resumes against the same stable key.
|
| 7197 |
-
|
| 7198 |
-
retain the all-model atomic publication and its shrink guards.
|
| 7199 |
"""
|
| 7200 |
requested = set(table_keys) if table_keys is not None else None
|
| 7201 |
if requested is not None and not requested:
|
|
@@ -7210,30 +7572,15 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
|
| 7210 |
and (requested is None or choice["key"] in requested)]
|
| 7211 |
if not selected:
|
| 7212 |
return {}
|
| 7213 |
-
|
| 7214 |
-
|
| 7215 |
-
|
| 7216 |
-
|
| 7217 |
-
# rows) only for the store door to discard the rows. The refusal and its fix are on the wire
|
| 7218 |
-
# through `limit_report`; disabling and re-enabling does not bypass it.
|
| 7219 |
-
oversized = [choice for choice in selected
|
| 7220 |
-
if ut.oversize_of(choice["key"], defn=existing.get(str(choice["key"])) or {})]
|
| 7221 |
-
if oversized:
|
| 7222 |
-
if requested is not None and on_progress is not None:
|
| 7223 |
-
raise Refused(str((ut.limit_report(oversized[0]["key"], st=rt) or {}).get("cause")
|
| 7224 |
-
or "this database is too large to store"))
|
| 7225 |
-
selected = [choice for choice in selected if choice not in oversized]
|
| 7226 |
-
if not selected:
|
| 7227 |
-
return {"tables": {}, "errors": [
|
| 7228 |
-
f"{choice['model']}: too large to store, served nowhere until read-through"
|
| 7229 |
-
for choice in oversized]}
|
| 7230 |
-
keys = [str(choice["key"]) for choice in selected]
|
| 7231 |
-
# D-3284: enabled Odoo models are connector databases and take no MAX_TABLES slot.
|
| 7232 |
source =(rt.odoo_source() if source_connection_id is None
|
| 7233 |
else rt.odoo_source(source_connection_id))
|
| 7234 |
if source is None:
|
| 7235 |
raise Refused("the selected Odoo connection is unavailable, so no discovered model was read")
|
| 7236 |
-
|
| 7237 |
from core.odoo import get_odoo
|
| 7238 |
with source._bound(): # tenant credential + scope
|
| 7239 |
client = get_odoo()
|
|
@@ -7241,39 +7588,16 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
|
| 7241 |
return _publish_discovered_model(
|
| 7242 |
rt, client, selected[0], by_model[str(selected[0]["model"])],
|
| 7243 |
username=username, today=today, on_progress=on_progress)
|
| 7244 |
-
for choice
|
| 7245 |
model = str(choice["model"])
|
| 7246 |
row = by_model[model]
|
| 7247 |
try:
|
| 7248 |
-
|
|
|
|
| 7249 |
except Exception as exc: # noqa: BLE001
|
| 7250 |
-
errors.append(f"{model}: {
|
| 7251 |
-
continue
|
| 7252 |
-
stored_n = len(((existing.get(key) or {}).get("rows")) or {})
|
| 7253 |
-
if stored_n and len(values) < stored_n * MAX_SHRINK:
|
| 7254 |
-
errors.append(f"{model}: read returned {len(values)} rows against {stored_n} "
|
| 7255 |
-
"stored; preserved the prior database instead")
|
| 7256 |
continue
|
| 7257 |
-
|
| 7258 |
-
written = {}
|
| 7259 |
-
stamp = today or _iso_today()
|
| 7260 |
-
ut.register_connected(*(key for key, _label, _fields, _rows, _model in prepared))
|
| 7261 |
-
|
| 7262 |
-
def _apply(cur):
|
| 7263 |
-
cur = cur if isinstance(cur, dict) else {}
|
| 7264 |
-
for key, label, fields, rows, model in prepared:
|
| 7265 |
-
written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp)
|
| 7266 |
-
from core import connector_sync
|
| 7267 |
-
cur[key]["connector"] = {
|
| 7268 |
-
**connector_sync.database_identity(
|
| 7269 |
-
"odoo", connection_for_table_key(key, rt), key, model),
|
| 7270 |
-
"key": "odoo", "readOnly": True}
|
| 7271 |
-
return cur
|
| 7272 |
-
|
| 7273 |
-
if prepared:
|
| 7274 |
-
rt.update(ut.STORE_KEY, _apply, flush="sync")
|
| 7275 |
-
from core import shares
|
| 7276 |
-
shares.repair_published_database_grants(rt, table_keys=tuple(written))
|
| 7277 |
return {"tables": written, "errors": errors}
|
| 7278 |
|
| 7279 |
|
|
@@ -7465,7 +7789,7 @@ def _scrub_cause(exc):
|
|
| 7465 |
text = _SCRUB_QUOTED.sub(
|
| 7466 |
lambda m: m.group(0) if _SCRUB_NAME.fullmatch(m.group(1) or m.group(2) or "")
|
| 7467 |
else "'<value>'", text)
|
| 7468 |
-
text = " ".join(
|
| 7469 |
return (f"{name}: {text}" if text else name)[:300]
|
| 7470 |
|
| 7471 |
|
|
@@ -7598,10 +7922,8 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 7598 |
|
| 7599 |
|
| 7600 |
def _canonical_mirror_ready(rt):
|
| 7601 |
-
#
|
| 7602 |
-
|
| 7603 |
-
from harness import datastore
|
| 7604 |
-
return rt is None or selected_mirror_entities(rt) is None or datastore.ready()
|
| 7605 |
|
| 7606 |
|
| 7607 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
|
|
|
| 49 |
"""
|
| 50 |
import datetime as _dt
|
| 51 |
import json as _json
|
| 52 |
+
import math as _math
|
| 53 |
import time as _time
|
| 54 |
+
from dash_norm import plain as _dash_plain
|
| 55 |
|
| 56 |
#: ⛔ ROUTE ENTITLEMENT, NOT THE SPAWN GATE — `refresh()` stopped reading this at W53-T39 / R3,
|
| 57 |
#: which admits on CONFIGURED CREDENTIALS (`available()`) so a second Odoo tenant needs no edit.
|
|
|
|
| 103 |
from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PURCHASE_LINES_KEY,
|
| 104 |
PICKINGS_KEY, LANDED_PICKINGS_FIELD, landed_picking_field,
|
| 105 |
landed_picking_sql, require_landed_cost_mirror,
|
| 106 |
+
picking_fields as _picking_fields, picking_source_spec,
|
| 107 |
order_fields as purchase_order_fields,
|
| 108 |
line_fields as _purchase_line_fields,
|
| 109 |
source_specs as purchase_source_specs,
|
|
|
|
| 112 |
#: S78-T14: the databases whose row id is a `stock.picking` id. Their read wall admits a delivery photo.
|
| 113 |
DELIVERY_KEYS = (SHIPMENTS_KEY, PICKINGS_KEY)
|
| 114 |
|
| 115 |
+
DELIVERY_PHOTO_REFERENCE = "delivery_photo_attachments"
|
| 116 |
+
_PHOTO_PRODUCER_MISSING = "delivery_photo_metadata_unavailable"
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def delivery_photo_reference_field():
|
| 120 |
+
"""Declare a metadata-only attachment-reference array using the supported JSON field type."""
|
| 121 |
+
return _preset({"key": DELIVERY_PHOTO_REFERENCE, "label": "Delivery photo attachments",
|
| 122 |
+
"type": "json", "source": "overlay", "default": False,
|
| 123 |
+
"description": "Attachment-reference array: id, res_model, res_id, name, mimetype, "
|
| 124 |
+
"file_size and create_date. Metadata only. No URLs or file contents. "
|
| 125 |
+
"An unavailable producer returns {error: {code, cause, fix}}.",
|
| 126 |
+
"attachmentReference": {"source": "odoo", "model": "ir.attachment", "multiple": True,
|
| 127 |
+
"fields": {"id": "int", "res_model": "text", "res_id": "int", "name": "text",
|
| 128 |
+
"mimetype": "text", "file_size": "int", "create_date": "datetime"}}})
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def picking_fields():
|
| 132 |
+
"""Add delivery attachment references to the existing picking field contract."""
|
| 133 |
+
return _picking_fields() + [delivery_photo_reference_field()]
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def _delivery_photo_references(value):
|
| 137 |
+
"""Decode references and distinguish a missing producer from an empty attachment list."""
|
| 138 |
+
if isinstance(value, str) and value != _PHOTO_PRODUCER_MISSING:
|
| 139 |
+
value = _json.loads(value)
|
| 140 |
+
if value is None or value == _PHOTO_PRODUCER_MISSING:
|
| 141 |
+
return {"error": {"code": "field_unavailable",
|
| 142 |
+
"cause": "Delivery photo attachments have no complete persisted metadata producer.",
|
| 143 |
+
"fix": "Run this tenant's Odoo source sync with delivery_photo_metadata, then publish "
|
| 144 |
+
"its complete observation snapshot."}}
|
| 145 |
+
return value
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _merge_delivery_photo_cols(sources):
|
| 149 |
+
"""Project tenant-local attachment metadata through the existing delivery row walls."""
|
| 150 |
+
out = dict(sources)
|
| 151 |
+
for key, spec in sources.items():
|
| 152 |
+
if base_table_key(key) not in DELIVERY_KEYS:
|
| 153 |
+
continue
|
| 154 |
+
owner = (spec["id"].rsplit(".", 1)[0] + "." if "." in spec["id"]
|
| 155 |
+
else spec["table"] + ".")
|
| 156 |
+
missing = ("'" + _json.dumps(_PHOTO_PRODUCER_MISSING) + "'", _delivery_photo_references)
|
| 157 |
+
marker = ("CASE (SELECT legacy FROM odoo_connections pc WHERE pc.connection_id = "
|
| 158 |
+
+ owner + "_connection_id) WHEN TRUE THEN 'delivery_photo_metadata' "
|
| 159 |
+
"WHEN FALSE THEN 'delivery_photo_metadata.connection.' || sha256("
|
| 160 |
+
+ owner + "_connection_id) END")
|
| 161 |
+
payload = ("SELECT to_json(list(struct_pack(id := coalesce(pa._source_id, pa.id), "
|
| 162 |
+
"res_model := pa.res_model, res_id := pa.res_id, name := pa.name, "
|
| 163 |
+
"mimetype := pa.mimetype, file_size := pa.file_size, create_date := pa.create_date) "
|
| 164 |
+
"ORDER BY coalesce(pa._source_id, pa.id))) FROM delivery_photo_metadata pa "
|
| 165 |
+
"WHERE pa.res_model = 'stock.picking' AND pa.res_id = coalesce("
|
| 166 |
+
+ owner + "_source_id, " + owner + "id) AND pa._connection_id "
|
| 167 |
+
"IS NOT DISTINCT FROM " + owner + "_connection_id")
|
| 168 |
+
sql = ("CASE WHEN EXISTS (SELECT 1 FROM _sync_state ps WHERE ps.entity = " + marker
|
| 169 |
+
+ " AND ps.phase = 'live') THEN coalesce((" + payload + "), '[]') ELSE "
|
| 170 |
+
+ missing[0] + " END")
|
| 171 |
+
cols = {**spec["cols"], DELIVERY_PHOTO_REFERENCE: (sql, _delivery_photo_references)}
|
| 172 |
+
optional = list(spec.get("optional") or ())
|
| 173 |
+
from harness import datastore
|
| 174 |
+
requirements = [("delivery_photo_metadata", field) for field in
|
| 175 |
+
(*datastore.DELIVERY_PHOTO_FIELDS, "_source_id", "_connection_id")]
|
| 176 |
+
requirements += [("odoo_connections", "connection_id"), ("odoo_connections", "legacy"),
|
| 177 |
+
("_sync_state", "entity"), ("_sync_state", "phase"),
|
| 178 |
+
("stock_picking", "_source_id"), ("stock_picking", "_connection_id")]
|
| 179 |
+
for requirement in requirements:
|
| 180 |
+
fallback = {"absent": (requirement,), "without": {"cols": {DELIVERY_PHOTO_REFERENCE: missing}}}
|
| 181 |
+
if fallback not in optional:
|
| 182 |
+
optional.append(fallback)
|
| 183 |
+
out[key] = {**spec, "cols": cols, "optional": tuple(optional)}
|
| 184 |
+
return out
|
| 185 |
+
|
| 186 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 187 |
JOIN_KEY = "partner_id"
|
| 188 |
#: The product-grain equivalent.
|
|
|
|
| 232 |
grid renders it grey and the preset walls refuse a rename or a delete."""
|
| 233 |
field = dict(field)
|
| 234 |
field["automation"] = {"flowId": flow, "preset": True}
|
| 235 |
+
if field.get('key') in {'customer', 'vendor', 'email', 'phone', 'mobile', 'street',
|
| 236 |
+
'street2', 'city', 'zip', 'country', 'contact', 'contact_name'}:
|
| 237 |
+
field['personal'] = True
|
| 238 |
+
if field.get('key') == 'team':
|
| 239 |
+
field['label'] = 'Sales team'
|
| 240 |
+
field['options'] = _team_options()
|
| 241 |
+
field['automation']['renamable'] = True
|
| 242 |
return field
|
| 243 |
|
| 244 |
|
|
|
|
| 780 |
if optional:
|
| 781 |
merged["optional"] = tuple(optional)
|
| 782 |
out[key] = merged
|
| 783 |
+
return _merge_delivery_photo_cols(out)
|
| 784 |
|
| 785 |
|
| 786 |
#: Materialised grids read their audited columns beside their reader's own SQL.
|
|
|
|
| 853 |
# Customer tags migrate before their order projections and presets are mounted.
|
| 854 |
if "category_id" not in out["res_partner"]:
|
| 855 |
out["res_partner"].append("category_id")
|
| 856 |
+
out["stock_landed_cost"] = list(LANDED_COST_HEADER_ADDITIONS)
|
| 857 |
+
out[BANK_LINE_TABLE] = list(BANK_LINE_ADDITIONS)
|
| 858 |
return out
|
| 859 |
|
| 860 |
|
|
|
|
| 896 |
"default": True, "description": "Elapsed days from the sale order date to the actual "
|
| 897 |
"shipment completion. Blank when either date is missing."},
|
| 898 |
_refreshed_field(),
|
| 899 |
+
)] + audit_fields(SHIPMENTS_KEY) + [delivery_photo_reference_field()]
|
| 900 |
|
| 901 |
|
| 902 |
def agent_fields():
|
|
|
|
| 1206 |
)] + audit_fields(INVOICES_KEY)
|
| 1207 |
|
| 1208 |
|
| 1209 |
+
def _team_options():
|
| 1210 |
+
from core import odoo
|
| 1211 |
+
try:
|
| 1212 |
+
return list(odoo.active_team_names().values())
|
| 1213 |
+
except odoo.TeamsUnknown:
|
| 1214 |
+
return []
|
| 1215 |
+
|
| 1216 |
+
|
| 1217 |
def order_fields():
|
| 1218 |
"""One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope."""
|
| 1219 |
return [_preset(f) for f in (
|
|
|
|
| 1241 |
"description": "Odoo's fulfilment status: pending, started, partial, or full."},
|
| 1242 |
{"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay",
|
| 1243 |
"default": True, "agg": "sum"},
|
| 1244 |
+
{"key": "team", "label": "Sales team", "type": "select", "source": "overlay",
|
| 1245 |
+
"default": True},
|
| 1246 |
{"key": "pricelist", "label": "Pricelist", "type": "text", "source": "overlay",
|
| 1247 |
"default": True, "description": "The pricelist on this Odoo sale order."},
|
| 1248 |
{"key": "state", "label": "State", "type": "select", "source": "overlay",
|
|
|
|
| 1325 |
"default": True},
|
| 1326 |
{"key": "customer_type", "label": "Customer type", "type": "text", "source": "odoo",
|
| 1327 |
"default": True, "description": "The customer's Odoo tags. Blank when no tag is set."},
|
| 1328 |
+
{"key": "team", "label": "Sales team", "type": "select", "source": "odoo",
|
| 1329 |
+
"default": True,
|
| 1330 |
"description": "The sales team on this line's Odoo order."},
|
| 1331 |
{"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
|
| 1332 |
"default": False},
|
|
|
|
| 1518 |
"format": {"decimals": 2}},
|
| 1519 |
{"key": "date", "label": "Date", "type": "date", "default": True},
|
| 1520 |
{"key": "state", "label": "State", "type": "text", "default": True},
|
| 1521 |
+
{"key": "amount_total", "label": "Landed cost total", "type": "currency", "default": False,
|
| 1522 |
+
"format": {"decimals": 2},
|
| 1523 |
+
"description": "The header total, repeated on each adjustment line."},
|
| 1524 |
{"key": "write_date", "label": "Updated", "type": "date", "default": False},
|
| 1525 |
]] + system_link_fields(LANDED_COST_LINES_KEY)
|
| 1526 |
|
|
|
|
| 1625 |
{"key": "street2", "label": "Street 2", "type": "text", "source": "overlay",
|
| 1626 |
"default": False},
|
| 1627 |
{"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
|
| 1628 |
+
{"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True,
|
| 1629 |
+
"personal": True},
|
| 1630 |
{"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False},
|
| 1631 |
{"key": "country", "label": "Country", "type": "text", "source": "overlay",
|
| 1632 |
"default": False},
|
|
|
|
| 2224 |
STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort."""
|
| 2225 |
if not value:
|
| 2226 |
return ""
|
| 2227 |
+
from core import periods
|
| 2228 |
+
day = periods.business_day(value)
|
| 2229 |
+
return day.isoformat() if day else str(value)[:10]
|
| 2230 |
|
| 2231 |
|
| 2232 |
def _in_scope(pid, excluded):
|
|
|
|
| 4516 |
"quantity": ("va.quantity", _stock_number), "former_cost": ("va.former_cost", _stock_number),
|
| 4517 |
"additional_landed_cost": ("va.additional_landed_cost", _stock_number),
|
| 4518 |
"final_cost": ("va.final_cost", _stock_number), "date": ("lc.date", _as_date),
|
| 4519 |
+
"state": ("lc.state", _stock_text), "amount_total": ("lc.amount_total", _stock_number),
|
| 4520 |
+
"write_date": ("va.write_date", _as_date),
|
| 4521 |
},
|
| 4522 |
}
|
| 4523 |
|
|
|
|
| 4535 |
if not fields <= columns(cur, entity):
|
| 4536 |
raise Refused("The landed cost lines mirror is incomplete. Sync Odoo before reading it.")
|
| 4537 |
try:
|
| 4538 |
+
state = datastore._state(cur, entity)
|
| 4539 |
except Exception:
|
| 4540 |
+
state = None
|
| 4541 |
spec = datastore.ENTITIES.get(entity)
|
| 4542 |
+
if (not state or state["phase"] != "live" or not spec
|
| 4543 |
or (spec.get("backfill_fields")
|
| 4544 |
and not datastore._columns_backfilled(cur, entity, spec))):
|
| 4545 |
raise Refused("The landed cost lines mirror is catching up. Try again shortly.")
|
|
|
|
| 4688 |
observed = dict(cur.execute(
|
| 4689 |
f"SELECT {partner}, {sem._measure_sql(metric, store['alias'], today=today)} "
|
| 4690 |
f"FROM {store['table']} {store['alias']} {store.get('join', '')} "
|
| 4691 |
+
f"WHERE {sem.render_scope(store['scope_sql'], cur=cur)} GROUP BY {partner}").fetchall())
|
| 4692 |
pids = [int(row[JOIN_KEY]) for row in read_customers(cur)]
|
| 4693 |
# The oracle stays on invoice_date when a mutant redirects the served binding to orders.
|
| 4694 |
independent = sem.customer_rollup_oracle('customer_days_since_invoice', client,
|
|
|
|
| 4720 |
value = cur.execute(
|
| 4721 |
f"SELECT {sem._measure_sql(metric, store['alias'])} FROM {store['table']} "
|
| 4722 |
f"{store['alias']} {store.get('join', '')} "
|
| 4723 |
+
f"WHERE {sem.render_scope(store['scope_sql'], cur=cur)}").fetchone()[0]
|
| 4724 |
independent = sem.customer_rollup_oracle(bag['measure'], client)
|
| 4725 |
ok = (str(value or '')[:10] == str(independent or '')[:10]
|
| 4726 |
if metric.get('format') == 'date' else
|
|
|
|
| 5945 |
return _dt.datetime(day.year, day.month, day.day, hour, minute, tzinfo=zone).timestamp()
|
| 5946 |
|
| 5947 |
|
| 5948 |
+
#: ⛔ A staging Space runs every scheduled pass this long after the configured slot, so staging and
|
| 5949 |
+
#: live never call Odoo or SP-API together. A code rule: a staging store reseeded from live keeps it.
|
| 5950 |
+
STAGING_SLOT_OFFSET_SECONDS = 3600
|
| 5951 |
+
|
| 5952 |
+
|
| 5953 |
+
def slot_offset_seconds():
|
| 5954 |
+
"""The scheduled-pass shift for this deployment: the staging offset on staging, 0 on live or local."""
|
| 5955 |
+
try:
|
| 5956 |
+
import sync_alerts
|
| 5957 |
+
return STAGING_SLOT_OFFSET_SECONDS if sync_alerts.environment() == "STAGING" else 0
|
| 5958 |
+
except Exception: # noqa: BLE001
|
| 5959 |
+
return 0
|
| 5960 |
+
|
| 5961 |
+
|
| 5962 |
def slot_before(now, slot):
|
| 5963 |
+
"""The latest occurrence of the slot at or before epoch `now`, after this deployment's offset."""
|
| 5964 |
+
zone, shift = sync_zone(slot["tz"]), slot_offset_seconds()
|
| 5965 |
+
day = _dt.datetime.fromtimestamp(float(now) - shift, zone).date()
|
| 5966 |
for back in range(4):
|
| 5967 |
+
at = _slot_on(day - _dt.timedelta(days=back), slot, zone) + shift
|
| 5968 |
if at <= float(now):
|
| 5969 |
return at
|
| 5970 |
raise ValueError("no slot occurrence found")
|
| 5971 |
|
| 5972 |
|
| 5973 |
def slot_after(now, slot):
|
| 5974 |
+
"""The first occurrence of the slot strictly after epoch `now`, after this deployment's offset."""
|
| 5975 |
+
zone, shift = sync_zone(slot["tz"]), slot_offset_seconds()
|
| 5976 |
+
day = _dt.datetime.fromtimestamp(float(now) - shift, zone).date()
|
| 5977 |
for ahead in range(4):
|
| 5978 |
+
at = _slot_on(day + _dt.timedelta(days=ahead), slot, zone) + shift
|
| 5979 |
if at > float(now):
|
| 5980 |
return at
|
| 5981 |
raise ValueError("no slot occurrence found")
|
| 5982 |
|
| 5983 |
|
| 5984 |
+
def slot_run_clock(at):
|
| 5985 |
+
"""The `HH:MM` a configured slot `at` runs at on this deployment, or `at` unchanged when unreadable."""
|
| 5986 |
+
try:
|
| 5987 |
+
hour, minute = (int(part) for part in str(at).split(":"))
|
| 5988 |
+
total = (hour * 60 + minute + slot_offset_seconds() // 60) % 1440
|
| 5989 |
+
return f"{total // 60:02d}:{total % 60:02d}"
|
| 5990 |
+
except Exception: # noqa: BLE001
|
| 5991 |
+
return at
|
| 5992 |
+
|
| 5993 |
+
|
| 5994 |
#: A first-load continuation does not start this close to the next slot, so it cannot delay the slot.
|
| 5995 |
FIRST_LOAD_SLOT_QUIET_SECONDS = 2 * 3600
|
| 5996 |
|
| 5997 |
+
#: Retained for older scheduling callers. Freshness uses the stored pass budget.
|
| 5998 |
SLOT_LATE_SECONDS = 90 * 60
|
| 5999 |
+
PASS_BUDGET_SECONDS = 30 * 60
|
| 6000 |
|
| 6001 |
|
| 6002 |
def slot_near(now, slot, seconds=FIRST_LOAD_SLOT_QUIET_SECONDS):
|
|
|
|
| 6142 |
return str(defn.get("label") or key) if isinstance(defn, dict) else key
|
| 6143 |
|
| 6144 |
def _plain(text, limit):
|
| 6145 |
+
return " ".join(_dash_plain(text).split())[:limit]
|
|
|
|
| 6146 |
|
| 6147 |
failed = []
|
| 6148 |
for item in failed_databases or ():
|
|
|
|
| 6355 |
return max(secs, env)
|
| 6356 |
|
| 6357 |
|
| 6358 |
+
def _filling_databases(rt, cfg, now):
|
| 6359 |
+
"""Read column-fill progress from the tenant store, including after a worker restart."""
|
| 6360 |
+
from harness import datastore
|
| 6361 |
+
try:
|
| 6362 |
+
saved = rt.get(datastore.ODOO_FILL_STATUS_KEY, fresh=True) if rt is not None else None
|
| 6363 |
+
except Exception: # noqa: BLE001
|
| 6364 |
+
saved = None
|
| 6365 |
+
entities = saved.get("entities", {}) if isinstance(saved, dict) else {}
|
| 6366 |
+
if not isinstance(entities, dict):
|
| 6367 |
+
return []
|
| 6368 |
+
result = []
|
| 6369 |
+
for entity, connections in entities.items():
|
| 6370 |
+
if not isinstance(connections, dict):
|
| 6371 |
+
continue
|
| 6372 |
+
parts = [part for part in connections.values()
|
| 6373 |
+
if isinstance(part, dict) and part.get("state") in ("filling", "done")]
|
| 6374 |
+
if not any(part.get("state") == "filling" for part in parts):
|
| 6375 |
+
continue
|
| 6376 |
+
filling = [part for part in parts if part.get("state") == "filling"]
|
| 6377 |
+
active = any(part.get("active") is True for part in filling)
|
| 6378 |
+
resumes = [at for part in filling if (at := _parse_iso(part.get("resumeAt"))) is not None]
|
| 6379 |
+
resume_at = None
|
| 6380 |
+
if not cfg["frozen"] and not active:
|
| 6381 |
+
resume_at = (min(resumes).astimezone(_dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
| 6382 |
+
if resumes else next_sync_at(cfg, now.timestamp()))
|
| 6383 |
+
model = (datastore.ENTITIES.get(entity) or {}).get("model")
|
| 6384 |
+
tables = [(key, label) for _bucket, key, label, _fields in TABLES
|
| 6385 |
+
if model and ODOO_TABLE_MODELS.get(key) == model]
|
| 6386 |
+
tables.extend((key, spec["topicKey"].removeprefix("odoo_").replace("_", " ").capitalize())
|
| 6387 |
+
for key, spec in CANONICAL_RECEIPT_SPECS.items() if model and spec["model"] == model)
|
| 6388 |
+
for key, label in tables or [(entity, entity.replace("_", " "))]:
|
| 6389 |
+
result.append({"key": key, "label": label, "entity": entity, "state": "filling",
|
| 6390 |
+
"rowsDone": sum(max(0, int(part.get("rowsDone") or 0)) for part in parts),
|
| 6391 |
+
"rowsTotal": sum(max(0, int(part.get("rowsTotal") or 0)) for part in parts),
|
| 6392 |
+
"active": active, "resumeAt": resume_at})
|
| 6393 |
+
return result
|
| 6394 |
+
|
| 6395 |
+
|
| 6396 |
+
def _sync_pass_freshness(entry, cfg, now):
|
| 6397 |
+
"""Read duration, daily receipts and resync state from one stored C-status entry."""
|
| 6398 |
+
entry = entry if isinstance(entry, dict) else {}
|
| 6399 |
+
saved = entry.get("pass")
|
| 6400 |
+
saved = dict(saved) if isinstance(saved, dict) else {}
|
| 6401 |
+
|
| 6402 |
+
def number(value):
|
| 6403 |
+
if isinstance(value, (int, float)) and not isinstance(value, bool) and _math.isfinite(value) and value >= 0:
|
| 6404 |
+
return float(value)
|
| 6405 |
+
return None
|
| 6406 |
+
|
| 6407 |
+
budget = number(saved.get("budgetSeconds")) or PASS_BUDGET_SECONDS
|
| 6408 |
+
duration = number(saved.get("durationSeconds"))
|
| 6409 |
+
started = _parse_iso(saved.get("startedAt"))
|
| 6410 |
+
ended = _parse_iso(saved.get("endedAt"))
|
| 6411 |
+
if duration is None and started is not None:
|
| 6412 |
+
end = now if saved.get("outcome") == "running" else ended
|
| 6413 |
+
if end is not None:
|
| 6414 |
+
duration = max(0.0, (end - started).total_seconds())
|
| 6415 |
+
late = not cfg["frozen"] and (saved.get("outcome") == "failed"
|
| 6416 |
+
or duration is not None and duration > budget)
|
| 6417 |
+
late_slot = None
|
| 6418 |
+
slot = sync_slot(cfg)
|
| 6419 |
+
if late and slot is not None:
|
| 6420 |
+
late_slot = slot_iso(slot_before((started or ended or now).timestamp(), slot))
|
| 6421 |
+
receipts = entry.get("dailyPasses")
|
| 6422 |
+
receipts = dict(receipts) if isinstance(receipts, dict) else {}
|
| 6423 |
+
day = (slot_iso(slot_before(now.timestamp(), slot))[:10] if slot is not None
|
| 6424 |
+
else now.astimezone(_dt.timezone.utc).date().isoformat())
|
| 6425 |
+
daily = receipts.get(day)
|
| 6426 |
+
return {"pass": saved or None,
|
| 6427 |
+
"passMinutes": round(duration / 60, 3) if duration is not None else None,
|
| 6428 |
+
"budgetMinutes": budget / 60, "dailyPass": dict(daily) if isinstance(daily, dict) else None,
|
| 6429 |
+
"dailyPasses": receipts,
|
| 6430 |
+
"resync": dict(entry["resync"]) if isinstance(entry.get("resync"), dict) else {},
|
| 6431 |
+
"late": bool(late), "lateSlotAt": late_slot}
|
| 6432 |
+
|
| 6433 |
+
|
| 6434 |
def sync_freshness(rt, now=None):
|
| 6435 |
"""C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
|
| 6436 |
fail. PURE over the store reads.
|
|
|
|
| 6439 |
`STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
|
| 6440 |
(frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
|
| 6441 |
the last run's own reason, its code prefix removed, when that run ended in error.
|
| 6442 |
+
`late` means a failed pass or a pass that exceeds its stored duration budget.
|
| 6443 |
"""
|
| 6444 |
now = now or _utc_now()
|
| 6445 |
if isinstance(now, str):
|
|
|
|
| 6459 |
except Exception: # noqa: BLE001
|
| 6460 |
status = None
|
| 6461 |
status = status if isinstance(status, dict) else {}
|
| 6462 |
+
from core import connector_sync
|
| 6463 |
+
reader = getattr(connector_sync, "status", None)
|
| 6464 |
+
canonical = reader(rt, "odoo", fresh=True) if reader is not None and rt is not None else None
|
| 6465 |
+
canonical = canonical if isinstance(canonical, dict) else None
|
| 6466 |
+
pass_status = _sync_pass_freshness(canonical, cfg, now)
|
| 6467 |
phase = str(status.get("phase") or "unknown")
|
| 6468 |
+
last_good = (canonical.get("lastGoodSyncAt") if canonical is not None else raw.get("lastGoodSyncAt")) or None
|
| 6469 |
# Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
|
| 6470 |
+
if canonical is None and not last_good and phase == "ok":
|
| 6471 |
last_good = status.get("lastRun") or None
|
| 6472 |
every = _effective_sync_seconds(cfg["syncEvery"])
|
| 6473 |
allowed = None if every is None else every + STALE_GRACE_SECONDS
|
|
|
|
| 6475 |
if allowed is not None and not cfg["frozen"]:
|
| 6476 |
good_at = _parse_iso(last_good)
|
| 6477 |
stale = good_at is None or (now - good_at).total_seconds() > allowed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6478 |
failure = None
|
| 6479 |
errors = status.get("errors") if isinstance(status.get("errors"), list) else []
|
| 6480 |
if phase == "error" and errors:
|
|
|
|
| 6484 |
code, message = "sync_failed", text
|
| 6485 |
failure = {"code": code.strip()[:80], "message": message.strip()[:300],
|
| 6486 |
"at": status.get("lastRun") or None}
|
| 6487 |
+
last_run = status.get("lastRun") or None
|
| 6488 |
+
if canonical is not None:
|
| 6489 |
+
saved_pass = pass_status["pass"] or {}
|
| 6490 |
+
outcome = saved_pass.get("outcome")
|
| 6491 |
+
phase = {"done": "ok", "failed": "error", "running": "syncing"}.get(outcome, outcome or "unknown")
|
| 6492 |
+
last_run = saved_pass.get("endedAt") or saved_pass.get("startedAt")
|
| 6493 |
+
detail = " ".join(_dash_plain(saved_pass.get("detail") or "Sync failed.").split())
|
| 6494 |
+
failure = ({"code": "sync_failed", "message": detail[:300],
|
| 6495 |
+
"at": last_run} if outcome == "failed" else None)
|
| 6496 |
# D-2679: a database that failed while its siblings synced is reported by itself.
|
| 6497 |
failed_databases = []
|
| 6498 |
for item in raw.get("failedDatabases") if isinstance(raw.get("failedDatabases"), list) else ():
|
| 6499 |
if isinstance(item, dict) and item.get("key"):
|
| 6500 |
failed_databases.append({
|
| 6501 |
+
name: (" ".join(_dash_plain(item.get(name)).split())[:300] or None)
|
|
|
|
| 6502 |
for name in ("key", "label", "code", "message", "stage", "at")
|
| 6503 |
+ (("cause",) if item.get("cause") else ())})
|
| 6504 |
return {"stale": bool(stale), "lastGoodSyncAt": last_good,
|
| 6505 |
+
**pass_status,
|
| 6506 |
"failedDatabases": failed_databases,
|
| 6507 |
+
"fillingDatabases": _filling_databases(rt, cfg, now),
|
| 6508 |
"schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
|
| 6509 |
+
"lastRunAt": last_run, "phase": phase,
|
| 6510 |
"frozen": cfg["frozen"], "failure": failure,
|
| 6511 |
"syncAt": cfg["syncAt"], "syncTz": cfg["syncTz"],
|
| 6512 |
"nextSyncAt": None if cfg["frozen"] else next_sync_at(cfg, now.timestamp())}
|
|
|
|
| 6828 |
"state": ("sp.state", _stock_text),
|
| 6829 |
"scheduled_date": (date_sql("sp.scheduled_date"), _as_date),
|
| 6830 |
"date_done": (date_sql("sp.date_done"), _as_date),
|
| 6831 |
+
"order_date": (__import__('routes_odoo_tables')._business_day_sql("so.date_order"), _as_date),
|
| 6832 |
"delivery_days": ("epoch(TRY_CAST(sp.date_done AS TIMESTAMPTZ) - "
|
| 6833 |
"TRY_CAST(so.date_order AS TIMESTAMPTZ)) / 86400.0", number),
|
| 6834 |
"refreshed": ("(SELECT substr(updated_at, 1, 10) FROM _sync_state "
|
|
|
|
| 7165 |
# ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a
|
| 7166 |
# column a user added to a locked database is never rewritten.
|
| 7167 |
stored = have[fkey]
|
| 7168 |
+
if (field.get('automation') or {}).get('renamable'):
|
| 7169 |
+
repaired = ut.keep_user_edits(stored, {**stored, **field})
|
| 7170 |
+
stored.clear()
|
| 7171 |
+
stored.update(repaired)
|
| 7172 |
+
continue
|
| 7173 |
# ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp,
|
| 7174 |
# same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the
|
| 7175 |
# loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on
|
|
|
|
| 7178 |
continue
|
| 7179 |
if (stored.get("automation") or {}).get("preset"):
|
| 7180 |
for prop in ("label", "type", "options", "choicesFrom", "link", "rollup", "description",
|
| 7181 |
+
"agg", "pinned", "default", "personal"):
|
| 7182 |
# A subsequent row-only refresh must not replace authoritative selections with
|
| 7183 |
# an older compiled vocabulary while its metadata catalogue is unavailable.
|
| 7184 |
if (stored.get("choicesFrom") == "odoo" and field.get("choicesFrom") != "odoo"
|
|
|
|
| 7331 |
|
| 7332 |
def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, today,
|
| 7333 |
on_progress):
|
| 7334 |
+
"""Publish a complete enabled population through the tenant's existing durable mirror."""
|
| 7335 |
+
from harness import datastore, mirror_store
|
| 7336 |
+
from core import connector_sync
|
| 7337 |
+
from uuid import uuid4
|
| 7338 |
ut = _ut()
|
| 7339 |
key, model = choice["key"], choice["model"]
|
| 7340 |
+
public_cid = cid = connection_for_table_key(key, rt)
|
| 7341 |
+
if cid == DEFAULT_CONNECTION and rt.key == "royal-imports":
|
| 7342 |
+
cid = "legacy-env"
|
| 7343 |
+
identity = datastore.odoo_dynamic_model_identity(rt.key, cid, model, key)
|
| 7344 |
+
if getattr(client, "READ_ONLY", True) is not True:
|
| 7345 |
+
raise Refused("Dynamic Odoo populations require the read-only source transport.")
|
| 7346 |
+
domain = ([("active", "in", [True, False])]
|
| 7347 |
+
if (datastore.ENTITIES.get(identity["source"], {}).get("archivable")
|
| 7348 |
+
or (catalogue_row.get("fields", {}).get("active") or {}).get("type") == "boolean") else [])
|
| 7349 |
+
total = client.search_count(model, domain)
|
| 7350 |
+
if type(total) is not int or total < 0:
|
| 7351 |
+
raise Refused("The Odoo population count is unavailable. The prior database was preserved.")
|
| 7352 |
+
stored_counts = ut.definition_counts(st=rt)
|
| 7353 |
+
if key not in stored_counts and ut.get_definition(key, st=rt) is not None:
|
| 7354 |
+
raise Refused("The prior database row count is unavailable. The prior database was preserved.")
|
| 7355 |
+
stored_n = int(stored_counts.get(key) or 0)
|
| 7356 |
fields, stamp = _model_field_specs(catalogue_row), today or _iso_today()
|
| 7357 |
+
metadata = {"model": model, "fields": {
|
| 7358 |
+
name: {prop: meta[prop] for prop in ("type", "relation") if prop in meta}
|
| 7359 |
+
for name, meta in (catalogue_row.get("fields") or {}).items()}}
|
| 7360 |
+
source = identity["source"]
|
| 7361 |
+
marker = source + ".dynamic." + key
|
| 7362 |
+
with mirror_store.sync_mirror("odoo", rt.key, rt, expected_tables=[]) as con:
|
| 7363 |
+
datastore.register_odoo_dynamic_model(rt, con, connection_id=cid, model=model, table_key=key)
|
| 7364 |
+
static = datastore.ENTITIES.get(source)
|
| 7365 |
+
if static:
|
| 7366 |
+
datastore._ensure_table(con, source, static)
|
| 7367 |
+
else:
|
| 7368 |
+
types = {"boolean": "BOOLEAN", "integer": "BIGINT", "float": "DOUBLE", "monetary": "DOUBLE"}
|
| 7369 |
+
columns = [f'{_model_ident(f["key"])} {types.get((metadata["fields"].get(f["key"]) or {}).get("type"), "VARCHAR")}'
|
| 7370 |
+
for f in fields if f["key"] != "id"
|
| 7371 |
+
and (metadata["fields"].get(f["key"]) or {}).get("type") != "binary"]
|
| 7372 |
+
con.execute(f'CREATE TABLE IF NOT EXISTS {_model_ident(source)} '
|
| 7373 |
+
'(id BIGINT PRIMARY KEY, Store VARCHAR, _connection_id VARCHAR, _source_id BIGINT'
|
| 7374 |
+
+ (", " + ", ".join(columns) if columns else "") + ")")
|
| 7375 |
+
# Resolve every declared field before touching a candidate's prior rows.
|
| 7376 |
+
_dynamic_model_spec(con, identity, metadata)
|
| 7377 |
+
prior = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?", [marker]).fetchone()
|
| 7378 |
+
prior_meta = _json.loads(prior[2]) if prior and prior[2] else {}
|
| 7379 |
+
baseline = con.execute(f'SELECT count(*) FROM {_model_ident(source)} '
|
| 7380 |
+
'WHERE _connection_id=? OR _connection_id IS NULL', [cid]).fetchone()[0]
|
| 7381 |
+
baseline = max(stored_n, baseline, int(prior_meta.get("baseline") or 0))
|
| 7382 |
+
if baseline and total < baseline * MAX_SHRINK:
|
| 7383 |
+
raise Refused(f"{model}: source count is {total} rows against {baseline} stored. "
|
| 7384 |
+
"The prior database was preserved. Check Odoo read access and retry sync.")
|
| 7385 |
+
resume = bool(prior and prior[0] == "backfill" and prior_meta.get("schema") == metadata
|
| 7386 |
+
and prior_meta.get("total") == total)
|
| 7387 |
+
last, count = (int(prior[1]), int(prior[3])) if resume else (0, 0)
|
| 7388 |
+
checkpoint = _json.dumps({"schema": metadata, "total": total, "baseline": baseline,
|
| 7389 |
+
"passId": prior_meta.get("passId") if resume else uuid4().hex}, sort_keys=True)
|
| 7390 |
+
entry = {"id": cid, "label": connection_label(rt, cid) or "Odoo", "creds": None}
|
| 7391 |
+
with datastore._odoo_connection(con, rt, entry, client_factory=lambda _creds: client) as binding:
|
| 7392 |
+
binding["relations"][model] = {name: (meta["relation"], meta["type"])
|
| 7393 |
+
for name, meta in metadata["fields"].items()
|
| 7394 |
+
if meta.get("relation") and meta.get("type") in {"many2one", "one2many", "many2many"}}
|
| 7395 |
+
if not resume:
|
| 7396 |
+
con.execute(f'DELETE FROM {_model_ident(source)} WHERE _connection_id=?', [cid])
|
| 7397 |
+
names = [f["key"] for f in fields if (metadata["fields"].get(f["key"]) or {}).get("type") != "binary"]
|
| 7398 |
+
requested = list(dict.fromkeys([*names, *(static["fields"] if static else ())]))
|
| 7399 |
+
def save(phase):
|
| 7400 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 7401 |
+
[marker, phase, last, checkpoint, count, _utc_now().isoformat()])
|
| 7402 |
+
save("backfill")
|
| 7403 |
+
while True:
|
| 7404 |
+
page = client.search_read(model, [*domain, ("id", ">", last)], requested,
|
| 7405 |
+
limit=MODEL_READ_PAGE, order="id")
|
| 7406 |
+
if not page:
|
| 7407 |
+
break
|
| 7408 |
+
ids = [item.get("id") for item in page]
|
| 7409 |
+
if (any(type(rid) is not int or rid <= last for rid in ids)
|
| 7410 |
+
or ids != sorted(set(ids))):
|
| 7411 |
+
raise Refused("Odoo returned an incomplete or unstable ID page. The prior database was preserved.")
|
| 7412 |
+
con.execute("BEGIN TRANSACTION")
|
| 7413 |
+
try:
|
| 7414 |
+
if static:
|
| 7415 |
+
datastore._upsert(con, source, static, page)
|
| 7416 |
+
else:
|
| 7417 |
+
mapped = datastore._connection_ids(con, model, ids)
|
| 7418 |
+
physical = ["id", "Store", "_connection_id", "_source_id", *[n for n in names if n != "id"]]
|
| 7419 |
+
values = [[mapped[item["id"]], entry["label"], cid, item["id"],
|
| 7420 |
+
*[_model_mirror_cell(item.get(n), metadata["fields"].get(n) or {})
|
| 7421 |
+
for n in names if n != "id"]] for item in page]
|
| 7422 |
+
con.executemany(f'INSERT INTO {_model_ident(source)} '
|
| 7423 |
+
f'({", ".join(map(_model_ident, physical))}) VALUES ({", ".join("?" for _ in physical)})', values)
|
| 7424 |
+
last, count = ids[-1], count + len(page)
|
| 7425 |
+
save("backfill")
|
| 7426 |
+
con.execute("COMMIT")
|
| 7427 |
+
except BaseException:
|
| 7428 |
+
con.execute("ROLLBACK")
|
| 7429 |
+
raise
|
| 7430 |
+
if on_progress is not None:
|
| 7431 |
+
on_progress(count, total)
|
| 7432 |
+
after = client.search_count(model, domain)
|
| 7433 |
+
if type(after) is not int or count != total or after != total:
|
| 7434 |
+
raise Refused("Odoo returned an incomplete or changed population. The prior database was preserved.")
|
| 7435 |
+
save("live")
|
| 7436 |
+
datastore._save_state(con, source, "live", last, None)
|
| 7437 |
+
con, _path = datastore._mirror_connection(rt.key, rt)
|
| 7438 |
try:
|
| 7439 |
+
spec = dynamic_model_source(con, key, tenant_key=rt.key)
|
| 7440 |
+
publication = datastore._publication(con, "odoo")
|
| 7441 |
+
published_state = con.execute("SELECT cursor_wd FROM _sync_state WHERE entity=? AND phase='live'", [marker]).fetchone()
|
| 7442 |
+
if spec is None or published_state != (checkpoint,):
|
| 7443 |
+
raise Refused("The complete dynamic Odoo publication is unavailable. The prior database was preserved.")
|
| 7444 |
+
# The mounted reader must resolve the same tenant source before generated rows retire.
|
| 7445 |
+
import routes_odoo_tables
|
| 7446 |
+
admitted = routes_odoo_tables._source_for(con, key)
|
| 7447 |
+
planned_definition = {**(ut.get_definition(key, st=rt) or {}), "readThrough": True}
|
| 7448 |
+
planned = ut.lend(rt, **{ut.STORE_KEY: {key: planned_definition}})
|
| 7449 |
+
if (admitted is None or admitted.get("table") != source or admitted.get("where") != spec["where"]
|
| 7450 |
+
or routes_odoo_tables._materialised_table(key, cur=con, st=planned)):
|
| 7451 |
+
raise Refused("The dynamic Odoo read-through binding is unavailable. The prior database was preserved.")
|
| 7452 |
+
finally:
|
| 7453 |
+
con.close()
|
| 7454 |
+
written = {}
|
| 7455 |
+
def publish(current):
|
| 7456 |
+
current = current if isinstance(current, dict) else {}
|
| 7457 |
+
prior = current.get(key) or {}
|
| 7458 |
+
authored = {f["key"] for f in prior.get("fields", ())
|
| 7459 |
+
if not (f.get("automation") or {}).get("preset") or ut.user_edited(f)}
|
| 7460 |
+
if any(any(name in authored for name in values) for values in (prior.get("rows") or {}).values()):
|
| 7461 |
+
raise Refused("Stored authored cells must retain their overlay before generated rows can retire.")
|
| 7462 |
+
if key in current:
|
| 7463 |
+
current[key]["readThrough"] = True
|
| 7464 |
+
counts = _ensure_table_inplace(current, key, choice["label"], fields, None, username, stamp)
|
| 7465 |
+
table = current[key]
|
| 7466 |
+
table["connector"] = {**connector_sync.database_identity("odoo", public_cid, key, model),
|
| 7467 |
+
"key": "odoo", "readOnly": True, "mirrorSource": source,
|
| 7468 |
+
"snapshotId": publication["snapshot_id"]}
|
| 7469 |
+
table["readThrough"], table["rows"] = True, {}
|
| 7470 |
+
table.pop(ut.OVERSIZE_FIELD, None)
|
| 7471 |
+
written[key] = {**counts, "rows": count}
|
| 7472 |
+
return current
|
| 7473 |
+
rt.update(ut.STORE_KEY, publish, flush="sync")
|
| 7474 |
+
ut.register_connected(key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7475 |
return {"tables": written, "errors": []}
|
| 7476 |
|
| 7477 |
|
| 7478 |
+
def _model_ident(name):
|
| 7479 |
+
"""Quote a native field identifier without accepting SQL syntax."""
|
| 7480 |
+
name = str(name)
|
| 7481 |
+
if not name or not name.replace("_", "").isalnum() or name[0].isdigit():
|
| 7482 |
+
raise Refused("The Odoo field name cannot be represented in the mirror.")
|
| 7483 |
+
return '"' + name + '"'
|
| 7484 |
+
|
| 7485 |
+
|
| 7486 |
+
def _model_mirror_cell(value, meta):
|
| 7487 |
+
"""Preserve typed scalar values and the existing durable relation/list representation."""
|
| 7488 |
+
if value is False and meta.get("type") != "boolean":
|
| 7489 |
+
return None
|
| 7490 |
+
return _model_cell(value)
|
| 7491 |
+
|
| 7492 |
+
|
| 7493 |
+
def _dynamic_model_spec(cur, identity, metadata):
|
| 7494 |
+
"""Bind every visible field to a validated connection-local mirror projection."""
|
| 7495 |
+
from harness import datastore
|
| 7496 |
+
source, cid = identity["source"], identity["connection_id"]
|
| 7497 |
+
have = datastore.columns_of(source, cur=cur) or set()
|
| 7498 |
+
specs = {}
|
| 7499 |
+
static = source in datastore.ENTITIES
|
| 7500 |
+
for field in _model_field_specs(metadata):
|
| 7501 |
+
name = field["key"]
|
| 7502 |
+
meta = (metadata.get("fields") or {}).get(name) or {}
|
| 7503 |
+
expr = "_source_id" if name == "id" else _model_ident(name)
|
| 7504 |
+
if meta.get("type") == "binary":
|
| 7505 |
+
expr = "NULL"
|
| 7506 |
+
elif name != "id" and name.lower() not in have:
|
| 7507 |
+
raise Refused(f"{identity['model']}: field {name!r} has no complete mirror binding. The prior database was preserved.")
|
| 7508 |
+
elif static and meta.get("type") == "many2one":
|
| 7509 |
+
label = name.removesuffix("_id") + "_name"
|
| 7510 |
+
if label.lower() not in have:
|
| 7511 |
+
raise Refused("The Odoo relation has no complete mirror binding. The prior database was preserved.")
|
| 7512 |
+
relation = str(meta.get("relation") or "").replace("'", "''")
|
| 7513 |
+
native = (f"coalesce((SELECT r.source_id FROM odoo_record_ids r WHERE r.connection_id='"
|
| 7514 |
+
+ cid.replace("'", "''") + f"' AND r.model='{relation}' AND r.id={expr}), {expr})")
|
| 7515 |
+
expr = (f"CASE WHEN {expr} IS NULL THEN NULL ELSE '[' || CAST({native} AS VARCHAR) || ', ' || "
|
| 7516 |
+
f"CAST(to_json(coalesce({_model_ident(label)}, '')) AS VARCHAR) || ']' END")
|
| 7517 |
+
specs[name] = (expr, lambda value, kind=field["type"]: _registry_col_value(value, kind))
|
| 7518 |
+
specs.update({"_connection_id": ("_connection_id", str), "Store": ("Store", str)})
|
| 7519 |
+
return {"table": source, "id": "_source_id", "page_key": "_source_id",
|
| 7520 |
+
"where": "_connection_id = '" + cid.replace("'", "''") + "'",
|
| 7521 |
+
"cols": specs, "mirror_only": True}
|
| 7522 |
+
|
| 7523 |
+
|
| 7524 |
+
def dynamic_model_source(cur, table_key, *, tenant_key=None):
|
| 7525 |
+
"""Resolve a complete dynamic source from this cursor's saved roster and checkpoint."""
|
| 7526 |
+
from harness import datastore
|
| 7527 |
+
if not str(table_key).startswith("ut_odoo_model_"):
|
| 7528 |
+
return None
|
| 7529 |
+
if not datastore.columns_of("odoo_dynamic_models", cur=cur):
|
| 7530 |
+
return None
|
| 7531 |
+
row = cur.execute("SELECT tenant, connection_id, model, source FROM odoo_dynamic_models WHERE table_key=?",
|
| 7532 |
+
[table_key]).fetchone()
|
| 7533 |
+
if row is None:
|
| 7534 |
+
return None
|
| 7535 |
+
tenant, cid, model, source = row
|
| 7536 |
+
handle = getattr(cur, "_mirror_handle", None)
|
| 7537 |
+
if (tenant_key is not None and tenant_key != tenant) or (handle is not None and handle.tenant_key != tenant):
|
| 7538 |
+
raise Refused("The dynamic Odoo binding belongs to a different tenant.")
|
| 7539 |
+
identity = datastore.odoo_dynamic_model_identity(tenant, cid, model, table_key)
|
| 7540 |
+
publication = datastore._publication(cur, "odoo")
|
| 7541 |
+
if source != identity["source"] or source not in (publication.get("tables") or {}):
|
| 7542 |
+
return None
|
| 7543 |
+
marker = source + ".dynamic." + table_key
|
| 7544 |
+
saved = cur.execute("SELECT phase, cursor_wd, rows FROM _sync_state WHERE entity=?", [marker]).fetchone()
|
| 7545 |
+
if saved is None or saved[0] != "live":
|
| 7546 |
+
return None
|
| 7547 |
+
metadata = _json.loads(saved[1])
|
| 7548 |
+
if metadata.get("total") != saved[2] or metadata.get("schema", {}).get("model") != model:
|
| 7549 |
+
return None
|
| 7550 |
+
return _dynamic_model_spec(cur, identity, metadata["schema"])
|
| 7551 |
+
|
| 7552 |
+
|
| 7553 |
def sync_discovered_models(rt, source_connection_id=None, namespace=None,
|
| 7554 |
username="automation", today=None, table_keys=None, on_progress=None):
|
| 7555 |
+
"""Publish explicitly enabled long-tail models through their tenant-bound durable mirror.
|
| 7556 |
|
| 7557 |
Disabled models are absent from this pass rather than represented by an empty row set. That
|
| 7558 |
distinction is the preservation guarantee: no updater touches their definition, rows, fields,
|
| 7559 |
grants, ownership, or tenant rename, and re-enabling resumes against the same stable key.
|
| 7560 |
+
Progress describes private checkpoints. Only a complete population becomes readable.
|
|
|
|
| 7561 |
"""
|
| 7562 |
requested = set(table_keys) if table_keys is not None else None
|
| 7563 |
if requested is not None and not requested:
|
|
|
|
| 7572 |
and (requested is None or choice["key"] in requested)]
|
| 7573 |
if not selected:
|
| 7574 |
return {}
|
| 7575 |
+
_cid, expected_namespace = connection_namespace(rt, source_connection_id)
|
| 7576 |
+
if namespace != expected_namespace or any(
|
| 7577 |
+
split_table_key(choice["key"])[1] != namespace for choice in selected):
|
| 7578 |
+
raise Refused("The dynamic Odoo databases belong to a different connection.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7579 |
source =(rt.odoo_source() if source_connection_id is None
|
| 7580 |
else rt.odoo_source(source_connection_id))
|
| 7581 |
if source is None:
|
| 7582 |
raise Refused("the selected Odoo connection is unavailable, so no discovered model was read")
|
| 7583 |
+
written, errors = {}, []
|
| 7584 |
from core.odoo import get_odoo
|
| 7585 |
with source._bound(): # tenant credential + scope
|
| 7586 |
client = get_odoo()
|
|
|
|
| 7588 |
return _publish_discovered_model(
|
| 7589 |
rt, client, selected[0], by_model[str(selected[0]["model"])],
|
| 7590 |
username=username, today=today, on_progress=on_progress)
|
| 7591 |
+
for choice in selected:
|
| 7592 |
model = str(choice["model"])
|
| 7593 |
row = by_model[model]
|
| 7594 |
try:
|
| 7595 |
+
result = _publish_discovered_model(rt, client, choice, row,
|
| 7596 |
+
username=username, today=today, on_progress=on_progress)
|
| 7597 |
except Exception as exc: # noqa: BLE001
|
| 7598 |
+
errors.append(f"{model}: {_scrub_cause(exc)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7599 |
continue
|
| 7600 |
+
written.update(result["tables"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7601 |
return {"tables": written, "errors": errors}
|
| 7602 |
|
| 7603 |
|
|
|
|
| 7789 |
text = _SCRUB_QUOTED.sub(
|
| 7790 |
lambda m: m.group(0) if _SCRUB_NAME.fullmatch(m.group(1) or m.group(2) or "")
|
| 7791 |
else "'<value>'", text)
|
| 7792 |
+
text = " ".join(_dash_plain(text).split())
|
| 7793 |
return (f"{name}: {text}" if text else name)[:300]
|
| 7794 |
|
| 7795 |
|
|
|
|
| 7922 |
|
| 7923 |
|
| 7924 |
def _canonical_mirror_ready(rt):
|
| 7925 |
+
# Every tenant's canonical totals require its own ready mirror, regardless of selection.
|
| 7926 |
+
return rt is None or rt.mirror_handle().ready()
|
|
|
|
|
|
|
| 7927 |
|
| 7928 |
|
| 7929 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
api/pages.py
CHANGED
|
@@ -261,12 +261,13 @@ def bu_control(session: Session, team_id):
|
|
| 261 |
"""The BU picker, built from the user's OWN permitted labels — which is exactly why `controls`
|
| 262 |
is assembled per request and never cached on a scope key."""
|
| 263 |
labels = perms.allowed_bu_labels(session.user)
|
|
|
|
| 264 |
options = []
|
| 265 |
for lab in labels:
|
| 266 |
if lab == "All":
|
| 267 |
options.append({"value": "all", "label": "All"})
|
| 268 |
continue
|
| 269 |
-
tid =
|
| 270 |
if tid is not None:
|
| 271 |
options.append({"value": tid, "label": lab})
|
| 272 |
return {"key": "bu", "kind": "bu", "label": "Business Unit",
|
|
@@ -279,10 +280,10 @@ def choice_control(key, label, value, options):
|
|
| 279 |
|
| 280 |
|
| 281 |
def bu_label(team_id):
|
| 282 |
-
"""'All Business Units'
|
| 283 |
if team_id is None:
|
| 284 |
return "All Business Units"
|
| 285 |
-
for lab, tid in ctxlib.
|
| 286 |
if tid == team_id:
|
| 287 |
return lab
|
| 288 |
return str(team_id)
|
|
@@ -371,7 +372,8 @@ def _metrics_for(rt, page_key, team_id, granularity, builder):
|
|
| 371 |
"""
|
| 372 |
import scope_cache
|
| 373 |
|
| 374 |
-
|
|
|
|
| 375 |
key = ("page", page_key, team_id, granularity, today)
|
| 376 |
|
| 377 |
def _build():
|
|
|
|
| 261 |
"""The BU picker, built from the user's OWN permitted labels — which is exactly why `controls`
|
| 262 |
is assembled per request and never cached on a scope key."""
|
| 263 |
labels = perms.allowed_bu_labels(session.user)
|
| 264 |
+
brands = ctxlib.brands()
|
| 265 |
options = []
|
| 266 |
for lab in labels:
|
| 267 |
if lab == "All":
|
| 268 |
options.append({"value": "all", "label": "All"})
|
| 269 |
continue
|
| 270 |
+
tid = brands.get(lab)
|
| 271 |
if tid is not None:
|
| 272 |
options.append({"value": tid, "label": lab})
|
| 273 |
return {"key": "bu", "kind": "bu", "label": "Business Unit",
|
|
|
|
| 280 |
|
| 281 |
|
| 282 |
def bu_label(team_id):
|
| 283 |
+
"""'All Business Units' or the team's label, for prose, never for a permission decision."""
|
| 284 |
if team_id is None:
|
| 285 |
return "All Business Units"
|
| 286 |
+
for lab, tid in ctxlib.brands().items():
|
| 287 |
if tid == team_id:
|
| 288 |
return lab
|
| 289 |
return str(team_id)
|
|
|
|
| 372 |
"""
|
| 373 |
import scope_cache
|
| 374 |
|
| 375 |
+
from core import periods
|
| 376 |
+
today = periods.today().isoformat()
|
| 377 |
key = ("page", page_key, team_id, granularity, today)
|
| 378 |
|
| 379 |
def _build():
|
api/process_role.py
CHANGED
|
@@ -5,16 +5,24 @@ Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs
|
|
| 5 |
never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
|
| 6 |
store in the background; `light` serves this process's own reads.
|
| 7 |
|
| 8 |
-
One
|
| 9 |
-
|
| 10 |
|
| 11 |
-
Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
One sync worker per tenant: the primary worker serves AIOS_SYNC_WRITER_TENANT (royal-imports when
|
| 15 |
unset) and runs every job that loops all tenants. Each other tenant that holds a sync connector
|
| 16 |
gets a tenant worker (AIOS_WORKER_SCOPE=tenant) with its own DuckDB file and lease, which runs only
|
| 17 |
-
that tenant's sync jobs. AIOS_SYNC_WORKERS_MAX caps the
|
|
|
|
| 18 |
"""
|
| 19 |
from __future__ import annotations
|
| 20 |
|
|
@@ -25,9 +33,11 @@ import shutil
|
|
| 25 |
import socket
|
| 26 |
import subprocess
|
| 27 |
import sys
|
|
|
|
| 28 |
import threading
|
| 29 |
import time
|
| 30 |
import uuid
|
|
|
|
| 31 |
from pathlib import Path
|
| 32 |
|
| 33 |
ROLES = ("web", "worker", "all", "reader")
|
|
@@ -47,6 +57,10 @@ PRIMARY_LOCK = "web-primary.lock"
|
|
| 47 |
PRIMARY_FILE = "web-primary.json"
|
| 48 |
INSTALL_ATTEMPTS = 3
|
| 49 |
INCOMPLETE_GRACE_SECONDS = 3600
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
|
| 52 |
def resolve_role(env=None):
|
|
@@ -78,6 +92,9 @@ _INSTALLED = {}
|
|
| 78 |
_INSTALL_FAILURES = {}
|
| 79 |
LAST_INSTALL = {}
|
| 80 |
LAST_INSTALL_ERROR = {}
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
|
| 83 |
def role():
|
|
@@ -126,7 +143,11 @@ def heartbeat_name(env=None):
|
|
| 126 |
|
| 127 |
|
| 128 |
def fleet_tenants(root=None):
|
| 129 |
-
"""Tenants with their own tenant worker
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
state = _read_json(Path(root or handoff_dir()) / FLEET_FILE)
|
| 131 |
tenants = state.get("tenants") if isinstance(state, dict) else None
|
| 132 |
return {str(slug) for slug in tenants} if isinstance(tenants, dict) else set()
|
|
@@ -143,7 +164,7 @@ def syncs_here(slug, root=None):
|
|
| 143 |
|
| 144 |
|
| 145 |
def sync_worker_cap(env=None):
|
| 146 |
-
"""AIOS_SYNC_WORKERS_MAX: sync
|
| 147 |
env = os.environ if env is None else env
|
| 148 |
try:
|
| 149 |
return max(1, int(env.get("AIOS_SYNC_WORKERS_MAX") or 4))
|
|
@@ -249,7 +270,7 @@ def writer_holder(env=None):
|
|
| 249 |
return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
|
| 250 |
|
| 251 |
|
| 252 |
-
def acquire_writer(rt, holder=None, *, now=None, ttl=None):
|
| 253 |
"""Take or renew the store's writer lease. True when `holder` holds it after the call.
|
| 254 |
|
| 255 |
The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
|
|
@@ -268,13 +289,14 @@ def acquire_writer(rt, holder=None, *, now=None, ttl=None):
|
|
| 268 |
doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
|
| 269 |
return doc
|
| 270 |
|
| 271 |
-
|
|
|
|
| 272 |
if not isinstance(doc, dict):
|
| 273 |
-
doc = rt.get(
|
| 274 |
return isinstance(doc, dict) and doc.get("holder") == holder
|
| 275 |
|
| 276 |
|
| 277 |
-
def release_writer(rt, holder=None):
|
| 278 |
"""Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
|
| 279 |
holder = holder or writer_holder()
|
| 280 |
|
|
@@ -284,7 +306,39 @@ def release_writer(rt, holder=None):
|
|
| 284 |
doc.update(holder=None, renewedAt=0, previous=holder)
|
| 285 |
return doc
|
| 286 |
|
| 287 |
-
rt.update(WRITER_LEASE_DOC, drop)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
|
| 289 |
|
| 290 |
def holds_writer():
|
|
@@ -312,7 +366,7 @@ def writer_lease_tick(rt_factory, holder, *, now=None, ttl=None):
|
|
| 312 |
stamp = time.time() if now is None else float(now)
|
| 313 |
ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
|
| 314 |
try:
|
| 315 |
-
held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl)
|
| 316 |
except Exception as exc: # noqa: BLE001
|
| 317 |
held = None
|
| 318 |
if _WRITER.get("error") != type(exc).__name__:
|
|
@@ -378,10 +432,11 @@ def arm_writer_lease(rt_factory=None, holder=None, exit_fn=None):
|
|
| 378 |
|
| 379 |
|
| 380 |
# ----------------------------------------------------------------------------- readiness (C9)
|
| 381 |
-
READY_CHECKS = ("mirror", "pools", "first_reads")
|
| 382 |
_READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
|
| 383 |
"pools": "The read pools are not open yet.",
|
| 384 |
-
"first_reads": "The first reads have not answered yet."
|
|
|
|
| 385 |
_READINESS = {}
|
| 386 |
_BOOT_RESTORE = {}
|
| 387 |
# The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
|
|
@@ -415,8 +470,43 @@ def note_boot_restore(state="done"):
|
|
| 415 |
print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
|
| 416 |
|
| 417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
def worker_boot_restored(root=None):
|
| 419 |
-
"""Web side: the
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 420 |
beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
|
| 421 |
restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
|
| 422 |
# A secondary web worker supervises nothing: the primary names the current sync worker.
|
|
@@ -432,10 +522,15 @@ def _installed_map(root=None):
|
|
| 432 |
|
| 433 |
|
| 434 |
def pending_handoffs(root=None):
|
| 435 |
-
"""`tenant/connector` for each
|
|
|
|
|
|
|
|
|
|
| 436 |
installed = _installed_map(root)
|
| 437 |
-
|
| 438 |
-
|
|
|
|
|
|
|
| 439 |
|
| 440 |
|
| 441 |
# ----------------------------------------------------------------------------- several web workers
|
|
@@ -476,7 +571,7 @@ def claim_primary(root=None):
|
|
| 476 |
os.close(fd)
|
| 477 |
return False
|
| 478 |
_PRIMARY["fd"] = fd
|
| 479 |
-
print(f"[process-role] web worker pid={os.getpid()} is the primary: it
|
| 480 |
f"and installs hand-offs", flush=True)
|
| 481 |
write_primary_state(root)
|
| 482 |
return True
|
|
@@ -493,7 +588,9 @@ def write_primary_state(root=None):
|
|
| 493 |
version = None
|
| 494 |
_write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
|
| 495 |
"pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
|
| 496 |
-
"installed": {f"{key[0]}/{key[1]}": snap for key, snap in _INSTALLED.items()},
|
|
|
|
|
|
|
| 497 |
"mirrorVersion": version})
|
| 498 |
return True
|
| 499 |
|
|
@@ -515,13 +612,21 @@ def readiness(mirror_live=None, role_name=None, root=None):
|
|
| 515 |
role_name = role_name or ROLE
|
| 516 |
checks, detail = {}, {}
|
| 517 |
for name in READY_CHECKS:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 518 |
row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
|
| 519 |
checks[name], detail[name] = bool(row["ok"]), row["detail"]
|
| 520 |
boot = role_name == "web" and _READY_LATCH["at"] is None
|
| 521 |
if "mirror" in checks and checks["mirror"]:
|
| 522 |
if mirror_live is False:
|
| 523 |
checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
|
| 524 |
-
elif boot and not worker_boot_restored(root):
|
| 525 |
checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
|
| 526 |
elif boot and pending_handoffs(root):
|
| 527 |
checks["mirror"] = False
|
|
@@ -556,7 +661,11 @@ def _read_json(path):
|
|
| 556 |
|
| 557 |
|
| 558 |
def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
|
| 559 |
-
"""Keep one
|
|
|
|
|
|
|
|
|
|
|
|
|
| 560 |
root = Path(root or handoff_dir())
|
| 561 |
snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
|
| 562 |
target = root / connector / snapshot
|
|
@@ -600,9 +709,9 @@ def ready_handoffs(root=None):
|
|
| 600 |
return groups
|
| 601 |
|
| 602 |
|
| 603 |
-
def _install_default(connector, tenant, ready):
|
| 604 |
from harness import mirror_store, runtime
|
| 605 |
-
rt = runtime.get_runtime(tenant)
|
| 606 |
from harness import datastore
|
| 607 |
paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
|
| 608 |
resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
|
|
@@ -633,7 +742,10 @@ def _remove_dir(path):
|
|
| 633 |
|
| 634 |
|
| 635 |
def install_ready(root=None, installer=None, *, now=None):
|
| 636 |
-
"""Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
|
|
|
|
|
|
|
|
|
|
| 637 |
root = Path(root or handoff_dir())
|
| 638 |
installer = installer or _install_default
|
| 639 |
stamp = time.time() if now is None else now
|
|
@@ -678,6 +790,72 @@ def install_ready(root=None, installer=None, *, now=None):
|
|
| 678 |
return installed
|
| 679 |
|
| 680 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 681 |
# ----------------------------------------------------------------------------- cross-process desk
|
| 682 |
def _forward_request(receipt, connection_id):
|
| 683 |
"""Web side: hand an accepted Sync now to the worker through the store desk."""
|
|
@@ -693,34 +871,61 @@ def _forward_request(receipt, connection_id):
|
|
| 693 |
"The sync request could not be passed to the sync worker. Try again.")
|
| 694 |
|
| 695 |
|
| 696 |
-
def
|
| 697 |
-
|
| 698 |
from harness import runtime
|
| 699 |
-
return
|
| 700 |
|
| 701 |
|
| 702 |
-
def
|
| 703 |
-
"""
|
| 704 |
from core import connector_sync
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 724 |
|
| 725 |
|
| 726 |
def _worker_desk_loop():
|
|
@@ -731,43 +936,60 @@ def _worker_desk_loop():
|
|
| 731 |
print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
|
| 732 |
|
| 733 |
|
| 734 |
-
def _apply_worker_state():
|
| 735 |
-
"""Web side:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 736 |
from core import connector_sync
|
| 737 |
from harness import mirror_store
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
if isinstance(
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 747 |
app_main = _running_main()
|
| 748 |
-
if
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
connector_sync.apply_remote_progress(rt)
|
| 754 |
-
except Exception as exc: # noqa: BLE001
|
| 755 |
-
print(f"[process-role] sync progress read refused for {slug} ({type(exc).__name__})", flush=True)
|
| 756 |
-
if slug in catch_up:
|
| 757 |
-
mirror_store.apply_catch_up(rt, catch_up[slug])
|
| 758 |
|
| 759 |
|
| 760 |
def _web_watch_loop():
|
| 761 |
while not _STOP.wait(POLL_SECONDS):
|
| 762 |
-
try:
|
| 763 |
-
install_ready()
|
| 764 |
-
write_primary_state()
|
| 765 |
-
except Exception as exc: # noqa: BLE001
|
| 766 |
-
print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
|
| 767 |
try:
|
| 768 |
_apply_worker_state()
|
| 769 |
except Exception as exc: # noqa: BLE001
|
| 770 |
print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 771 |
|
| 772 |
|
| 773 |
# ----------------------------------------------------------------------------- the worker process
|
|
@@ -775,17 +997,72 @@ def _running_main():
|
|
| 775 |
return sys.modules.get("api.main") or sys.modules.get("main")
|
| 776 |
|
| 777 |
|
| 778 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 779 |
try:
|
| 780 |
from harness import mirror_store
|
| 781 |
catch_up = mirror_store.catch_up_snapshot()
|
| 782 |
except Exception: # noqa: BLE001
|
| 783 |
catch_up = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 784 |
app_main = _running_main()
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 789 |
|
| 790 |
|
| 791 |
def _heartbeat_loop(parent_pid):
|
|
@@ -803,9 +1080,9 @@ def _heartbeat_loop(parent_pid):
|
|
| 803 |
|
| 804 |
|
| 805 |
def start_worker_services():
|
| 806 |
-
"""Called by sync_worker.py: heartbeat,
|
| 807 |
from harness import mirror_store
|
| 808 |
-
mirror_store.register_publish_listener(
|
| 809 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 810 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 811 |
# Before the first heavy job: each one now waits until this worker holds the writer lease.
|
|
@@ -814,6 +1091,7 @@ def start_worker_services():
|
|
| 814 |
|
| 815 |
|
| 816 |
def _worker_env(tenant=None):
|
|
|
|
| 817 |
env = dict(os.environ)
|
| 818 |
env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
|
| 819 |
AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
|
|
@@ -828,84 +1106,35 @@ def _worker_env(tenant=None):
|
|
| 828 |
|
| 829 |
|
| 830 |
def _supervise(tenant=None):
|
| 831 |
-
"""
|
| 832 |
-
|
| 833 |
-
label = "sync worker" if tenant is None else f"sync worker for {tenant}"
|
| 834 |
-
backoff = 5.0
|
| 835 |
-
while not _STOP.is_set():
|
| 836 |
-
started = time.time()
|
| 837 |
-
try:
|
| 838 |
-
proc = subprocess.Popen([sys.executable, "sync_worker.py"], cwd=str(API_DIR),
|
| 839 |
-
env=_worker_env(tenant))
|
| 840 |
-
except OSError as exc:
|
| 841 |
-
print(f"[process-role] {label} could not start ({type(exc).__name__}: {exc})", flush=True)
|
| 842 |
-
proc = None
|
| 843 |
-
if proc is not None:
|
| 844 |
-
record.update(proc=proc, pid=proc.pid, startedAt=started)
|
| 845 |
-
print(f"[process-role] {label} started pid={proc.pid}", flush=True)
|
| 846 |
-
if tenant is None:
|
| 847 |
-
write_primary_state()
|
| 848 |
-
code = proc.wait()
|
| 849 |
-
record["lastExit"] = code
|
| 850 |
-
else:
|
| 851 |
-
code = None
|
| 852 |
-
if _STOP.is_set():
|
| 853 |
-
return
|
| 854 |
-
if time.time() - started > 600:
|
| 855 |
-
backoff = 5.0
|
| 856 |
-
record["restarts"] += 1
|
| 857 |
-
print(f"[process-role] {label} exited code={code}; restart {record['restarts']} "
|
| 858 |
-
f"in {int(backoff)}s", flush=True)
|
| 859 |
-
if _STOP.wait(backoff):
|
| 860 |
-
return
|
| 861 |
-
backoff = min(backoff * 2, 300.0)
|
| 862 |
|
| 863 |
|
|
|
|
| 864 |
_FLEET = {}
|
| 865 |
_FLEET_STATE = {"overCap": [], "error": None, "at": None}
|
| 866 |
|
| 867 |
|
| 868 |
-
def _write_fleet(root=None):
|
| 869 |
-
_write_json(Path(root or handoff_dir()) / FLEET_FILE, {
|
| 870 |
-
"at": time.time(), "primary": writer_tenant(),
|
| 871 |
-
"tenants": {slug: {"pid": row.get("pid")} for slug, row in _FLEET.items()},
|
| 872 |
-
"overCap": list(_FLEET_STATE["overCap"])})
|
| 873 |
-
|
| 874 |
-
|
| 875 |
def _report_over_cap(over, cap):
|
| 876 |
"""Rule 1: a tenant past the cap is told, with the cause and the fix, never skipped in silence."""
|
| 877 |
-
cause = (f"its daily Odoo and Amazon sync
|
| 878 |
-
f"(AIOS_SYNC_WORKERS_MAX) and they are all
|
| 879 |
-
f"to a larger box.")
|
| 880 |
for slug in over:
|
| 881 |
-
print(f"[process-role] sync
|
| 882 |
try:
|
| 883 |
import sync_alerts
|
| 884 |
from harness import runtime
|
| 885 |
sync_alerts.notify(slug, "sync-worker", [{"key": "worker", "label": "Sync worker",
|
| 886 |
"code": "over_cap", "cause": cause}],
|
| 887 |
-
store=runtime.get_runtime(slug))
|
| 888 |
except Exception as exc: # noqa: BLE001
|
| 889 |
print(f"[process-role] over-cap alert for {slug} failed ({type(exc).__name__})", flush=True)
|
| 890 |
|
| 891 |
|
| 892 |
-
def fleet_tick(wanted=None, *
|
| 893 |
-
"""
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
new = [slug for slug in wanted if slug not in _FLEET]
|
| 897 |
-
room = max(0, cap - 1 - len(_FLEET))
|
| 898 |
-
starting, over = new[:room], new[room:]
|
| 899 |
-
for slug in starting:
|
| 900 |
-
_FLEET[slug] = {"proc": None, "pid": None, "restarts": 0, "startedAt": None, "lastExit": None}
|
| 901 |
-
_FLEET_STATE.update(overCap=over, at=time.time(), error=None)
|
| 902 |
-
# The fleet file names a tenant before its worker starts, so the primary stops serving it first.
|
| 903 |
-
_write_fleet(root)
|
| 904 |
-
for slug in starting:
|
| 905 |
-
(starter or (lambda s: start(f"sync-worker-supervisor-{s}", "light", _supervise, args=(s,))))(slug)
|
| 906 |
-
if over:
|
| 907 |
-
_report_over_cap(over, cap)
|
| 908 |
-
return starting
|
| 909 |
|
| 910 |
|
| 911 |
def _fleet_loop():
|
|
@@ -914,7 +1143,7 @@ def _fleet_loop():
|
|
| 914 |
fleet_tick()
|
| 915 |
except Exception as exc: # noqa: BLE001
|
| 916 |
_FLEET_STATE.update(error=type(exc).__name__, at=time.time())
|
| 917 |
-
print(f"[process-role]
|
| 918 |
if _STOP.wait(FLEET_SCAN_SECONDS):
|
| 919 |
return
|
| 920 |
|
|
@@ -933,7 +1162,7 @@ def stop_worker():
|
|
| 933 |
|
| 934 |
|
| 935 |
def start_web_services():
|
| 936 |
-
"""Web role only: forward Sync now,
|
| 937 |
|
| 938 |
With several uvicorn workers only the primary does this. The others stand by and read.
|
| 939 |
"""
|
|
@@ -946,38 +1175,45 @@ def start_web_services():
|
|
| 946 |
if _WORKER.get("standby") is None:
|
| 947 |
_WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
|
| 948 |
print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
|
| 949 |
-
f"the primary
|
| 950 |
return False
|
| 951 |
-
|
| 952 |
-
start("sync-worker-
|
| 953 |
start("mirror-handoff-watcher", "light", _web_watch_loop)
|
| 954 |
atexit.register(stop_worker)
|
| 955 |
return True
|
| 956 |
|
| 957 |
|
| 958 |
def status():
|
| 959 |
-
"""{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
|
| 960 |
-
|
| 961 |
-
|
| 962 |
-
|
| 963 |
-
|
| 964 |
-
|
| 965 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 966 |
pid, alive = os.getpid(), True
|
|
|
|
|
|
|
|
|
|
|
|
|
| 967 |
else:
|
| 968 |
pid, alive = None, False
|
| 969 |
-
age = None
|
| 970 |
-
if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
|
| 971 |
-
age = round(max(0.0, time.time() - float(beat["at"])), 1)
|
| 972 |
fleet = {}
|
| 973 |
for slug, row in _FLEET.items():
|
| 974 |
-
|
| 975 |
-
|
| 976 |
-
|
| 977 |
return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
|
|
|
|
| 978 |
"tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
|
| 979 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 980 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 981 |
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 982 |
-
"lastInstallError": dict(LAST_INSTALL_ERROR) or None
|
|
|
|
| 983 |
"writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
|
|
|
|
| 5 |
never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
|
| 6 |
store in the background; `light` serves this process's own reads.
|
| 7 |
|
| 8 |
+
One writer per connector: resident jobs hold the Odoo lease, while each Amazon pass claims and
|
| 9 |
+
renews its own lease independently of the first-load job.
|
| 10 |
|
| 11 |
+
Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock, asks
|
| 12 |
+
the sync runtime for passes and installs mirrors. The others only read the mirror read-only.
|
| 13 |
+
|
| 14 |
+
The web never starts a sync worker child. sync_runtime picks where the worker runs: one HF job
|
| 15 |
+
per pass, per tenant (`hf-job`), a separately run resident worker (`external`), or nothing (`off`).
|
| 16 |
+
|
| 17 |
+
The web reads the worker's heartbeat, resync state and published mirror ids from the store's
|
| 18 |
+
C-status document (connector_sync.STATUS_KEY, provider WORKER_STATUS) and installs each mirror from
|
| 19 |
+
its stored manifest. No web read depends on a local child process or a hand-off file.
|
| 20 |
|
| 21 |
One sync worker per tenant: the primary worker serves AIOS_SYNC_WRITER_TENANT (royal-imports when
|
| 22 |
unset) and runs every job that loops all tenants. Each other tenant that holds a sync connector
|
| 23 |
gets a tenant worker (AIOS_WORKER_SCOPE=tenant) with its own DuckDB file and lease, which runs only
|
| 24 |
+
that tenant's sync jobs. AIOS_SYNC_WORKERS_MAX caps the sync jobs that run at once; a tenant past
|
| 25 |
+
the cap is reported.
|
| 26 |
"""
|
| 27 |
from __future__ import annotations
|
| 28 |
|
|
|
|
| 33 |
import socket
|
| 34 |
import subprocess
|
| 35 |
import sys
|
| 36 |
+
import tempfile
|
| 37 |
import threading
|
| 38 |
import time
|
| 39 |
import uuid
|
| 40 |
+
from contextlib import contextmanager
|
| 41 |
from pathlib import Path
|
| 42 |
|
| 43 |
ROLES = ("web", "worker", "all", "reader")
|
|
|
|
| 57 |
PRIMARY_FILE = "web-primary.json"
|
| 58 |
INSTALL_ATTEMPTS = 3
|
| 59 |
INCOMPLETE_GRACE_SECONDS = 3600
|
| 60 |
+
#: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
|
| 61 |
+
WORKER_STATUS = "sync_worker"
|
| 62 |
+
#: A stored heartbeat older than this reads as a dead worker. verify_live allows 120 s.
|
| 63 |
+
HEARTBEAT_STALE_SECONDS = 120
|
| 64 |
|
| 65 |
|
| 66 |
def resolve_role(env=None):
|
|
|
|
| 92 |
_INSTALL_FAILURES = {}
|
| 93 |
LAST_INSTALL = {}
|
| 94 |
LAST_INSTALL_ERROR = {}
|
| 95 |
+
# The web's last read of the store: the worker heartbeat and each tenant's published mirror ids.
|
| 96 |
+
_STORED = {"beat": None, "mirrors": {}}
|
| 97 |
+
_RESYNC_APPLIED = {"at": ""}
|
| 98 |
|
| 99 |
|
| 100 |
def role():
|
|
|
|
| 143 |
|
| 144 |
|
| 145 |
def fleet_tenants(root=None):
|
| 146 |
+
"""Tenants with their own tenant worker: AIOS_SYNC_FLEET_TENANTS in a job, else the fleet file."""
|
| 147 |
+
named = os.environ.get("AIOS_SYNC_FLEET_TENANTS")
|
| 148 |
+
if named is not None:
|
| 149 |
+
# The web names them when it submits the primary's job: a job has no shared fleet file.
|
| 150 |
+
return {slug.strip() for slug in named.split(",") if slug.strip()}
|
| 151 |
state = _read_json(Path(root or handoff_dir()) / FLEET_FILE)
|
| 152 |
tenants = state.get("tenants") if isinstance(state, dict) else None
|
| 153 |
return {str(slug) for slug in tenants} if isinstance(tenants, dict) else set()
|
|
|
|
| 164 |
|
| 165 |
|
| 166 |
def sync_worker_cap(env=None):
|
| 167 |
+
"""AIOS_SYNC_WORKERS_MAX: sync jobs that run at once, the primary's included. Default 4."""
|
| 168 |
env = os.environ if env is None else env
|
| 169 |
try:
|
| 170 |
return max(1, int(env.get("AIOS_SYNC_WORKERS_MAX") or 4))
|
|
|
|
| 270 |
return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
|
| 271 |
|
| 272 |
|
| 273 |
+
def acquire_writer(rt, holder=None, *, now=None, ttl=None, connector=None):
|
| 274 |
"""Take or renew the store's writer lease. True when `holder` holds it after the call.
|
| 275 |
|
| 276 |
The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
|
|
|
|
| 289 |
doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
|
| 290 |
return doc
|
| 291 |
|
| 292 |
+
lease_key = f"{WRITER_LEASE_DOC}:{connector or 'odoo'}"
|
| 293 |
+
doc = rt.update(lease_key, take)
|
| 294 |
if not isinstance(doc, dict):
|
| 295 |
+
doc = rt.get(lease_key, fresh=True)
|
| 296 |
return isinstance(doc, dict) and doc.get("holder") == holder
|
| 297 |
|
| 298 |
|
| 299 |
+
def release_writer(rt, holder=None, *, connector=None):
|
| 300 |
"""Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
|
| 301 |
holder = holder or writer_holder()
|
| 302 |
|
|
|
|
| 306 |
doc.update(holder=None, renewedAt=0, previous=holder)
|
| 307 |
return doc
|
| 308 |
|
| 309 |
+
rt.update(f"{WRITER_LEASE_DOC}:{connector or 'odoo'}", drop)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
@contextmanager
|
| 313 |
+
def connector_writer(rt, connector, *, exit_fn=None):
|
| 314 |
+
"""Hold and renew one connector lease for a pass, refusing a competing first-load writer."""
|
| 315 |
+
holder = f"{writer_holder()}:{connector}:{uuid.uuid4().hex}"
|
| 316 |
+
if not acquire_writer(rt, holder, connector=connector):
|
| 317 |
+
raise RuntimeError(f"Another job holds the {connector} writer lease.")
|
| 318 |
+
stopped = threading.Event()
|
| 319 |
+
|
| 320 |
+
def renew():
|
| 321 |
+
while not stopped.wait(WRITER_LEASE_SECONDS / 3):
|
| 322 |
+
try:
|
| 323 |
+
held = acquire_writer(rt, holder, connector=connector)
|
| 324 |
+
except Exception:
|
| 325 |
+
held = False
|
| 326 |
+
if not held:
|
| 327 |
+
print(f"[process-role] {connector} writer lease lost; this worker stops writing", flush=True)
|
| 328 |
+
(exit_fn or os._exit)(3)
|
| 329 |
+
return
|
| 330 |
+
|
| 331 |
+
job = None
|
| 332 |
+
try:
|
| 333 |
+
job = start("connector-writer-renew", "light", renew)
|
| 334 |
+
if job is None:
|
| 335 |
+
raise RuntimeError("The connector writer lease could not be renewed.")
|
| 336 |
+
yield
|
| 337 |
+
finally:
|
| 338 |
+
stopped.set()
|
| 339 |
+
if job is not None:
|
| 340 |
+
job.join()
|
| 341 |
+
release_writer(rt, holder, connector=connector)
|
| 342 |
|
| 343 |
|
| 344 |
def holds_writer():
|
|
|
|
| 366 |
stamp = time.time() if now is None else float(now)
|
| 367 |
ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
|
| 368 |
try:
|
| 369 |
+
held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl, connector="odoo")
|
| 370 |
except Exception as exc: # noqa: BLE001
|
| 371 |
held = None
|
| 372 |
if _WRITER.get("error") != type(exc).__name__:
|
|
|
|
| 432 |
|
| 433 |
|
| 434 |
# ----------------------------------------------------------------------------- readiness (C9)
|
| 435 |
+
READY_CHECKS = ("mirror", "pools", "first_reads", "tenant_keys")
|
| 436 |
_READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
|
| 437 |
"pools": "The read pools are not open yet.",
|
| 438 |
+
"first_reads": "The first reads have not answered yet.",
|
| 439 |
+
"tenant_keys": "The tenant data keys have not been checked yet."}
|
| 440 |
_READINESS = {}
|
| 441 |
_BOOT_RESTORE = {}
|
| 442 |
# The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
|
|
|
|
| 470 |
print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
|
| 471 |
|
| 472 |
|
| 473 |
+
def _known_beat(root=None):
|
| 474 |
+
"""The stored worker heartbeat this web worker last read; a secondary takes the primary's read."""
|
| 475 |
+
beat = _STORED["beat"] if is_primary() else primary_state(root).get("workerBeat")
|
| 476 |
+
return beat if isinstance(beat, dict) else None
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def _known_mirrors(root=None):
|
| 480 |
+
"""{(tenant, connector): snapshotId} the store says the worker published, as last read."""
|
| 481 |
+
if is_primary():
|
| 482 |
+
return dict(_STORED["mirrors"])
|
| 483 |
+
stored = primary_state(root).get("storeMirrors") or {}
|
| 484 |
+
return {tuple(key.split("/", 1)): snap for key, snap in stored.items() if "/" in key}
|
| 485 |
+
|
| 486 |
+
|
| 487 |
+
def _waits_for_worker_boot():
|
| 488 |
+
"""Only a resident worker reports a boot restore. A job per pass leaves the web to its own restore."""
|
| 489 |
+
import sync_runtime
|
| 490 |
+
return sync_runtime.runtime_name()[0] == "external"
|
| 491 |
+
|
| 492 |
+
|
| 493 |
+
def beat_age(beat, now=None):
|
| 494 |
+
"""Seconds since a heartbeat's stamp, or None when it carries none."""
|
| 495 |
+
try:
|
| 496 |
+
return round(max(0.0, (time.time() if now is None else now) - float(beat["at"])), 1)
|
| 497 |
+
except (KeyError, TypeError, ValueError):
|
| 498 |
+
return None
|
| 499 |
+
|
| 500 |
+
|
| 501 |
def worker_boot_restored(root=None):
|
| 502 |
+
"""Web side: the worker's stored heartbeat is fresh and says its boot restore finished."""
|
| 503 |
+
beat = _known_beat(root)
|
| 504 |
+
if beat is not None:
|
| 505 |
+
restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
|
| 506 |
+
age = beat_age(beat)
|
| 507 |
+
return (bool(restore.get("state")) and restore.get("pid") == beat.get("pid")
|
| 508 |
+
and age is not None and age < HEARTBEAT_STALE_SECONDS)
|
| 509 |
+
# Fallback for hand-off dir fixtures only (verify_readiness, verify_box_sizing): the store outranks it.
|
| 510 |
beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
|
| 511 |
restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
|
| 512 |
# A secondary web worker supervises nothing: the primary names the current sync worker.
|
|
|
|
| 522 |
|
| 523 |
|
| 524 |
def pending_handoffs(root=None):
|
| 525 |
+
"""`tenant/connector` for each stored publication the primary has not installed yet.
|
| 526 |
+
|
| 527 |
+
READY hand-offs count too, for the fixtures of verify_readiness and verify_box_sizing.
|
| 528 |
+
"""
|
| 529 |
installed = _installed_map(root)
|
| 530 |
+
pending = {f"{key[0]}/{key[1]}" for key, snap in _known_mirrors(root).items() if installed.get(key) != snap}
|
| 531 |
+
pending.update(f"{key[0]}/{key[1]}" for key, rows in ready_handoffs(root).items()
|
| 532 |
+
if installed.get(key) != (rows[-1][2].get("snapshotId") or rows[-1][1].name))
|
| 533 |
+
return sorted(pending)
|
| 534 |
|
| 535 |
|
| 536 |
# ----------------------------------------------------------------------------- several web workers
|
|
|
|
| 571 |
os.close(fd)
|
| 572 |
return False
|
| 573 |
_PRIMARY["fd"] = fd
|
| 574 |
+
print(f"[process-role] web worker pid={os.getpid()} is the primary: it ticks the sync runtime "
|
| 575 |
f"and installs hand-offs", flush=True)
|
| 576 |
write_primary_state(root)
|
| 577 |
return True
|
|
|
|
| 588 |
version = None
|
| 589 |
_write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
|
| 590 |
"pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
|
| 591 |
+
"installed": {f"{key[0]}/{key[1]}": snap for key, snap in dict(_INSTALLED).items()},
|
| 592 |
+
"workerBeat": _STORED["beat"],
|
| 593 |
+
"storeMirrors": {f"{key[0]}/{key[1]}": snap for key, snap in dict(_STORED["mirrors"]).items()},
|
| 594 |
"mirrorVersion": version})
|
| 595 |
return True
|
| 596 |
|
|
|
|
| 612 |
role_name = role_name or ROLE
|
| 613 |
checks, detail = {}, {}
|
| 614 |
for name in READY_CHECKS:
|
| 615 |
+
if name == "tenant_keys":
|
| 616 |
+
from core import tenant_keys
|
| 617 |
+
from harness import runtime
|
| 618 |
+
try:
|
| 619 |
+
checks[name], detail[name] = tenant_keys.readiness_check(runtime.known_tenants())
|
| 620 |
+
except Exception:
|
| 621 |
+
checks[name], detail[name] = False, "The tenant data keys did not answer."
|
| 622 |
+
continue
|
| 623 |
row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
|
| 624 |
checks[name], detail[name] = bool(row["ok"]), row["detail"]
|
| 625 |
boot = role_name == "web" and _READY_LATCH["at"] is None
|
| 626 |
if "mirror" in checks and checks["mirror"]:
|
| 627 |
if mirror_live is False:
|
| 628 |
checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
|
| 629 |
+
elif boot and _waits_for_worker_boot() and not worker_boot_restored(root):
|
| 630 |
checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
|
| 631 |
elif boot and pending_handoffs(root):
|
| 632 |
checks["mirror"] = False
|
|
|
|
| 661 |
|
| 662 |
|
| 663 |
def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
|
| 664 |
+
"""Keep one snapshot's parquet in a hand-off dir. READY.json is written last.
|
| 665 |
+
|
| 666 |
+
No runtime caller: the web installs from the store (install_stored). verify_readiness and
|
| 667 |
+
verify_box_sizing build their fixtures with it.
|
| 668 |
+
"""
|
| 669 |
root = Path(root or handoff_dir())
|
| 670 |
snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
|
| 671 |
target = root / connector / snapshot
|
|
|
|
| 709 |
return groups
|
| 710 |
|
| 711 |
|
| 712 |
+
def _install_default(connector, tenant, ready, rt=None):
|
| 713 |
from harness import mirror_store, runtime
|
| 714 |
+
rt = rt or runtime.get_runtime(tenant)
|
| 715 |
from harness import datastore
|
| 716 |
paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
|
| 717 |
resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
|
|
|
|
| 742 |
|
| 743 |
|
| 744 |
def install_ready(root=None, installer=None, *, now=None):
|
| 745 |
+
"""Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
|
| 746 |
+
|
| 747 |
+
No runtime caller since the store install; kept for the verify_readiness fixture.
|
| 748 |
+
"""
|
| 749 |
root = Path(root or handoff_dir())
|
| 750 |
installer = installer or _install_default
|
| 751 |
stamp = time.time() if now is None else now
|
|
|
|
| 790 |
return installed
|
| 791 |
|
| 792 |
|
| 793 |
+
def _resident_snapshot(connector, tenant):
|
| 794 |
+
"""The snapshot id this process already serves, through its own read path. None when unreadable."""
|
| 795 |
+
from harness import datastore
|
| 796 |
+
try:
|
| 797 |
+
return datastore.snapshot_id(tenant, connector)
|
| 798 |
+
except Exception: # noqa: BLE001
|
| 799 |
+
return None
|
| 800 |
+
|
| 801 |
+
|
| 802 |
+
def _install_from_store(connector, tenant, rt, snapshot_id):
|
| 803 |
+
"""Download the store's published manifest and parquet, then install them. False if resident."""
|
| 804 |
+
from harness import datastore, mirror_store
|
| 805 |
+
store = mirror_store.for_connector(connector)
|
| 806 |
+
if snapshot_id and _resident_snapshot(store.connector, tenant) == snapshot_id:
|
| 807 |
+
return False
|
| 808 |
+
raw = rt.download_bytes(store.prefix + mirror_store._MANIFEST_KEY)
|
| 809 |
+
if raw is None:
|
| 810 |
+
raise LookupError(f"no published {connector} manifest in the store")
|
| 811 |
+
with tempfile.TemporaryDirectory(prefix="mirror-store-install-") as tmp:
|
| 812 |
+
# An in-memory DuckDB counts the parquet rows; the tenant file stays closed until install.
|
| 813 |
+
con = datastore.duckdb.connect()
|
| 814 |
+
try:
|
| 815 |
+
manifest, rows, paths, _total, _refused = store._validated_snapshot(tenant, rt, con, raw, tmp)
|
| 816 |
+
finally:
|
| 817 |
+
con.close()
|
| 818 |
+
return _install_default(connector, tenant, {"manifest": manifest, "stateRows": rows,
|
| 819 |
+
"paths": {name: list(files) for name, files in paths.items()}},
|
| 820 |
+
rt)
|
| 821 |
+
|
| 822 |
+
|
| 823 |
+
def install_stored(tenant, rt, mirrors, installer=None, *, now=None):
|
| 824 |
+
"""Install each mirror the store says the worker published and this process has not installed.
|
| 825 |
+
|
| 826 |
+
`mirrors` is the C-status `sync_worker.mirrors` section: {connector: {"snapshotId": ...}}.
|
| 827 |
+
"""
|
| 828 |
+
installer = installer or _install_from_store
|
| 829 |
+
stamp = time.time() if now is None else now
|
| 830 |
+
installed = []
|
| 831 |
+
for connector, row in sorted((mirrors or {}).items()):
|
| 832 |
+
snapshot = row.get("snapshotId") if isinstance(row, dict) else None
|
| 833 |
+
if not snapshot:
|
| 834 |
+
continue
|
| 835 |
+
key = (str(tenant), str(connector))
|
| 836 |
+
_STORED["mirrors"][key] = snapshot
|
| 837 |
+
if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS:
|
| 838 |
+
continue
|
| 839 |
+
try:
|
| 840 |
+
changed = installer(key[1], key[0], rt, snapshot)
|
| 841 |
+
except Exception as exc: # noqa: BLE001
|
| 842 |
+
_INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
|
| 843 |
+
LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
|
| 844 |
+
attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
|
| 845 |
+
print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
|
| 846 |
+
f"({type(exc).__name__}: {exc})", flush=True)
|
| 847 |
+
continue
|
| 848 |
+
_INSTALLED[key] = snapshot
|
| 849 |
+
if changed is False:
|
| 850 |
+
continue
|
| 851 |
+
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
| 852 |
+
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
|
| 853 |
+
LAST_INSTALL_ERROR.clear()
|
| 854 |
+
installed.append(key)
|
| 855 |
+
print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
|
| 856 |
+
return installed
|
| 857 |
+
|
| 858 |
+
|
| 859 |
# ----------------------------------------------------------------------------- cross-process desk
|
| 860 |
def _forward_request(receipt, connection_id):
|
| 861 |
"""Web side: hand an accepted Sync now to the worker through the store desk."""
|
|
|
|
| 871 |
"The sync request could not be passed to the sync worker. Try again.")
|
| 872 |
|
| 873 |
|
| 874 |
+
def _status_runtime(tenant=None):
|
| 875 |
+
"""The runtime whose C-status document holds the sync worker's heartbeat."""
|
| 876 |
from harness import runtime
|
| 877 |
+
return runtime.get_runtime(tenant or writer_tenant())
|
| 878 |
|
| 879 |
|
| 880 |
+
def stored_worker(rt):
|
| 881 |
+
"""The `sync_worker` C-status entry of one tenant, read past this process's cache. {} when absent."""
|
| 882 |
from core import connector_sync
|
| 883 |
+
return connector_sync.status(rt, WORKER_STATUS, fresh=True) or {}
|
| 884 |
+
|
| 885 |
+
|
| 886 |
+
_HEARTBEAT_READ = threading.local()
|
| 887 |
+
|
| 888 |
+
|
| 889 |
+
def stored_heartbeat(tenant=None):
|
| 890 |
+
"""The worker heartbeat in the store, or None. A store that does not answer reads as None."""
|
| 891 |
+
try:
|
| 892 |
+
worker = stored_worker(_status_runtime(tenant))
|
| 893 |
+
_HEARTBEAT_READ.worker = worker
|
| 894 |
+
beat = worker.get("heartbeat")
|
| 895 |
+
except Exception: # noqa: BLE001
|
| 896 |
+
_HEARTBEAT_READ.worker = {}
|
| 897 |
+
return None
|
| 898 |
+
return beat if isinstance(beat, dict) else None
|
| 899 |
+
|
| 900 |
+
|
| 901 |
+
def poll_worker_desk(tenants=None, get_runtime=None):
|
| 902 |
+
"""Worker side: pick up event receipts and Sync now hand-offs, one isolated job per tenant."""
|
| 903 |
+
import tenant_loops
|
| 904 |
+
# Light: the heavy desk loop that calls this already holds the writer lease.
|
| 905 |
+
return tenant_loops.for_each_tenant(_poll_tenant_desk, name="sync-desk-tenant", tenants=tenants,
|
| 906 |
+
get_runtime=get_runtime, kind="light")
|
| 907 |
+
|
| 908 |
+
|
| 909 |
+
def _poll_tenant_desk(slug, rt):
|
| 910 |
+
from core import connector_sync
|
| 911 |
+
if not syncs_here(slug):
|
| 912 |
+
return
|
| 913 |
+
try:
|
| 914 |
+
connector_sync.restore_sync_requests(rt)
|
| 915 |
+
admitted = connector_sync.poll_remote_requests(rt)
|
| 916 |
+
except Exception as exc: # noqa: BLE001
|
| 917 |
+
print(f"[process-role] sync desk poll refused for {slug} ({type(exc).__name__})", flush=True)
|
| 918 |
+
raise
|
| 919 |
+
for entry in admitted:
|
| 920 |
+
print(f"[process-role] sync request {entry['provider']} for {slug} admitted from the web", flush=True)
|
| 921 |
+
import routes_connectors
|
| 922 |
+
key = (entry["provider"], slug)
|
| 923 |
+
with routes_connectors._SYNC_REQUEST_CONNECTION_LOCK:
|
| 924 |
+
# No connection id means the default connection, never the previous request's.
|
| 925 |
+
if entry.get("connectionId"):
|
| 926 |
+
routes_connectors._SYNC_REQUEST_CONNECTIONS[key] = entry["connectionId"]
|
| 927 |
+
else:
|
| 928 |
+
routes_connectors._SYNC_REQUEST_CONNECTIONS.pop(key, None)
|
| 929 |
|
| 930 |
|
| 931 |
def _worker_desk_loop():
|
|
|
|
| 936 |
print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
|
| 937 |
|
| 938 |
|
| 939 |
+
def _apply_worker_state(tenants=None, get_runtime=None):
|
| 940 |
+
"""Web side: apply each tenant's stored worker state and install its mirrors, one job per tenant.
|
| 941 |
+
|
| 942 |
+
One tenant whose store read raises leaves the other tenants' state applied (D-11881).
|
| 943 |
+
"""
|
| 944 |
+
import tenant_loops
|
| 945 |
+
return tenant_loops.for_each_tenant(_apply_tenant_state, name="worker-state", tenants=tenants,
|
| 946 |
+
get_runtime=get_runtime, kind="light")
|
| 947 |
+
|
| 948 |
+
|
| 949 |
+
def _apply_tenant_state(slug, rt):
|
| 950 |
+
"""Read one tenant's C-status document once, then apply heartbeat, resync, progress and mirrors."""
|
| 951 |
from core import connector_sync
|
| 952 |
from harness import mirror_store
|
| 953 |
+
document = connector_sync.status_all(rt, fresh=True)
|
| 954 |
+
worker = (document.get(WORKER_STATUS) or {}).get("default") or {}
|
| 955 |
+
# The runtime tick reads this copy instead of a second store read.
|
| 956 |
+
_STORED.setdefault("workers", {})[slug] = (time.time(), dict(worker))
|
| 957 |
+
if slug == writer_tenant():
|
| 958 |
+
if isinstance(worker.get("heartbeat"), dict):
|
| 959 |
+
_STORED["beat"] = worker["heartbeat"]
|
| 960 |
+
_apply_resync(document.get("odoo") or {})
|
| 961 |
+
connector_sync.apply_remote_progress(rt)
|
| 962 |
+
if isinstance(worker.get("catchUp"), dict) and worker["catchUp"]:
|
| 963 |
+
mirror_store.apply_catch_up(rt, worker["catchUp"])
|
| 964 |
+
if isinstance(worker.get("mirrors"), dict):
|
| 965 |
+
install_stored(slug, rt, worker["mirrors"])
|
| 966 |
+
|
| 967 |
+
|
| 968 |
+
def _apply_resync(connections):
|
| 969 |
+
"""Copy the newest stored Odoo resync state into this process's RESYNC_STATE for the status routes."""
|
| 970 |
+
rows = [entry["resync"] for entry in connections.values()
|
| 971 |
+
if isinstance(entry, dict) and isinstance(entry.get("resync"), dict)]
|
| 972 |
+
if not rows:
|
| 973 |
+
return
|
| 974 |
+
newest = max(rows, key=lambda row: str(row.get("at") or ""))
|
| 975 |
app_main = _running_main()
|
| 976 |
+
if app_main is None or str(newest.get("at") or "") < _RESYNC_APPLIED["at"]:
|
| 977 |
+
return
|
| 978 |
+
_RESYNC_APPLIED["at"] = str(newest.get("at") or "")
|
| 979 |
+
# Restore errors stay this process's own.
|
| 980 |
+
app_main.RESYNC_STATE.update({k: v for k, v in newest.items() if "estore" not in k and k != "at"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 981 |
|
| 982 |
|
| 983 |
def _web_watch_loop():
|
| 984 |
while not _STOP.wait(POLL_SECONDS):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 985 |
try:
|
| 986 |
_apply_worker_state()
|
| 987 |
except Exception as exc: # noqa: BLE001
|
| 988 |
print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
|
| 989 |
+
try:
|
| 990 |
+
write_primary_state()
|
| 991 |
+
except Exception as exc: # noqa: BLE001
|
| 992 |
+
print(f"[process-role] primary state write failed ({type(exc).__name__}: {exc})", flush=True)
|
| 993 |
|
| 994 |
|
| 995 |
# ----------------------------------------------------------------------------- the worker process
|
|
|
|
| 997 |
return sys.modules.get("api.main") or sys.modules.get("main")
|
| 998 |
|
| 999 |
|
| 1000 |
+
_CATCH_UP_SENT = {}
|
| 1001 |
+
|
| 1002 |
+
|
| 1003 |
+
def write_heartbeat(now=None, get_runtime=None):
|
| 1004 |
+
"""Worker side: write the heartbeat, each tenant's catch-up and RESYNC_STATE to the C-status store.
|
| 1005 |
+
|
| 1006 |
+
Only the writer lease holder beats once the lease is armed, so a waiting worker never masks it.
|
| 1007 |
+
"""
|
| 1008 |
+
from core import connector_sync
|
| 1009 |
+
if _WRITER["armed"] and not _WRITER_HELD.is_set():
|
| 1010 |
+
return False
|
| 1011 |
+
get_runtime = get_runtime or _status_runtime
|
| 1012 |
+
beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
|
| 1013 |
+
"at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
|
| 1014 |
+
rt = get_runtime(writer_tenant())
|
| 1015 |
+
import worker_provenance
|
| 1016 |
+
attestation = worker_provenance.heartbeat_attestation(rt, beat)
|
| 1017 |
+
|
| 1018 |
+
def change(entry):
|
| 1019 |
+
job = entry.get("job") or {}
|
| 1020 |
+
actual_job = os.environ.get("JOB_ID")
|
| 1021 |
+
if job.get("id") and (actual_job != job["id"] or job.get("tenant") not in (None, writer_tenant())):
|
| 1022 |
+
raise ValueError("stale_worker_heartbeat")
|
| 1023 |
+
entry["heartbeat"] = {**(entry.get("heartbeat") or {}), **beat}
|
| 1024 |
+
# Hash the final stored heartbeat, including fields from an earlier schema.
|
| 1025 |
+
entry["attestation"] = None
|
| 1026 |
+
if attestation and actual_job == job.get("id") and job.get("tenant") == writer_tenant():
|
| 1027 |
+
try:
|
| 1028 |
+
worker_provenance.bounded(entry["heartbeat"])
|
| 1029 |
+
entry["attestation"] = worker_provenance.bounded({**attestation, "heartbeatSha256": worker_provenance.sha(
|
| 1030 |
+
worker_provenance.encoded(entry["heartbeat"]))})
|
| 1031 |
+
except Exception:
|
| 1032 |
+
pass
|
| 1033 |
+
|
| 1034 |
+
written = connector_sync._status_update(rt, WORKER_STATUS, None, change)
|
| 1035 |
try:
|
| 1036 |
from harness import mirror_store
|
| 1037 |
catch_up = mirror_store.catch_up_snapshot()
|
| 1038 |
except Exception: # noqa: BLE001
|
| 1039 |
catch_up = {}
|
| 1040 |
+
for tenant, sources in catch_up.items():
|
| 1041 |
+
text = json.dumps(sources, sort_keys=True, default=str)
|
| 1042 |
+
if _CATCH_UP_SENT.get(tenant) == text:
|
| 1043 |
+
continue
|
| 1044 |
+
try:
|
| 1045 |
+
if connector_sync.update_status(get_runtime(tenant), WORKER_STATUS, "catchUp", sources):
|
| 1046 |
+
_CATCH_UP_SENT[tenant] = text
|
| 1047 |
+
except Exception as exc: # noqa: BLE001
|
| 1048 |
+
print(f"[sync-worker] catch-up not stored for {tenant} ({type(exc).__name__})", flush=True)
|
| 1049 |
app_main = _running_main()
|
| 1050 |
+
publish = getattr(app_main, "_publish_resync_state", None)
|
| 1051 |
+
tenant = str((getattr(app_main, "RESYNC_STATE", None) or {}).get("tenant") or "")
|
| 1052 |
+
if callable(publish) and tenant:
|
| 1053 |
+
# Writes only when RESYNC_STATE changed since its last copy.
|
| 1054 |
+
publish(tenant)
|
| 1055 |
+
return written
|
| 1056 |
+
|
| 1057 |
+
|
| 1058 |
+
def record_publication(connector, tenant_key, manifest, _rows=None, _paths=None):
|
| 1059 |
+
"""Publish listener: name the stored snapshot in C-status, so every web process installs it."""
|
| 1060 |
+
from core import connector_sync
|
| 1061 |
+
snapshot = str((manifest or {}).get("snapshot_id") or "")
|
| 1062 |
+
if not snapshot:
|
| 1063 |
+
return False
|
| 1064 |
+
return connector_sync.update_status(_status_runtime(tenant_key), WORKER_STATUS, "mirrors", {
|
| 1065 |
+
str(connector): {"snapshotId": snapshot, "at": time.time()}})
|
| 1066 |
|
| 1067 |
|
| 1068 |
def _heartbeat_loop(parent_pid):
|
|
|
|
| 1080 |
|
| 1081 |
|
| 1082 |
def start_worker_services():
|
| 1083 |
+
"""Called by sync_worker.py: heartbeat, the stored publication record and the store desk poll."""
|
| 1084 |
from harness import mirror_store
|
| 1085 |
+
mirror_store.register_publish_listener(record_publication)
|
| 1086 |
parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
|
| 1087 |
start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
|
| 1088 |
# Before the first heavy job: each one now waits until this worker holds the writer lease.
|
|
|
|
| 1091 |
|
| 1092 |
|
| 1093 |
def _worker_env(tenant=None):
|
| 1094 |
+
"""A resident worker's env. No runtime caller since the web runs no child; gates read it."""
|
| 1095 |
env = dict(os.environ)
|
| 1096 |
env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
|
| 1097 |
AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
|
|
|
|
| 1106 |
|
| 1107 |
|
| 1108 |
def _supervise(tenant=None):
|
| 1109 |
+
"""One sync runtime tick, for `tenant` or every tenant. The web never starts a sync worker child."""
|
| 1110 |
+
return fleet_tick(None if tenant is None else [tenant])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1111 |
|
| 1112 |
|
| 1113 |
+
#: {tenant: {job, jobId, jobs, alive, aliveWhy}} from this web process's runtime ticks.
|
| 1114 |
_FLEET = {}
|
| 1115 |
_FLEET_STATE = {"overCap": [], "error": None, "at": None}
|
| 1116 |
|
| 1117 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1118 |
def _report_over_cap(over, cap):
|
| 1119 |
"""Rule 1: a tenant past the cap is told, with the cause and the fix, never skipped in silence."""
|
| 1120 |
+
cause = (f"its daily Odoo and Amazon sync waits: {cap} sync job(s) may run at once "
|
| 1121 |
+
f"(AIOS_SYNC_WORKERS_MAX) and they are all running. Raise AIOS_SYNC_WORKERS_MAX.")
|
|
|
|
| 1122 |
for slug in over:
|
| 1123 |
+
print(f"[process-role] sync job for {slug} NOT submitted: {cause}", flush=True)
|
| 1124 |
try:
|
| 1125 |
import sync_alerts
|
| 1126 |
from harness import runtime
|
| 1127 |
sync_alerts.notify(slug, "sync-worker", [{"key": "worker", "label": "Sync worker",
|
| 1128 |
"code": "over_cap", "cause": cause}],
|
| 1129 |
+
observed={"worker"}, store=runtime.get_runtime(slug))
|
| 1130 |
except Exception as exc: # noqa: BLE001
|
| 1131 |
print(f"[process-role] over-cap alert for {slug} failed ({type(exc).__name__})", flush=True)
|
| 1132 |
|
| 1133 |
|
| 1134 |
+
def fleet_tick(wanted=None, **kwargs):
|
| 1135 |
+
"""Ask the sync runtime for each due tenant's pass inside the cap. Returns the tenants it submitted."""
|
| 1136 |
+
import sync_runtime
|
| 1137 |
+
return sync_runtime.tick(wanted, **kwargs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1138 |
|
| 1139 |
|
| 1140 |
def _fleet_loop():
|
|
|
|
| 1143 |
fleet_tick()
|
| 1144 |
except Exception as exc: # noqa: BLE001
|
| 1145 |
_FLEET_STATE.update(error=type(exc).__name__, at=time.time())
|
| 1146 |
+
print(f"[process-role] sync runtime tick failed ({type(exc).__name__}: {exc})", flush=True)
|
| 1147 |
if _STOP.wait(FLEET_SCAN_SECONDS):
|
| 1148 |
return
|
| 1149 |
|
|
|
|
| 1162 |
|
| 1163 |
|
| 1164 |
def start_web_services():
|
| 1165 |
+
"""Web role only: forward Sync now, tick the sync runtime, install the worker's snapshots.
|
| 1166 |
|
| 1167 |
With several uvicorn workers only the primary does this. The others stand by and read.
|
| 1168 |
"""
|
|
|
|
| 1175 |
if _WORKER.get("standby") is None:
|
| 1176 |
_WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
|
| 1177 |
print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
|
| 1178 |
+
f"the primary ticks the sync runtime", flush=True)
|
| 1179 |
return False
|
| 1180 |
+
# No worker child: the supervisor thread only ticks the sync runtime.
|
| 1181 |
+
_WORKER["supervisor"] = start("sync-worker-supervisor", "light", _fleet_loop)
|
| 1182 |
start("mirror-handoff-watcher", "light", _web_watch_loop)
|
| 1183 |
atexit.register(stop_worker)
|
| 1184 |
return True
|
| 1185 |
|
| 1186 |
|
| 1187 |
def status():
|
| 1188 |
+
"""{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
|
| 1189 |
+
|
| 1190 |
+
A serving role reads the worker from its stored heartbeat, never from a child process (D-11183).
|
| 1191 |
+
"""
|
| 1192 |
+
_HEARTBEAT_READ.worker = {}
|
| 1193 |
+
beat = stored_heartbeat() if ROLE != "all" else None
|
| 1194 |
+
stored = _HEARTBEAT_READ.worker
|
| 1195 |
+
beat = beat if isinstance(beat, dict) else None
|
| 1196 |
+
import worker_provenance
|
| 1197 |
+
age = beat_age(beat) if beat is not None else None
|
| 1198 |
+
if ROLE == "worker":
|
| 1199 |
pid, alive = os.getpid(), True
|
| 1200 |
+
age = age if beat is not None and beat.get("pid") == pid else None
|
| 1201 |
+
elif ROLE in ("web", "reader") and beat is not None:
|
| 1202 |
+
import sync_runtime
|
| 1203 |
+
pid, alive = beat.get("pid"), sync_runtime.beat_alive(beat)
|
| 1204 |
else:
|
| 1205 |
pid, alive = None, False
|
|
|
|
|
|
|
|
|
|
| 1206 |
fleet = {}
|
| 1207 |
for slug, row in _FLEET.items():
|
| 1208 |
+
fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
|
| 1209 |
+
"lastExit": None, "jobId": row.get("jobId")}
|
| 1210 |
+
import sync_runtime
|
| 1211 |
return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
|
| 1212 |
+
"syncRuntime": sync_runtime.runtime_name()[0],
|
| 1213 |
"tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
|
| 1214 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 1215 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 1216 |
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 1217 |
+
"lastInstallError": dict(LAST_INSTALL_ERROR) or None,
|
| 1218 |
+
"attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
|
| 1219 |
"writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
|
api/registry_source.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""Bounded stored-source reads before canonical grid projection."""
|
| 2 |
import copy
|
| 3 |
-
from contextlib import closing
|
| 4 |
from dataclasses import is_dataclass, replace
|
| 5 |
|
| 6 |
from deps import err
|
|
@@ -530,6 +530,198 @@ def canonical_metadata(session, table_key, *, metadata=None, cur=None):
|
|
| 530 |
consume_corrections=False, with_rows=False, needed=frozenset())
|
| 531 |
|
| 532 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 533 |
def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
|
| 534 |
"""Refuse a failed compiled source by name without publishing an empty population."""
|
| 535 |
from fastapi import HTTPException
|
|
|
|
| 1 |
"""Bounded stored-source reads before canonical grid projection."""
|
| 2 |
import copy
|
| 3 |
+
from contextlib import closing, contextmanager
|
| 4 |
from dataclasses import is_dataclass, replace
|
| 5 |
|
| 6 |
from deps import err
|
|
|
|
| 530 |
consume_corrections=False, with_rows=False, needed=frozenset())
|
| 531 |
|
| 532 |
|
| 533 |
+
def _selection_teams(session):
|
| 534 |
+
"""Validate the explicit runtime identity and its configured teams before SQL."""
|
| 535 |
+
runtime = session.runtime
|
| 536 |
+
key = str(getattr(runtime, 'key', '') or '').strip().lower()
|
| 537 |
+
tenant = getattr(runtime, 'tenant', None)
|
| 538 |
+
owner = str(getattr(tenant, 'key', '') or '').strip().lower()
|
| 539 |
+
actor = str(session.user.get('tenant') or key).strip().lower()
|
| 540 |
+
if not key or key != owner or actor != key:
|
| 541 |
+
raise err(403, 'tenant_mismatch', 'Canonical selection requires a bound tenant identity.')
|
| 542 |
+
config = getattr(tenant, 'config', None) or {}
|
| 543 |
+
units = config.get('business_units')
|
| 544 |
+
if (not isinstance(units, dict) or not units
|
| 545 |
+
or any(not isinstance(label, str) or not label.strip()
|
| 546 |
+
or type(tid) is not int or tid <= 0 for label, tid in units.items())
|
| 547 |
+
or len(set(units.values())) != len(units)):
|
| 548 |
+
raise err(403, 'bu_scope_denied', 'Canonical selection requires known tenant teams.')
|
| 549 |
+
labels = {tid: label for label, tid in units.items()}
|
| 550 |
+
bus = session.user.get('bus', 'all')
|
| 551 |
+
if bus != 'all' and (not isinstance(bus, (list, tuple, set, frozenset)) or not bus
|
| 552 |
+
or any(type(tid) is not int or tid not in labels for tid in bus)):
|
| 553 |
+
raise err(403, 'bu_scope_denied', 'Account teams are empty or unknown for this tenant.')
|
| 554 |
+
return labels
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
@contextmanager
|
| 558 |
+
def canonical_selection(session, cur, table_key, metadata, needed, *, ids=None):
|
| 559 |
+
"""Lend SQL identities and proven native operands before canonical page hydration."""
|
| 560 |
+
from core import perm_scope
|
| 561 |
+
from harness import canonical_source as cs
|
| 562 |
+
from modules import ar, customer_data, product_data
|
| 563 |
+
import routes_keychain
|
| 564 |
+
import routes_odoo_tables as rot
|
| 565 |
+
import time
|
| 566 |
+
|
| 567 |
+
team_labels = _selection_teams(session)
|
| 568 |
+
if (needed is None or getattr(session.runtime, 'key', None) != 'royal-imports'
|
| 569 |
+
or routes_keychain.odoo_paused(session.runtime)):
|
| 570 |
+
yield None
|
| 571 |
+
return
|
| 572 |
+
session.require(table_key)
|
| 573 |
+
team, agent = perm_scope.derive_pool_scope(session.user, table_key)
|
| 574 |
+
cache_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 575 |
+
cached = session.runtime.pool_cache.get(cache_key)
|
| 576 |
+
if cached and time.time() - cached[0] < 900:
|
| 577 |
+
yield None
|
| 578 |
+
return # The canonical page must read the same source as the selection.
|
| 579 |
+
fields = metadata['fields']
|
| 580 |
+
scope_fields = (metadata.get('_permission_scope') or {}).get('fields', fields)
|
| 581 |
+
tree = {} if session.admin else (perm_scope.entry(session.user, table_key) or {}).get('filter') or {}
|
| 582 |
+
from harness import filter_eval
|
| 583 |
+
nodes, _conj = filter_eval.tree_parts(tree)
|
| 584 |
+
required = set(needed) | rot._leaf_cols(nodes)
|
| 585 |
+
# Set-dependent and authored permission operands retain their canonical evaluator.
|
| 586 |
+
def complex_wall(nodes):
|
| 587 |
+
return any(isinstance(n, dict) and (n.get('window') is not None
|
| 588 |
+
or n.get('op') in rot._fs().RANK_OPS or complex_wall(n.get('children') or [])) for n in nodes)
|
| 589 |
+
if complex_wall(nodes) or perm_scope._row_audience_field(table_key, st=session.runtime) is not None:
|
| 590 |
+
yield None
|
| 591 |
+
return
|
| 592 |
+
whitespace = "'" + ''.join(chr(c) for c in (*range(9, 14), *range(28, 33), 133, 160,
|
| 593 |
+
5760, *range(8192, 8203), 8232, 8233, 8239, 8287, 12288)) + "'"
|
| 594 |
+
if table_key == 'customer_data':
|
| 595 |
+
expressions = {
|
| 596 |
+
'archived': "CASE WHEN NOT live THEN true WHEN p.id IS NULL THEN false ELSE NOT coalesce(p.active, false) END",
|
| 597 |
+
'dba': "CASE WHEN live THEN coalesce(brands.brand, '') END",
|
| 598 |
+
'agent': "CASE WHEN live THEN coalesce((SELECT string_agg(a.name, ', ' ORDER BY cast(member.key AS BIGINT)) "
|
| 599 |
+
"FROM json_each(p.agent_ids) member JOIN res_partner a ON a.id=try_cast(member.value AS BIGINT) "
|
| 600 |
+
"WHERE coalesce(a.active, false) AND a.name IS NOT NULL AND a.name<>''), '(none)') ELSE '(none)' END"}
|
| 601 |
+
# These source attributes have no authored display override or case transformation.
|
| 602 |
+
for key in ('street', 'street2', 'zip'):
|
| 603 |
+
expressions[key] = f"CASE WHEN live THEN coalesce(nullif(trim(p.{key}, {whitespace}), ''), '(none)') ELSE '(none)' END"
|
| 604 |
+
for key, column in (('state', 'state_name'), ('country', 'country_name'),
|
| 605 |
+
('payment_terms', 'property_payment_term_name'),
|
| 606 |
+
('pricelist', 'property_product_pricelist_name')):
|
| 607 |
+
expressions[key] = f"CASE WHEN live THEN coalesce(nullif(p.{column}, ''), '(none)') ELSE '(none)' END"
|
| 608 |
+
for key in ('customer_since', 'created_on'):
|
| 609 |
+
expressions[key] = "CASE WHEN live THEN coalesce(substr(p.create_date, 1, 10), '') ELSE '' END"
|
| 610 |
+
else:
|
| 611 |
+
expressions = {'code': 'code'}
|
| 612 |
+
vocabulary = {f['key']: f for f in scope_fields}
|
| 613 |
+
visible = {f['key'] for f in fields}
|
| 614 |
+
if (not set(needed) <= visible or not required <= expressions.keys()
|
| 615 |
+
or any((vocabulary.get(k) or {}).get('source') != 'odoo'
|
| 616 |
+
or (vocabulary.get(k) or {}).get('derived')
|
| 617 |
+
or (vocabulary.get(k) or {}).get('type') not in {
|
| 618 |
+
'text', 'long_text', 'status', 'select', 'multiselect', 'date', 'checkbox'} for k in required)):
|
| 619 |
+
yield None
|
| 620 |
+
return
|
| 621 |
+
kind = 'product' if table_key == 'product_data' else 'customer'
|
| 622 |
+
missing = cs.mirror_missing(cur, kind)
|
| 623 |
+
if missing:
|
| 624 |
+
raise err(503, 'canonical_source_unavailable', cs._unready(missing))
|
| 625 |
+
hash_function = None
|
| 626 |
+
with cs.mirrored(cur, session.runtime, kind=kind) as source:
|
| 627 |
+
try:
|
| 628 |
+
if table_key == 'customer_data':
|
| 629 |
+
params, legs = [], []
|
| 630 |
+
domains = ([('sale.order', [('state', 'in', ('sale', 'done')), ('team_id', '=', team)], 'partner_id')]
|
| 631 |
+
if team else [
|
| 632 |
+
('res.partner', [('customer_rank', '>', 0), ('active', 'in', [True, False])], 'id'),
|
| 633 |
+
('sale.order', [('state', 'in', ('sale', 'done'))], 'partner_id'),
|
| 634 |
+
('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))], 'partner_id'),
|
| 635 |
+
('account.move.line', list(ar.AGED_LINE_DOMAIN), 'partner_id')])
|
| 636 |
+
for model, domain, key in domains:
|
| 637 |
+
query = cs._Query(source, model, params)
|
| 638 |
+
value, field_kind, _ = query.field(key)
|
| 639 |
+
value = value[0] if field_kind == 'm2o' else value
|
| 640 |
+
where = query.where(domain if ids is None else [*domain, (key, 'in', list(ids))])
|
| 641 |
+
legs.append(f'SELECT {value} pid FROM {query.from_sql()} WHERE {where}')
|
| 642 |
+
cur.execute('CREATE TEMP TABLE canonical_live AS SELECT DISTINCT pid FROM ('
|
| 643 |
+
+ ' UNION ALL '.join(legs) + ') source WHERE pid IS NOT NULL', params)
|
| 644 |
+
if agent and agent not in {'All', 'All agents'}:
|
| 645 |
+
members = source.agent_members(agent)
|
| 646 |
+
cur.execute('DELETE FROM canonical_live WHERE pid NOT IN '
|
| 647 |
+
f'(SELECT try_cast(json_extract_string(body, \'/id\') AS BIGINT) FROM "{members.table}")')
|
| 648 |
+
cur.execute('CREATE TEMP TABLE canonical_retained (pid BIGINT)')
|
| 649 |
+
if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY):
|
| 650 |
+
with closing(items(session.runtime, customer_data.LEDGER_KEY, selected=ids)) as batches:
|
| 651 |
+
for batch in batches:
|
| 652 |
+
retained = []
|
| 653 |
+
for key, _entry in batch:
|
| 654 |
+
try:
|
| 655 |
+
retained.append(int(key))
|
| 656 |
+
except (TypeError, ValueError):
|
| 657 |
+
continue
|
| 658 |
+
if retained:
|
| 659 |
+
cur.execute('INSERT INTO canonical_retained SELECT unnest(?::BIGINT[])', [retained])
|
| 660 |
+
population = ('SELECT pid, true live FROM canonical_live UNION ALL '
|
| 661 |
+
'SELECT DISTINCT pid, false live FROM canonical_retained '
|
| 662 |
+
'WHERE pid NOT IN (SELECT pid FROM canonical_live)')
|
| 663 |
+
projected = ', '.join(f'{expressions[k]} AS "{k}"' for k in sorted(required))
|
| 664 |
+
brand_join = ''
|
| 665 |
+
if 'dba' in required:
|
| 666 |
+
import datetime
|
| 667 |
+
start = (datetime.date.fromisoformat(str(metadata['today'])) - datetime.timedelta(days=731)).isoformat()
|
| 668 |
+
query = cs._Query(source, 'sale.order')
|
| 669 |
+
where = query.where([('state', 'in', ('sale', 'done')), ('date_order', '>=', start),
|
| 670 |
+
('team_id', 'in', list(team_labels))])
|
| 671 |
+
cases = ' '.join('WHEN ? THEN ?' for _ in team_labels)
|
| 672 |
+
label_params = [value for pair in team_labels.items() for value in pair]
|
| 673 |
+
cur.execute('CREATE TEMP TABLE canonical_brands AS SELECT b.partner_id pid, '
|
| 674 |
+
f'CASE WHEN count(DISTINCT b.team_id)>1 THEN ? ELSE CASE min(b.team_id) {cases} END END AS brand '
|
| 675 |
+
f'FROM {query.from_sql()} WHERE {where} AND b.partner_id<>0 GROUP BY b.partner_id',
|
| 676 |
+
[customer_data._multi_team_label(len(team_labels)), *label_params, *query.params])
|
| 677 |
+
brand_join = ' LEFT JOIN canonical_brands brands ON brands.pid=u.pid'
|
| 678 |
+
# Absent optional columns retain the canonical path's blank/refusal behavior.
|
| 679 |
+
try:
|
| 680 |
+
cur.execute('CREATE TEMP TABLE canonical_selection AS SELECT u.pid'
|
| 681 |
+
+ (', ' + projected if projected else '')
|
| 682 |
+
+ f' FROM ({population}) u LEFT JOIN res_partner p ON p.id=u.pid' + brand_join)
|
| 683 |
+
except Exception as exc:
|
| 684 |
+
from core.request_timing import raise_if_resource_error
|
| 685 |
+
raise_if_resource_error(exc)
|
| 686 |
+
import duckdb
|
| 687 |
+
if not isinstance(exc, duckdb.BinderException):
|
| 688 |
+
raise
|
| 689 |
+
yield None
|
| 690 |
+
return
|
| 691 |
+
else:
|
| 692 |
+
import uuid
|
| 693 |
+
function_name = 'canonical_pid_' + uuid.uuid4().hex
|
| 694 |
+
cur.create_function(function_name, lambda code: product_data.sku_pid(code), ['VARCHAR'], 'BIGINT')
|
| 695 |
+
hash_function = function_name
|
| 696 |
+
cur.execute('CREATE TEMP TABLE canonical_selection AS SELECT '
|
| 697 |
+
f'{hash_function}(code) pid, code FROM (SELECT DISTINCT '
|
| 698 |
+
"CASE WHEN default_code IS NULL OR default_code='' THEN 'pid:'||CAST(id AS VARCHAR) "
|
| 699 |
+
f'ELSE trim(default_code, {whitespace}) END code FROM product_product WHERE active=true) codes')
|
| 700 |
+
if cur.execute('SELECT pid FROM canonical_selection GROUP BY pid HAVING count(*)>1 LIMIT 1').fetchone():
|
| 701 |
+
raise err(503, 'canonical_source_unavailable', 'product_data: pid collision. Refusing the catalogue.')
|
| 702 |
+
bindings = {k: (f'"{k}"', str) for k in required}
|
| 703 |
+
spec = {'table': 'canonical_selection', 'id': 'pid', 'cols': bindings}
|
| 704 |
+
wall_cols = rot._filter_columns(spec, scope_fields)
|
| 705 |
+
# Permission booleans retain their source spelling in the canonical payload.
|
| 706 |
+
wall, params = perm_scope._window_wall(session.user, table_key, scope_fields,
|
| 707 |
+
wall_cols, id_sql='pid', st=session.runtime)
|
| 708 |
+
if ids is not None:
|
| 709 |
+
wall = f'({wall}) AND pid IN (SELECT unnest(?::BIGINT[]))'
|
| 710 |
+
params.append(list(ids))
|
| 711 |
+
cur.execute(f'DELETE FROM canonical_selection WHERE NOT coalesce(({wall}), false)', params)
|
| 712 |
+
yield spec
|
| 713 |
+
except Exception as exc:
|
| 714 |
+
from fastapi import HTTPException
|
| 715 |
+
from core.request_timing import raise_if_resource_error
|
| 716 |
+
if isinstance(exc, HTTPException):
|
| 717 |
+
raise
|
| 718 |
+
raise_if_resource_error(exc)
|
| 719 |
+
raise err(503, 'canonical_source_unavailable', f'{table_key}: {type(exc).__name__}: {str(exc)[:180]}') from exc
|
| 720 |
+
finally:
|
| 721 |
+
if hash_function:
|
| 722 |
+
cur.remove_function(hash_function)
|
| 723 |
+
|
| 724 |
+
|
| 725 |
def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
|
| 726 |
"""Refuse a failed compiled source by name without publishing an empty population."""
|
| 727 |
from fastapi import HTTPException
|
api/rollup_sql.py
CHANGED
|
@@ -59,7 +59,7 @@ def source_fields(table_def):
|
|
| 59 |
return out
|
| 60 |
|
| 61 |
|
| 62 |
-
def group_values(bag, today=None):
|
| 63 |
"""`{group_id: value}` for one source bag — ONE grouped query over the whole scope.
|
| 64 |
|
| 65 |
⚠ Keyed by the dim's `_id` column, never its label. Two customers can share a display name;
|
|
@@ -67,6 +67,7 @@ def group_values(bag, today=None):
|
|
| 67 |
"""
|
| 68 |
from harness import semantic as sem
|
| 69 |
from harness import windows as W
|
|
|
|
| 70 |
|
| 71 |
topic = str(bag.get("topic") or "")
|
| 72 |
measure = str(bag.get("measure") or "")
|
|
@@ -86,9 +87,20 @@ def group_values(bag, today=None):
|
|
| 86 |
date_from, date_to = rng
|
| 87 |
|
| 88 |
try:
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
except Exception as e: # noqa: BLE001
|
| 93 |
raise RollupSourceError(f"{type(e).__name__}: {e}") from e
|
| 94 |
|
|
@@ -121,8 +133,8 @@ def group_values(bag, today=None):
|
|
| 121 |
|
| 122 |
|
| 123 |
def _today():
|
| 124 |
-
|
| 125 |
-
return
|
| 126 |
|
| 127 |
|
| 128 |
def compute(rt, table_key, today=None, tables=None, persist=False):
|
|
@@ -146,7 +158,7 @@ def compute(rt, table_key, today=None, tables=None, persist=False):
|
|
| 146 |
bag = f["rollup"]["source"]
|
| 147 |
# ⚠ Resolved BEFORE anything is written. A rollup that refuses must leave every cell as it
|
| 148 |
# was — a partially applied pass would mix two vintages of the same column.
|
| 149 |
-
plans.append((f["key"], str(bag.get("on") or ""), group_values(bag, today=stamp)))
|
| 150 |
|
| 151 |
written = {}
|
| 152 |
|
|
|
|
| 59 |
return out
|
| 60 |
|
| 61 |
|
| 62 |
+
def group_values(rt, bag, today=None):
|
| 63 |
"""`{group_id: value}` for one source bag — ONE grouped query over the whole scope.
|
| 64 |
|
| 65 |
⚠ Keyed by the dim's `_id` column, never its label. Two customers can share a display name;
|
|
|
|
| 67 |
"""
|
| 68 |
from harness import semantic as sem
|
| 69 |
from harness import windows as W
|
| 70 |
+
from harness.runtime import mirror_cursor
|
| 71 |
|
| 72 |
topic = str(bag.get("topic") or "")
|
| 73 |
measure = str(bag.get("measure") or "")
|
|
|
|
| 87 |
date_from, date_to = rng
|
| 88 |
|
| 89 |
try:
|
| 90 |
+
if rt is None:
|
| 91 |
+
raise RollupSourceError("A tenant runtime is required for source rollups")
|
| 92 |
+
handle = rt.mirror_handle()
|
| 93 |
+
if handle is None:
|
| 94 |
+
raise RollupSourceError("The runtime has no tenant-owned mirror handle")
|
| 95 |
+
cur = mirror_cursor(rt)
|
| 96 |
+
try:
|
| 97 |
+
if getattr(cur, "_mirror_handle", None) is not handle:
|
| 98 |
+
raise RollupSourceError("The runtime has no tenant-owned mirror cursor")
|
| 99 |
+
res = sem.store_query(topic, [measure], group_by=[dim],
|
| 100 |
+
date_from=date_from, date_to=date_to,
|
| 101 |
+
limit=sem.MAX_GROUPS, today=today, runtime=rt, cur=cur)
|
| 102 |
+
finally:
|
| 103 |
+
cur.close()
|
| 104 |
except Exception as e: # noqa: BLE001
|
| 105 |
raise RollupSourceError(f"{type(e).__name__}: {e}") from e
|
| 106 |
|
|
|
|
| 133 |
|
| 134 |
|
| 135 |
def _today():
|
| 136 |
+
from core import periods
|
| 137 |
+
return periods.today().isoformat()
|
| 138 |
|
| 139 |
|
| 140 |
def compute(rt, table_key, today=None, tables=None, persist=False):
|
|
|
|
| 158 |
bag = f["rollup"]["source"]
|
| 159 |
# ⚠ Resolved BEFORE anything is written. A rollup that refuses must leave every cell as it
|
| 160 |
# was — a partially applied pass would mix two vintages of the same column.
|
| 161 |
+
plans.append((f["key"], str(bag.get("on") or ""), group_values(rt, bag, today=stamp)))
|
| 162 |
|
| 163 |
written = {}
|
| 164 |
|
api/routes_account_health.py
CHANGED
|
@@ -5,7 +5,8 @@ from urllib.parse import urlencode
|
|
| 5 |
from deps import Session, err, require_session
|
| 6 |
|
| 7 |
TEMPLATE_KIND = "account_health"
|
| 8 |
-
router = APIRouter(
|
|
|
|
| 9 |
SOURCE = "ut_amzn_account_health"
|
| 10 |
|
| 11 |
|
|
@@ -14,13 +15,15 @@ def _module():
|
|
| 14 |
return account_health
|
| 15 |
|
| 16 |
|
| 17 |
-
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 18 |
-
"""
|
| 19 |
-
|
| 20 |
if session.tenant != getattr(session.runtime, "key", None):
|
| 21 |
raise err(403, "account_health_forbidden", "This workspace does not match your session.")
|
| 22 |
-
|
| 23 |
-
|
|
|
|
|
|
|
| 24 |
return session
|
| 25 |
|
| 26 |
|
|
@@ -99,62 +102,88 @@ def owners(session):
|
|
| 99 |
return sorted((n for n in names if n), key=str.lower)
|
| 100 |
|
| 101 |
|
| 102 |
-
@
|
| 103 |
def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
| 104 |
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 105 |
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 106 |
ah = _module()
|
| 107 |
-
|
| 108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 110 |
|
| 111 |
|
| 112 |
-
@
|
| 113 |
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 114 |
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
|
| 115 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
|
| 117 |
|
| 118 |
|
| 119 |
-
@
|
| 120 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 121 |
search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None),
|
| 122 |
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
|
| 123 |
session: Session = Depends(_app_reader)):
|
| 124 |
ah = _module()
|
| 125 |
-
|
| 126 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 128 |
|
| 129 |
|
| 130 |
-
@
|
| 131 |
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
| 132 |
session: Session = Depends(_app_reader)):
|
| 133 |
ah = _module()
|
|
|
|
| 134 |
alert = _call(ah.alert_record, _tenant(session), item_id, st=session.runtime)
|
| 135 |
-
context = _call(
|
|
|
|
| 136 |
if not _source_allowed(alert, context):
|
| 137 |
raise err(403, "account_health_source_forbidden", "Your account cannot read this alert's source records and fields.")
|
| 138 |
return {"item": ah._public(alert, session.user), "owners": owners(session),
|
| 139 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 140 |
|
| 141 |
|
| 142 |
-
@
|
| 143 |
def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 144 |
due: str | None = Body(None), note: str | None = Body(None),
|
| 145 |
session: Session = Depends(_app_writer)):
|
| 146 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 148 |
due=due, note=note, st=session.runtime)
|
| 149 |
|
| 150 |
|
| 151 |
-
@
|
| 152 |
def read_settings(session: Session = Depends(_app_reader)):
|
| 153 |
ah = _module()
|
| 154 |
return _call(ah.settings, _tenant(session), st=session.runtime)
|
| 155 |
|
| 156 |
|
| 157 |
-
@
|
| 158 |
def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None),
|
| 159 |
riskOverdueDays: float | None = Body(None),
|
| 160 |
session: Session = Depends(_app_writer)):
|
|
@@ -163,26 +192,145 @@ def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: st
|
|
| 163 |
default_owner=defaultOwner, risk_overdue_days=riskOverdueDays, st=session.runtime)
|
| 164 |
|
| 165 |
|
| 166 |
-
@
|
| 167 |
def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 168 |
search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 169 |
session: Session = Depends(_app_reader)):
|
| 170 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
out = _call(ah.risks, _tenant(session), status=status, search=search, offset=offset, limit=limit,
|
| 172 |
st=session.runtime)
|
| 173 |
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 174 |
|
| 175 |
|
| 176 |
-
@
|
| 177 |
def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 178 |
caseId: str | None = Body(None), appealDate: str | None = Body(None),
|
| 179 |
outcome: str | None = Body(None), note: str | None = Body(None),
|
| 180 |
session: Session = Depends(_app_writer)):
|
| 181 |
ah = _module()
|
|
|
|
|
|
|
|
|
|
| 182 |
return _call(ah.update_risk, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 183 |
case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
|
| 184 |
|
| 185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
# --------------------------------------------------------------------------- C5 worker bridge
|
| 187 |
|
| 188 |
def _source_context(rt, user):
|
|
|
|
| 5 |
from deps import Session, err, require_session
|
| 6 |
|
| 7 |
TEMPLATE_KIND = "account_health"
|
| 8 |
+
router = APIRouter(tags=["account-health"])
|
| 9 |
+
app_router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"])
|
| 10 |
SOURCE = "ut_amzn_account_health"
|
| 11 |
|
| 12 |
|
|
|
|
| 15 |
return account_health
|
| 16 |
|
| 17 |
|
| 18 |
+
def _reader(app: str | None = Query(None), session: Session = Depends(require_session)) -> Session:
|
| 19 |
+
"""R62: reads need the App grant plus the shared source check, never the source open grant."""
|
| 20 |
+
import application_registry
|
| 21 |
if session.tenant != getattr(session.runtime, "key", None):
|
| 22 |
raise err(403, "account_health_forbidden", "This workspace does not match your session.")
|
| 23 |
+
record = _app(session, app) if app else _call(application_registry.readable_app, session.runtime,
|
| 24 |
+
_tenant(session), session.uname, session.admin, TEMPLATE_KIND)
|
| 25 |
+
application_registry.admit_app(session, record, (SOURCE,), code="account_health")
|
| 26 |
+
session.source_app = record
|
| 27 |
return session
|
| 28 |
|
| 29 |
|
|
|
|
| 102 |
return sorted((n for n in names if n), key=str.lower)
|
| 103 |
|
| 104 |
|
| 105 |
+
@app_router.get("/summary")
|
| 106 |
def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
|
| 107 |
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 108 |
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
|
| 109 |
ah = _module()
|
| 110 |
+
context = _call(_narrowed, session)
|
| 111 |
+
if context is None:
|
| 112 |
+
out = _call(ah.summary_page, _tenant(session), search=search, offset=offset, limit=limit,
|
| 113 |
+
metric=metric, st=session.runtime)
|
| 114 |
+
else:
|
| 115 |
+
out = _narrow_summary(session, context, search=search, offset=offset, limit=limit, metric=metric)
|
| 116 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 117 |
|
| 118 |
|
| 119 |
+
@app_router.get("/history")
|
| 120 |
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 121 |
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
|
| 122 |
ah = _module()
|
| 123 |
+
context = _call(_narrowed, session)
|
| 124 |
+
if context is not None:
|
| 125 |
+
kept = _allowed_metrics(session, context)["metrics"]
|
| 126 |
+
row = next((row for row in kept if row["id"] == metric), None) or next(
|
| 127 |
+
(row for row in kept if str(row["id"]).split("@", 1)[0] == metric), None)
|
| 128 |
+
if row is None:
|
| 129 |
+
raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
|
| 130 |
+
out = _call(ah.history, _tenant(session), row["id"], cursor=cursor, limit=limit, st=session.runtime)
|
| 131 |
+
out["points"] = [point for point in out["points"] if _source_allowed({"sourceEvidence": _call(
|
| 132 |
+
ah._alert_source, _tenant(session), session.runtime, row, point["day"])}, context)]
|
| 133 |
+
return out
|
| 134 |
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
|
| 135 |
|
| 136 |
|
| 137 |
+
@app_router.get("/items")
|
| 138 |
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 139 |
search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None),
|
| 140 |
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
|
| 141 |
session: Session = Depends(_app_reader)):
|
| 142 |
ah = _module()
|
| 143 |
+
context = _call(_narrowed, session)
|
| 144 |
+
if context is None:
|
| 145 |
+
listed = _call(ah.items, _tenant(session), session.user, status=status, search=search,
|
| 146 |
+
offset=offset, cursor=cursor, limit=limit, metric=metric, st=session.runtime)
|
| 147 |
+
else:
|
| 148 |
+
listed = _narrow_items(session, context, offset, limit, status=status, search=search,
|
| 149 |
+
cursor=cursor, metric=metric)
|
| 150 |
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 151 |
|
| 152 |
|
| 153 |
+
@app_router.get("/items/{item_id}")
|
| 154 |
def read_item(item_id: str, app: str = Query(..., min_length=1),
|
| 155 |
session: Session = Depends(_app_reader)):
|
| 156 |
ah = _module()
|
| 157 |
+
import application_registry
|
| 158 |
alert = _call(ah.alert_record, _tenant(session), item_id, st=session.runtime)
|
| 159 |
+
context = _call(application_registry.app_source_scope, session.user, session.runtime, SOURCE,
|
| 160 |
+
app=session.source_app)
|
| 161 |
if not _source_allowed(alert, context):
|
| 162 |
raise err(403, "account_health_source_forbidden", "Your account cannot read this alert's source records and fields.")
|
| 163 |
return {"item": ah._public(alert, session.user), "owners": owners(session),
|
| 164 |
"canEdit": _may_edit(session, _app(session, app))}
|
| 165 |
|
| 166 |
|
| 167 |
+
@app_router.patch("/items/{item_id}")
|
| 168 |
def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 169 |
due: str | None = Body(None), note: str | None = Body(None),
|
| 170 |
session: Session = Depends(_app_writer)):
|
| 171 |
ah = _module()
|
| 172 |
+
context = _call(_narrowed, session)
|
| 173 |
+
if context is not None and not _source_allowed(
|
| 174 |
+
_call(ah.alert_record, _tenant(session), item_id, st=session.runtime), context):
|
| 175 |
+
raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
|
| 176 |
return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 177 |
due=due, note=note, st=session.runtime)
|
| 178 |
|
| 179 |
|
| 180 |
+
@app_router.get("/settings")
|
| 181 |
def read_settings(session: Session = Depends(_app_reader)):
|
| 182 |
ah = _module()
|
| 183 |
return _call(ah.settings, _tenant(session), st=session.runtime)
|
| 184 |
|
| 185 |
|
| 186 |
+
@app_router.put("/settings")
|
| 187 |
def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None),
|
| 188 |
riskOverdueDays: float | None = Body(None),
|
| 189 |
session: Session = Depends(_app_writer)):
|
|
|
|
| 192 |
default_owner=defaultOwner, risk_overdue_days=riskOverdueDays, st=session.runtime)
|
| 193 |
|
| 194 |
|
| 195 |
+
@app_router.get("/risks")
|
| 196 |
def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(None),
|
| 197 |
search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
|
| 198 |
session: Session = Depends(_app_reader)):
|
| 199 |
ah = _module()
|
| 200 |
+
context = _call(_narrowed, session)
|
| 201 |
+
if context is not None:
|
| 202 |
+
out = _narrow_risks(session, context, status=status, search=search, offset=offset, limit=limit)
|
| 203 |
+
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 204 |
out = _call(ah.risks, _tenant(session), status=status, search=search, offset=offset, limit=limit,
|
| 205 |
st=session.runtime)
|
| 206 |
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
|
| 207 |
|
| 208 |
|
| 209 |
+
@app_router.patch("/risks/{item_id}")
|
| 210 |
def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 211 |
caseId: str | None = Body(None), appealDate: str | None = Body(None),
|
| 212 |
outcome: str | None = Body(None), note: str | None = Body(None),
|
| 213 |
session: Session = Depends(_app_writer)):
|
| 214 |
ah = _module()
|
| 215 |
+
context = _call(_narrowed, session)
|
| 216 |
+
if context is not None:
|
| 217 |
+
raise err(403, "account_health_source_forbidden", "Risk changes need unrestricted source access.")
|
| 218 |
return _call(ah.update_risk, _tenant(session), session.user, item_id, owner=owner, status=status,
|
| 219 |
case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
|
| 220 |
|
| 221 |
|
| 222 |
+
router.include_router(app_router)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
@router.get("/api/v1/account-health/categories/{key}")
|
| 226 |
+
def read_category(key: str, session: Session = Depends(_app_reader)):
|
| 227 |
+
ah = _module()
|
| 228 |
+
context = _call(_narrowed, session)
|
| 229 |
+
|
| 230 |
+
def allowed(row, snapshot):
|
| 231 |
+
evidence = _call(ah.category_source, _tenant(session), session.runtime, row, snapshot)
|
| 232 |
+
return _source_allowed({"sourceEvidence": evidence}, context)
|
| 233 |
+
|
| 234 |
+
return _call(ah.category_detail, _tenant(session), key, user=session.user,
|
| 235 |
+
allowed=allowed if context is not None else None, st=session.runtime)
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
# --------------------------------------------------------------------------- R62 source narrowing
|
| 239 |
+
|
| 240 |
+
def _narrowed(session):
|
| 241 |
+
"""The caller's (visible fields, permitted row ids) on the source, or None when no wall applies."""
|
| 242 |
+
import application_registry
|
| 243 |
+
wall = application_registry.app_access(session.user, session.runtime, None, (SOURCE,))["narrow"][SOURCE]
|
| 244 |
+
if not wall["rows"] and not wall["hidden"]:
|
| 245 |
+
return None
|
| 246 |
+
return application_registry.app_source_scope(session.user, session.runtime, SOURCE, app=session.source_app)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _allowed_metrics(session, context, *, search="", metric=None):
|
| 250 |
+
"""Every metric row whose source row and fields the caller's source wall admits."""
|
| 251 |
+
ah = _module()
|
| 252 |
+
tenant, rt = _tenant(session), session.runtime
|
| 253 |
+
first = _call(ah.summary_page, tenant, search=search, offset=0, limit=1, metric=metric, st=rt)
|
| 254 |
+
full = first if first["metricTotal"] <= 1 else _call(
|
| 255 |
+
ah.summary_page, tenant, search=search, offset=0, limit=first["metricTotal"], metric=metric, st=rt)
|
| 256 |
+
kept = [row for row in full["metrics"] if _source_allowed(
|
| 257 |
+
{"sourceEvidence": _call(ah._alert_source, tenant, rt, row, full["snapshotDate"])}, context)]
|
| 258 |
+
return {**full, "metrics": kept}
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def _page(rows, offset, limit):
|
| 262 |
+
start = min(max(0, offset), max(0, (len(rows) - 1) // limit * limit))
|
| 263 |
+
return start, rows[start:start + limit]
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def _narrow_summary(session, context, *, search, offset, limit, metric):
|
| 267 |
+
"""A source row filter narrows the metric rows; a hidden source field hides its metric."""
|
| 268 |
+
ah = _module()
|
| 269 |
+
out = _allowed_metrics(session, context, search=search, metric=metric)
|
| 270 |
+
kept = out["metrics"]
|
| 271 |
+
if metric is not None and not kept:
|
| 272 |
+
raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
|
| 273 |
+
markets = {row.get("marketplaceId") for row in kept}
|
| 274 |
+
if "groups" in out:
|
| 275 |
+
scoped = _allowed_metrics(session, context)["metrics"] if search or metric is not None else kept
|
| 276 |
+
groups, gauge = ah.group_scores(ah.count_groups(scoped))
|
| 277 |
+
out.update(groups=groups, gauge=gauge,
|
| 278 |
+
foldedCount=sum(row.get("status") in ah.FOLDED for row in scoped))
|
| 279 |
+
start, page = _page(kept, offset, limit)
|
| 280 |
+
out.update(metrics=page, metricTotal=len(kept), metricOffset=start, metricLimit=limit,
|
| 281 |
+
marketCount=len(markets), cursor=str(start + limit) if start + limit < len(kept) else None)
|
| 282 |
+
if out.get("marketplaceId") not in markets or "account_health_rating_ahr_score" not in context[0]:
|
| 283 |
+
out.update(rating=None, ratingStatus=None, marketplaceId=None)
|
| 284 |
+
elif "account_health_rating_ahr_status" not in context[0]:
|
| 285 |
+
out["ratingStatus"] = None
|
| 286 |
+
if not kept:
|
| 287 |
+
out.update(state="empty", message=ah.EMPTY, snapshotDate=None)
|
| 288 |
+
return out
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def _narrow_items(session, context, offset, limit, **options):
|
| 292 |
+
"""Keep the alerts whose source rows and fields the caller's source wall admits."""
|
| 293 |
+
ah = _module()
|
| 294 |
+
rows, scan = [], 0
|
| 295 |
+
while True:
|
| 296 |
+
listed = _call(ah.items, _tenant(session), session.user, offset=scan, limit=200,
|
| 297 |
+
st=session.runtime, **options)
|
| 298 |
+
for row in listed["items"]:
|
| 299 |
+
alert = _call(ah.alert_record, _tenant(session), row["id"], st=session.runtime)
|
| 300 |
+
if _source_allowed(alert, context):
|
| 301 |
+
rows.append(row)
|
| 302 |
+
scan += len(listed["items"])
|
| 303 |
+
if not listed["items"] or scan >= listed["total"]:
|
| 304 |
+
break
|
| 305 |
+
start, page = _page(rows, offset, limit)
|
| 306 |
+
more = start + len(page) < len(rows)
|
| 307 |
+
return {**listed, "items": page, "cursor": page[-1]["id"] if more and page else None,
|
| 308 |
+
"total": len(rows), "offset": start, "limit": limit}
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def _narrow_risks(session, context, *, status, search, offset, limit):
|
| 312 |
+
"""Serve only risks whose source evidence fits the Account health wall."""
|
| 313 |
+
ah = _module()
|
| 314 |
+
metrics = {row["id"]: row for row in _allowed_metrics(session, context)["metrics"]}
|
| 315 |
+
kept, scan = [], 0
|
| 316 |
+
while True:
|
| 317 |
+
out = _call(ah.risks, _tenant(session), status=status, search=search,
|
| 318 |
+
offset=scan, limit=200, st=session.runtime)
|
| 319 |
+
for risk in out["items"]:
|
| 320 |
+
metric = metrics.get(risk.get("metricId"))
|
| 321 |
+
if metric is not None and _source_allowed({"sourceEvidence": _call(
|
| 322 |
+
ah._alert_source, _tenant(session), session.runtime, metric, risk.get("snapshotDate"))}, context):
|
| 323 |
+
kept.append(risk)
|
| 324 |
+
scan += len(out["items"])
|
| 325 |
+
if not out["items"] or scan >= out["total"]:
|
| 326 |
+
break
|
| 327 |
+
start, page = _page(kept, offset, limit)
|
| 328 |
+
return {**out, "items": page, "total": len(kept), "offset": start, "limit": limit,
|
| 329 |
+
"cursor": str(start + limit) if start + limit < len(kept) else None,
|
| 330 |
+
"reserveSummary": {"available": False, "total": None, "byCurrency": {}, "lineCount": 0,
|
| 331 |
+
"unknownReasons": ["Reserve figures need unrestricted source access."]}}
|
| 332 |
+
|
| 333 |
+
|
| 334 |
# --------------------------------------------------------------------------- C5 worker bridge
|
| 335 |
|
| 336 |
def _source_context(rt, user):
|
api/routes_admin.py
CHANGED
|
@@ -143,17 +143,7 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
|
| 143 |
# editor will open, so the two cannot disagree.
|
| 144 |
"accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
|
| 145 |
surface_keys=surface_keys),
|
| 146 |
-
"perms_v": int(rec.get("perms_v") or 0)
|
| 147 |
-
# S79-T63: only a copied test account carries `test`, so a real account's keys stay exact.
|
| 148 |
-
**({"test": _test_marker(rec)} if _test_marker(rec) else {})}
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
def _test_marker(rec):
|
| 152 |
-
mark = (rec or {}).get(users.TEST_ACCOUNT_KEY)
|
| 153 |
-
if not isinstance(mark, dict):
|
| 154 |
-
return None
|
| 155 |
-
return {"copiedFrom": mark.get("copied_from"), "createdBy": mark.get("created_by"),
|
| 156 |
-
"createdAt": mark.get("created_at")}
|
| 157 |
|
| 158 |
|
| 159 |
def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
|
|
@@ -464,9 +454,6 @@ def update_user(username: str, body: dict = Body(default=None),
|
|
| 464 |
if result == "last_admin":
|
| 465 |
raise err(400, "last_admin",
|
| 466 |
"an active tenant must retain at least one administrator")
|
| 467 |
-
if result in _TEST_ACCOUNT_REFUSALS:
|
| 468 |
-
# R42: a test account gets no email and no admin role, refused in the core writer.
|
| 469 |
-
raise err(400, result, _TEST_ACCOUNT_REFUSALS[result])
|
| 470 |
if result != "updated":
|
| 471 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 472 |
|
|
@@ -552,166 +539,6 @@ def expire_password(username: str, session: Session = Depends(admin_gate)):
|
|
| 552 |
return Response(status_code=204)
|
| 553 |
|
| 554 |
|
| 555 |
-
# ── S79-T63 (R28, R42): test accounts that copy one member's permissions exactly ─────────────────
|
| 556 |
-
_TEST_ACCOUNT_REFUSALS = {
|
| 557 |
-
"test_account_no_email": "A test account cannot have an email address.",
|
| 558 |
-
"test_account_no_whatsapp": "A test account cannot connect WhatsApp.",
|
| 559 |
-
"test_account_not_admin": "A test account cannot be an administrator.",
|
| 560 |
-
}
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
def _copy_refusal(exc):
|
| 564 |
-
return err(exc.status, exc.code, str(exc))
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
#: The en and em dash, built from code points so this file carries no dash character (rule 2).
|
| 568 |
-
_DASHES = re.compile("[" + chr(0x2013) + chr(0x2014) + "]")
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
def _copy_failure(exc, verb):
|
| 572 |
-
# A store error can carry any text; rule 2 keeps dashes off the screen.
|
| 573 |
-
cause = _DASHES.sub(",", str(exc.cause or "")).strip().rstrip(".")
|
| 574 |
-
message = f"The test account was not {verb}. {cause}."
|
| 575 |
-
if exc.leftovers:
|
| 576 |
-
message += (" These parts could not be removed: " + ", ".join(exc.leftovers)
|
| 577 |
-
+ ". Remove the test account again to finish.")
|
| 578 |
-
elif verb == "created":
|
| 579 |
-
message += " Nothing was kept."
|
| 580 |
-
return err(503, "test_account_failed", message)
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
def _test_account_steps(session):
|
| 584 |
-
"""Copy steps for grant stores outside `core`: Connector grants, through their own route."""
|
| 585 |
-
import core.perm_scope as perm_scope
|
| 586 |
-
import routes_keychain as keychain
|
| 587 |
-
|
| 588 |
-
def held():
|
| 589 |
-
doc = session.runtime.get(perm_scope.CONNECTOR_GRANT_KEY, fresh=True)
|
| 590 |
-
if doc is not None and not isinstance(doc, dict):
|
| 591 |
-
raise RuntimeError("connector access could not be read")
|
| 592 |
-
return doc or {}
|
| 593 |
-
|
| 594 |
-
def copy_grants(username, source):
|
| 595 |
-
known = {kind["key"] for kind in keychain.connector_grant_kinds()}
|
| 596 |
-
kinds = sorted(k for k in (held().get(source) or ()) if k in known)
|
| 597 |
-
if kinds:
|
| 598 |
-
keychain.put_connector_grants(username, {"granted": kinds}, session)
|
| 599 |
-
|
| 600 |
-
def clear_grants(username):
|
| 601 |
-
if username in held():
|
| 602 |
-
keychain.put_connector_grants(username, {"granted": []}, session)
|
| 603 |
-
|
| 604 |
-
return [("connector access", copy_grants, clear_grants)]
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
@router.get("/admin/users/{username}/test-account/preview")
|
| 608 |
-
def preview_test_account(username: str, session: Session = Depends(admin_gate)):
|
| 609 |
-
"""What a test account copied from this member would carry, before anything is written."""
|
| 610 |
-
from core import test_accounts, user_tables
|
| 611 |
-
reg = _registry()
|
| 612 |
-
try:
|
| 613 |
-
plan = test_accounts.plan(username, reg, tenant=_tenant_of(session.user),
|
| 614 |
-
st=session.runtime)
|
| 615 |
-
lent = user_tables.lend_defs(session.runtime, schema_only=True)
|
| 616 |
-
except test_accounts.CopyRefused as exc:
|
| 617 |
-
raise _copy_refusal(exc)
|
| 618 |
-
except Exception as exc: # noqa: BLE001
|
| 619 |
-
raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc
|
| 620 |
-
limits = _source_limits(plan["source"], reg.get(plan["source"]), session)
|
| 621 |
-
body = test_accounts.preview(plan, st=lent, limits=limits)
|
| 622 |
-
_label_grants(session, body["copy"]["grants"])
|
| 623 |
-
return body
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
def _source_limits(uname, rec, session):
|
| 627 |
-
"""The source's row limits, phrased for the administrator; None if they cannot be read.
|
| 628 |
-
|
| 629 |
-
Field walls are lifted for the phrasing only, so a condition on a hidden field is named
|
| 630 |
-
instead of reading "Limited by your access". The filters themselves are the source's.
|
| 631 |
-
"""
|
| 632 |
-
import core.perm_scope as perm_scope
|
| 633 |
-
try:
|
| 634 |
-
reader = deepcopy(users._public(uname, rec))
|
| 635 |
-
for entry in (reader.get("perms") or {}).values():
|
| 636 |
-
if isinstance(entry, dict):
|
| 637 |
-
entry.pop("hiddenFields", None)
|
| 638 |
-
return perm_scope.scope_limits(reader, st=session.runtime)
|
| 639 |
-
except Exception: # noqa: BLE001
|
| 640 |
-
return None
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
#: What a preview row says when a shared item's name cannot be read. Never a raw id.
|
| 644 |
-
_GRANT_FALLBACK = {"view": "A shared view", "folder": "A shared folder", "database": "A database",
|
| 645 |
-
"field": "A shared field", "cohort": "A locked view", "workbook": "A shared sheet"}
|
| 646 |
-
#: Naming stops after this many seconds; every grant is still listed, the rest by kind.
|
| 647 |
-
_LABEL_BUDGET_S = 3.0
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
def _label_grants(session, grants):
|
| 651 |
-
"""Name each grant through the share door's own resolver (`routes_shares._object_ref`)."""
|
| 652 |
-
import time
|
| 653 |
-
try:
|
| 654 |
-
from routes_shares import _object_ref
|
| 655 |
-
except Exception: # noqa: BLE001
|
| 656 |
-
_object_ref = None
|
| 657 |
-
deadline = time.monotonic() + _LABEL_BUDGET_S
|
| 658 |
-
for grant in grants:
|
| 659 |
-
if grant.get("label"):
|
| 660 |
-
continue
|
| 661 |
-
label = None
|
| 662 |
-
if time.monotonic() < deadline:
|
| 663 |
-
try:
|
| 664 |
-
if grant["kind"] == "workbook":
|
| 665 |
-
# `_object_ref` has no workbook branch; the workbook's own name is the label.
|
| 666 |
-
import core.sheets as sheets
|
| 667 |
-
label = str(sheets.get_workbook(session.runtime, grant["oid"]).get("name") or "")
|
| 668 |
-
elif _object_ref is not None:
|
| 669 |
-
label = str(_object_ref(session, grant["kind"], grant["oid"])[0] or "")
|
| 670 |
-
except Exception: # noqa: BLE001
|
| 671 |
-
label = None
|
| 672 |
-
label = (label or "").strip()
|
| 673 |
-
grant["label"] = label or _GRANT_FALLBACK.get(grant["kind"], "A shared item")
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
@router.post("/admin/test-accounts", status_code=201)
|
| 677 |
-
def create_test_account(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
|
| 678 |
-
"""Create a test account whose permissions copy `source` exactly. The password shows once."""
|
| 679 |
-
from core import test_accounts
|
| 680 |
-
body = body if isinstance(body, dict) else {}
|
| 681 |
-
_require_store()
|
| 682 |
-
try:
|
| 683 |
-
made = test_accounts.create(body.get("source"), admin=session.uname,
|
| 684 |
-
tenant=_tenant_of(session.user), st=session.runtime,
|
| 685 |
-
steps=_test_account_steps(session))
|
| 686 |
-
except test_accounts.CopyRefused as exc:
|
| 687 |
-
raise _copy_refusal(exc)
|
| 688 |
-
except test_accounts.CopyFailed as exc:
|
| 689 |
-
raise _copy_failure(exc, "created")
|
| 690 |
-
except Exception as exc: # noqa: BLE001
|
| 691 |
-
raise err(503, "store_unavailable", "The test account was not created. Try again.") from exc
|
| 692 |
-
rec = _registry().get(made["username"])
|
| 693 |
-
if not isinstance(rec, dict):
|
| 694 |
-
raise err(503, "store_unavailable", "The test account was not saved. Try again.")
|
| 695 |
-
return {"user": _view(made["username"], rec), "password": made["password"]}
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
@router.delete("/admin/test-accounts/{username}", status_code=204)
|
| 699 |
-
def delete_test_account(username: str, session: Session = Depends(admin_gate)):
|
| 700 |
-
"""Remove a test account, its sign in and every grant naming it, in one call."""
|
| 701 |
-
from core import test_accounts
|
| 702 |
-
_require_store()
|
| 703 |
-
try:
|
| 704 |
-
test_accounts.remove(username, tenant=_tenant_of(session.user), st=session.runtime,
|
| 705 |
-
granter=session.uname, steps=_test_account_steps(session))
|
| 706 |
-
except test_accounts.CopyRefused as exc:
|
| 707 |
-
raise _copy_refusal(exc)
|
| 708 |
-
except test_accounts.CopyFailed as exc:
|
| 709 |
-
raise _copy_failure(exc, "fully removed")
|
| 710 |
-
except Exception as exc: # noqa: BLE001
|
| 711 |
-
raise err(503, "store_unavailable", "The test account was not removed. Try again.") from exc
|
| 712 |
-
return Response(status_code=204)
|
| 713 |
-
|
| 714 |
-
|
| 715 |
# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
|
| 716 |
#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
|
| 717 |
#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
|
|
@@ -1464,6 +1291,8 @@ _VOCAB_ARM_CACHE = {}
|
|
| 1464 |
#: The one arm whose answer belongs to the CALLER's tenant, named once so both the memo and the
|
| 1465 |
#: stamp order can refuse it rather than each restating the rule.
|
| 1466 |
_TENANT_OWNED_ARM = "_product_shared_choices"
|
|
|
|
|
|
|
| 1467 |
|
| 1468 |
|
| 1469 |
def _server_side_vocabularies(session=None):
|
|
@@ -1560,15 +1389,31 @@ def _server_side_vocabularies(session=None):
|
|
| 1560 |
"occassion", "shape_style", "size", "sub_category")
|
| 1561 |
return {key: (stored.get(key) or {}).get("options") or () for key in keys}
|
| 1562 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1563 |
if _VOCAB_CACHE is not None:
|
| 1564 |
out = dict(_VOCAB_CACHE)
|
| 1565 |
-
|
| 1566 |
-
|
| 1567 |
-
|
| 1568 |
-
|
| 1569 |
-
|
| 1570 |
-
|
| 1571 |
-
|
|
|
|
| 1572 |
return out
|
| 1573 |
|
| 1574 |
out = {}
|
|
@@ -1582,7 +1427,7 @@ def _server_side_vocabularies(session=None):
|
|
| 1582 |
normalisation rather than two that can drift.
|
| 1583 |
"""
|
| 1584 |
name = getattr(fn, "__name__", "")
|
| 1585 |
-
memoable = name and name
|
| 1586 |
try:
|
| 1587 |
got = _VOCAB_ARM_CACHE[name] if memoable and name in _VOCAB_ARM_CACHE else fn()
|
| 1588 |
for key, values in (got or {}).items():
|
|
@@ -1708,6 +1553,7 @@ def _server_side_vocabularies(session=None):
|
|
| 1708 |
# them beside Odoo's shared Royal vocabulary would leak one tenant's Product taxonomy into
|
| 1709 |
# another tenant's permission editor for the life of the worker.
|
| 1710 |
arm(_product_shared_choices)
|
|
|
|
| 1711 |
return out
|
| 1712 |
|
| 1713 |
|
|
@@ -2294,7 +2140,7 @@ def _leaf_count(nodes):
|
|
| 2294 |
return n
|
| 2295 |
|
| 2296 |
|
| 2297 |
-
def _fold_legacy_scope(rec, cleaned):
|
| 2298 |
"""⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold.
|
| 2299 |
|
| 2300 |
Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent`
|
|
@@ -2343,7 +2189,8 @@ def _fold_legacy_scope(rec, cleaned):
|
|
| 2343 |
return cleaned # nothing to preserve: the account was unrestricted
|
| 2344 |
out = {}
|
| 2345 |
for key, e in cleaned.items():
|
| 2346 |
-
legacy_here = _prune_to_module(
|
|
|
|
| 2347 |
tree = e.get("filter")
|
| 2348 |
if not legacy_here:
|
| 2349 |
# This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than
|
|
@@ -2543,6 +2390,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2543 |
return {"username": uname,
|
| 2544 |
"perms": perms_out,
|
| 2545 |
"stored_perms": deepcopy(stored),
|
|
|
|
|
|
|
| 2546 |
# S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a
|
| 2547 |
# migrated record and LEGACY on an un-migrated one: opposite meanings for identical
|
| 2548 |
# JSON, so the client cannot render honestly without knowing which world it is in.
|
|
@@ -2758,7 +2607,7 @@ def grant_database_access(session, username, database, role):
|
|
| 2758 |
is_table = key.startswith(user_tables.KEY_PREFIX)
|
| 2759 |
if role not in ('view', 'edit') or (not is_table and role != 'view'):
|
| 2760 |
raise err(409, 'unsupported_role', 'This database supports only viewing access.')
|
| 2761 |
-
tenant = str(session.
|
| 2762 |
if _tenant_of(session.user) != tenant:
|
| 2763 |
raise err(403, 'forbidden', 'Database access could not be granted.')
|
| 2764 |
# Read the account store strictly rather than the directory's fail-silent projection.
|
|
@@ -2783,7 +2632,7 @@ def grant_database_access(session, username, database, role):
|
|
| 2783 |
candidate = dict(prior or {}, access=True)
|
| 2784 |
cleaned = _clean_perms({key: candidate}, governed_keys=governed,
|
| 2785 |
session=session, stored=rec.get('perms'))
|
| 2786 |
-
folded = _fold_legacy_scope(rec, cleaned)
|
| 2787 |
_guard_bu_widening(rec, folded, False)
|
| 2788 |
# Validation may prune retired fields. This click changes access, so retain stored walls.
|
| 2789 |
if folded[key].get('filter') != cleaned[key].get('filter'):
|
|
@@ -2867,6 +2716,84 @@ def grant_database_access(session, username, database, role):
|
|
| 2867 |
return {'username': uname, 'database': key, 'role': role}
|
| 2868 |
|
| 2869 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2870 |
@router.put("/admin/users/{username}/perms")
|
| 2871 |
def put_perms(username: str, body: dict = Body(default=None),
|
| 2872 |
session: Session = Depends(admin_gate)):
|
|
@@ -2914,7 +2841,8 @@ def put_perms(username: str, body: dict = Body(default=None),
|
|
| 2914 |
governed = {m["key"] for m in _mods}
|
| 2915 |
opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
|
| 2916 |
known = _fold_legacy_scope(reg[uname],
|
| 2917 |
-
{key: entry for key, entry in cleaned.items() if key in governed}
|
|
|
|
| 2918 |
_guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
|
| 2919 |
cleaned = dict(known, **opaque)
|
| 2920 |
users.set_access(uname, perms=cleaned)
|
|
@@ -2977,6 +2905,7 @@ def settings(session: Session = Depends(require_session)):
|
|
| 2977 |
would answer a question about our customer list.
|
| 2978 |
"""
|
| 2979 |
modules = perms.allowed_modules(session.user)
|
|
|
|
| 2980 |
from core import freeze_notice
|
| 2981 |
import process_role
|
| 2982 |
from request_timing import route_stats
|
|
@@ -2995,10 +2924,10 @@ def settings(session: Session = Depends(require_session)):
|
|
| 2995 |
# which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
|
| 2996 |
# Space secret at every deploy; absent means "somebody started this container by hand".
|
| 2997 |
"version": os.environ.get("AIOS_VERSION") or "unknown",
|
| 2998 |
-
"user":
|
| 2999 |
"scope": {
|
| 3000 |
"bus": perms.allowed_bu_labels(session.user),
|
| 3001 |
-
"team_id":
|
| 3002 |
"agent": perms.scope_agent(session.user),
|
| 3003 |
"modules": sorted(modules) if modules is not None else "all",
|
| 3004 |
# C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey
|
|
|
|
| 143 |
# editor will open, so the two cannot disagree.
|
| 144 |
"accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
|
| 145 |
surface_keys=surface_keys),
|
| 146 |
+
"perms_v": int(rec.get("perms_v") or 0)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
|
| 148 |
|
| 149 |
def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
|
|
|
|
| 454 |
if result == "last_admin":
|
| 455 |
raise err(400, "last_admin",
|
| 456 |
"an active tenant must retain at least one administrator")
|
|
|
|
|
|
|
|
|
|
| 457 |
if result != "updated":
|
| 458 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 459 |
|
|
|
|
| 539 |
return Response(status_code=204)
|
| 540 |
|
| 541 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 542 |
# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
|
| 543 |
#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
|
| 544 |
#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
|
|
|
|
| 1291 |
#: The one arm whose answer belongs to the CALLER's tenant, named once so both the memo and the
|
| 1292 |
#: stamp order can refuse it rather than each restating the rule.
|
| 1293 |
_TENANT_OWNED_ARM = "_product_shared_choices"
|
| 1294 |
+
#: Every arm whose answer depends on the session's tenant: never memoed, never in the process cache.
|
| 1295 |
+
_TENANT_OWNED_ARMS = frozenset({_TENANT_OWNED_ARM, "_dba_choices"})
|
| 1296 |
|
| 1297 |
|
| 1298 |
def _server_side_vocabularies(session=None):
|
|
|
|
| 1389 |
"occassion", "shape_style", "size", "sub_category")
|
| 1390 |
return {key: (stored.get(key) or {}).get("options") or () for key in keys}
|
| 1391 |
|
| 1392 |
+
def _dba_choices():
|
| 1393 |
+
"""The customer `dba` choices: the session tenant's team labels plus one multi-team value.
|
| 1394 |
+
|
| 1395 |
+
Read under the tenant's own Odoo binding, so its configured teams answer. A tenant with
|
| 1396 |
+
no Odoo source supplies nothing rather than the process default tenant's teams."""
|
| 1397 |
+
if session is None:
|
| 1398 |
+
return {}
|
| 1399 |
+
import modules.customer_data as customer_data
|
| 1400 |
+
|
| 1401 |
+
src = session.runtime.odoo_source()
|
| 1402 |
+
if src is None:
|
| 1403 |
+
return {}
|
| 1404 |
+
with src._bound():
|
| 1405 |
+
return {"dba": customer_data.dba_options()}
|
| 1406 |
+
|
| 1407 |
if _VOCAB_CACHE is not None:
|
| 1408 |
out = dict(_VOCAB_CACHE)
|
| 1409 |
+
for tenant_arm in (_product_shared_choices, _dba_choices):
|
| 1410 |
+
try:
|
| 1411 |
+
for key, values in tenant_arm().items():
|
| 1412 |
+
clean = sorted({str(v) for v in values if str(v or "").strip()})
|
| 1413 |
+
if clean:
|
| 1414 |
+
out[key] = clean
|
| 1415 |
+
except Exception: # noqa: BLE001
|
| 1416 |
+
pass
|
| 1417 |
return out
|
| 1418 |
|
| 1419 |
out = {}
|
|
|
|
| 1427 |
normalisation rather than two that can drift.
|
| 1428 |
"""
|
| 1429 |
name = getattr(fn, "__name__", "")
|
| 1430 |
+
memoable = name and name not in _TENANT_OWNED_ARMS
|
| 1431 |
try:
|
| 1432 |
got = _VOCAB_ARM_CACHE[name] if memoable and name in _VOCAB_ARM_CACHE else fn()
|
| 1433 |
for key, values in (got or {}).items():
|
|
|
|
| 1553 |
# them beside Odoo's shared Royal vocabulary would leak one tenant's Product taxonomy into
|
| 1554 |
# another tenant's permission editor for the life of the worker.
|
| 1555 |
arm(_product_shared_choices)
|
| 1556 |
+
arm(_dba_choices)
|
| 1557 |
return out
|
| 1558 |
|
| 1559 |
|
|
|
|
| 2140 |
return n
|
| 2141 |
|
| 2142 |
|
| 2143 |
+
def _fold_legacy_scope(rec, cleaned, session=None):
|
| 2144 |
"""⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold.
|
| 2145 |
|
| 2146 |
Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent`
|
|
|
|
| 2189 |
return cleaned # nothing to preserve: the account was unrestricted
|
| 2190 |
out = {}
|
| 2191 |
for key, e in cleaned.items():
|
| 2192 |
+
legacy_here = _prune_to_module(
|
| 2193 |
+
legacy, {f["key"] for f in _module_fields(key, session=session)})
|
| 2194 |
tree = e.get("filter")
|
| 2195 |
if not legacy_here:
|
| 2196 |
# This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than
|
|
|
|
| 2390 |
return {"username": uname,
|
| 2391 |
"perms": perms_out,
|
| 2392 |
"stored_perms": deepcopy(stored),
|
| 2393 |
+
# The `expect` that `PUT /admin/users/{u}/perms/{database}` compares before writing.
|
| 2394 |
+
"access_state": database_access_token(rec),
|
| 2395 |
# S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a
|
| 2396 |
# migrated record and LEGACY on an un-migrated one: opposite meanings for identical
|
| 2397 |
# JSON, so the client cannot render honestly without knowing which world it is in.
|
|
|
|
| 2607 |
is_table = key.startswith(user_tables.KEY_PREFIX)
|
| 2608 |
if role not in ('view', 'edit') or (not is_table and role != 'view'):
|
| 2609 |
raise err(409, 'unsupported_role', 'This database supports only viewing access.')
|
| 2610 |
+
tenant = str(session.tenant).strip().lower()
|
| 2611 |
if _tenant_of(session.user) != tenant:
|
| 2612 |
raise err(403, 'forbidden', 'Database access could not be granted.')
|
| 2613 |
# Read the account store strictly rather than the directory's fail-silent projection.
|
|
|
|
| 2632 |
candidate = dict(prior or {}, access=True)
|
| 2633 |
cleaned = _clean_perms({key: candidate}, governed_keys=governed,
|
| 2634 |
session=session, stored=rec.get('perms'))
|
| 2635 |
+
folded = _fold_legacy_scope(rec, cleaned, session=session)
|
| 2636 |
_guard_bu_widening(rec, folded, False)
|
| 2637 |
# Validation may prune retired fields. This click changes access, so retain stored walls.
|
| 2638 |
if folded[key].get('filter') != cleaned[key].get('filter'):
|
|
|
|
| 2716 |
return {'username': uname, 'database': key, 'role': role}
|
| 2717 |
|
| 2718 |
|
| 2719 |
+
def database_access_token(record):
|
| 2720 |
+
"""A stable digest of `users.database_access_state(record)` that a client sends back as `expect`."""
|
| 2721 |
+
import hashlib
|
| 2722 |
+
state = users.database_access_state(record or {})
|
| 2723 |
+
raw = json.dumps(state, sort_keys=True, separators=(",", ":"), default=str)
|
| 2724 |
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
| 2725 |
+
|
| 2726 |
+
|
| 2727 |
+
def set_database_block(session, username, database, entry, *, expect, confirm_widen=False):
|
| 2728 |
+
"""Write one database's access, row filter and hidden fields in one account compare-and-set.
|
| 2729 |
+
|
| 2730 |
+
`expect` is the `users.database_access_state` dict, or its `database_access_token`, the caller
|
| 2731 |
+
read. A changed account answers 409 and writes nothing.
|
| 2732 |
+
"""
|
| 2733 |
+
import copy
|
| 2734 |
+
from core import user_tables
|
| 2735 |
+
|
| 2736 |
+
admin_gate(session)
|
| 2737 |
+
uname = str(username or "").strip().lower()
|
| 2738 |
+
key = str(database or "").strip()
|
| 2739 |
+
if not isinstance(entry, dict) or type(entry.get("access")) is not bool:
|
| 2740 |
+
raise err(400, "bad_perms", "The entry needs access set to true or false.")
|
| 2741 |
+
if not isinstance(expect, (dict, str)) or not expect:
|
| 2742 |
+
raise err(400, "missing_expect", "Read the account permissions first, then send its access state.")
|
| 2743 |
+
tenant = str(session.tenant).strip().lower()
|
| 2744 |
+
if _tenant_of(session.user) != tenant:
|
| 2745 |
+
raise err(403, "forbidden", "Database access could not be changed.")
|
| 2746 |
+
# Read the account store strictly rather than the directory's fail-silent projection.
|
| 2747 |
+
reg = store.get("users", fresh=True)
|
| 2748 |
+
rec = (reg or {}).get(uname)
|
| 2749 |
+
if not isinstance(rec, dict) or not rec.get("active", True) or _tenant_of(rec) != tenant:
|
| 2750 |
+
raise err(404, "no_such_user", "That account is not active in this workspace.")
|
| 2751 |
+
if uname == session.uname:
|
| 2752 |
+
raise err(400, "self_perms", "Ask another administrator to change your access.")
|
| 2753 |
+
governed = {m["key"] for m in _perm_modules(session)}
|
| 2754 |
+
if key not in governed:
|
| 2755 |
+
raise err(404, "no_database", "That database is not available in this workspace.")
|
| 2756 |
+
state = users.database_access_state(rec)
|
| 2757 |
+
if (state if isinstance(expect, dict) else database_access_token(rec)) != expect:
|
| 2758 |
+
raise err(409, "permissions_changed", "Account access changed. Read it again and retry.")
|
| 2759 |
+
submitted = {field: copy.deepcopy(entry[field]) for field in users.DATABASE_ENTRY_KEYS
|
| 2760 |
+
if field in entry}
|
| 2761 |
+
cleaned = _clean_perms({key: submitted}, governed_keys=governed, session=session,
|
| 2762 |
+
stored=rec.get("perms"))
|
| 2763 |
+
folded = _fold_legacy_scope(rec, cleaned, session=session)
|
| 2764 |
+
_guard_bu_widening(rec, folded, bool(confirm_widen))
|
| 2765 |
+
candidate = {field: copy.deepcopy(folded[key][field]) for field in users.DATABASE_ENTRY_KEYS}
|
| 2766 |
+
prior = copy.deepcopy((rec.get("perms") or {}).get(key))
|
| 2767 |
+
result = users.set_database_access(uname, key, candidate, actor=session.uname,
|
| 2768 |
+
tenant=tenant, expect=state)
|
| 2769 |
+
status = result.get("status")
|
| 2770 |
+
if status == "conflict":
|
| 2771 |
+
raise err(409, "permissions_changed", "Account access changed. Read it again and retry.")
|
| 2772 |
+
if status == "missing":
|
| 2773 |
+
raise err(404, "no_such_user", "That account is not active in this workspace.")
|
| 2774 |
+
if status != "updated":
|
| 2775 |
+
raise err(403, "forbidden", "Database access could not be changed.")
|
| 2776 |
+
written = ((result.get("after") or {}).get("perms") or {}).get(key) or candidate
|
| 2777 |
+
if key.startswith(user_tables.KEY_PREFIX):
|
| 2778 |
+
_sync_database_shares(session, uname, {key: prior} if prior else {}, {key: written})
|
| 2779 |
+
return {"username": uname, "database": key, "entry": copy.deepcopy(written),
|
| 2780 |
+
"access_state": database_access_token(result.get("after"))}
|
| 2781 |
+
|
| 2782 |
+
|
| 2783 |
+
@router.put("/admin/users/{username}/perms/{database}")
|
| 2784 |
+
def put_database_perms(username: str, database: str, body: dict = Body(default=None),
|
| 2785 |
+
session: Session = Depends(admin_gate)):
|
| 2786 |
+
"""Replace one database's access, row filter and hidden fields if the account is unchanged.
|
| 2787 |
+
|
| 2788 |
+
Body: `{entry: {access, filter?, hiddenFields?}, expect: <access_state from GET perms>,
|
| 2789 |
+
confirm_widen?}`. A stale `expect` answers 409 and writes nothing.
|
| 2790 |
+
"""
|
| 2791 |
+
body = body if isinstance(body, dict) else {}
|
| 2792 |
+
return set_database_block(session, username, database, body.get("entry"),
|
| 2793 |
+
expect=body.get("expect"),
|
| 2794 |
+
confirm_widen=bool(body.get("confirm_widen")))
|
| 2795 |
+
|
| 2796 |
+
|
| 2797 |
@router.put("/admin/users/{username}/perms")
|
| 2798 |
def put_perms(username: str, body: dict = Body(default=None),
|
| 2799 |
session: Session = Depends(admin_gate)):
|
|
|
|
| 2841 |
governed = {m["key"] for m in _mods}
|
| 2842 |
opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
|
| 2843 |
known = _fold_legacy_scope(reg[uname],
|
| 2844 |
+
{key: entry for key, entry in cleaned.items() if key in governed},
|
| 2845 |
+
session=session)
|
| 2846 |
_guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
|
| 2847 |
cleaned = dict(known, **opaque)
|
| 2848 |
users.set_access(uname, perms=cleaned)
|
|
|
|
| 2905 |
would answer a question about our customer list.
|
| 2906 |
"""
|
| 2907 |
modules = perms.allowed_modules(session.user)
|
| 2908 |
+
public_user = _public_user(session.user)
|
| 2909 |
from core import freeze_notice
|
| 2910 |
import process_role
|
| 2911 |
from request_timing import route_stats
|
|
|
|
| 2924 |
# which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
|
| 2925 |
# Space secret at every deploy; absent means "somebody started this container by hand".
|
| 2926 |
"version": os.environ.get("AIOS_VERSION") or "unknown",
|
| 2927 |
+
"user": public_user,
|
| 2928 |
"scope": {
|
| 2929 |
"bus": perms.allowed_bu_labels(session.user),
|
| 2930 |
+
"team_id": public_user["team_id"],
|
| 2931 |
"agent": perms.scope_agent(session.user),
|
| 2932 |
"modules": sorted(modules) if modules is not None else "all",
|
| 2933 |
# C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey
|
api/routes_aged_stock.py
CHANGED
|
@@ -19,10 +19,9 @@ def _module():
|
|
| 19 |
|
| 20 |
|
| 21 |
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 22 |
-
"""
|
| 23 |
-
from
|
| 24 |
-
|
| 25 |
-
raise err(403, "aged_stock_forbidden", "Your account cannot read Amazon storage fees.")
|
| 26 |
return session
|
| 27 |
|
| 28 |
|
|
@@ -60,6 +59,8 @@ def _app(session, app_id):
|
|
| 60 |
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 61 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 62 |
raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
|
|
|
|
|
|
|
| 63 |
return app
|
| 64 |
|
| 65 |
|
|
@@ -140,9 +141,11 @@ def read_item(item_id: str, app: str = Query(..., min_length=1), offset: int = Q
|
|
| 140 |
@router.patch("/items/{item_id}")
|
| 141 |
def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 142 |
due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
|
|
|
|
| 143 |
session: Session = Depends(_app_writer)):
|
|
|
|
| 144 |
return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
|
| 145 |
-
action=action, note=note, user=session.uname or "unknown", st=session.runtime)
|
| 146 |
|
| 147 |
|
| 148 |
@router.get("/settings")
|
|
|
|
| 19 |
|
| 20 |
|
| 21 |
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 22 |
+
"""Admit a live App holder and refuse unsupported source row or field limits."""
|
| 23 |
+
from routes_app_dashboard import admit_reader
|
| 24 |
+
admit_reader(session, 'aged_stock')
|
|
|
|
| 25 |
return session
|
| 26 |
|
| 27 |
|
|
|
|
| 59 |
if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 60 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 61 |
raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
|
| 62 |
+
from routes_app_dashboard import admit_reader
|
| 63 |
+
admit_reader(session, 'aged_stock', app)
|
| 64 |
return app
|
| 65 |
|
| 66 |
|
|
|
|
| 141 |
@router.patch("/items/{item_id}")
|
| 142 |
def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
|
| 143 |
due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
|
| 144 |
+
price: StrictFloat | StrictInt | StrictStr | None = Body(None),
|
| 145 |
session: Session = Depends(_app_writer)):
|
| 146 |
+
"""A price is stored as a proposal on the item and never sent to Amazon. A blank price clears it."""
|
| 147 |
return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
|
| 148 |
+
action=action, note=note, price=price, user=session.uname or "unknown", st=session.runtime)
|
| 149 |
|
| 150 |
|
| 151 |
@router.get("/settings")
|
api/routes_amazon_dashboard.py
CHANGED
|
@@ -13,57 +13,28 @@ from modules import amazon_dashboard as dashboard_module
|
|
| 13 |
router = APIRouter(prefix="/api/v1/app-kinds/amazon_dashboard", tags=["amazon-dashboard"])
|
| 14 |
|
| 15 |
|
| 16 |
-
def _unrestricted_bu(user):
|
| 17 |
-
from core import perms, users
|
| 18 |
-
raw = user.get("bus", "all")
|
| 19 |
-
if raw == "all":
|
| 20 |
-
return "All" in perms.allowed_bu_labels(user)
|
| 21 |
-
if not isinstance(raw, (list, tuple)) or not raw:
|
| 22 |
-
return False
|
| 23 |
-
by_label = {label: key for key, label in users.BU_LABELS.items()}
|
| 24 |
-
identities = []
|
| 25 |
-
for value in raw:
|
| 26 |
-
if type(value) is int and value in users.BU_LABELS:
|
| 27 |
-
identities.append(value)
|
| 28 |
-
elif isinstance(value, str) and value in by_label:
|
| 29 |
-
identities.append(by_label[value])
|
| 30 |
-
elif value != "All":
|
| 31 |
-
return False
|
| 32 |
-
return bool(identities) and "All" in perms.allowed_bu_labels(dict(user, bus=identities))
|
| 33 |
-
|
| 34 |
-
|
| 35 |
def admit_sources(session):
|
| 36 |
"""Resolve source, recipient, field and BU walls before any mirror metadata or row I/O."""
|
| 37 |
-
|
|
|
|
| 38 |
runtime, user = session.runtime, session.user
|
| 39 |
if (not getattr(runtime, "key", None) or session.tenant != runtime.key
|
| 40 |
or user.get("tenant", session.tenant) != runtime.key):
|
| 41 |
raise err(403, "amazon_dashboard_forbidden", "The account does not belong to this workspace.")
|
| 42 |
-
if not _unrestricted_bu(user) or perms.scope_agent(user) is not None:
|
| 43 |
-
raise err(403, "amazon_dashboard_scope_unsupported", "This dashboard cannot apply your business unit or agent scope.")
|
| 44 |
allowed, refused = {}, {}
|
| 45 |
-
config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
|
| 46 |
-
tenant_modules = config.get("modules", "all")
|
| 47 |
for table, key in dashboard_module.SOURCE_DATABASES.items():
|
| 48 |
# No production cost mapping is available, so this door never opens the Odoo cost mirror.
|
| 49 |
if table == "product_product":
|
| 50 |
refused[table] = "No proven Amazon SKU and Odoo unit mapping is available for this period."
|
| 51 |
continue
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
| 55 |
continue
|
| 56 |
definition = user_tables.get_definition(key, st=runtime)
|
| 57 |
-
if not isinstance(definition, dict) or not definition.get("fields"):
|
| 58 |
-
refused[table] = "A required source database has no readable field definition."
|
| 59 |
-
continue
|
| 60 |
-
wall = perm_scope.entry(user, key) or {}
|
| 61 |
-
# Explicit source restrictions remain binding on aggregate reads, including admin records.
|
| 62 |
-
if (wall.get("access") is False or wall.get("filter") or wall.get("hiddenFields")
|
| 63 |
-
or perm_scope.row_scope_applies(user, key, st=runtime, defn=definition)
|
| 64 |
-
or perm_scope.hidden_keys(user, key, definition["fields"], st=runtime)
|
| 65 |
-
or route_system.is_managed(definition) or route_system.is_managed(key)):
|
| 66 |
-
raise err(403, "amazon_dashboard_scope_unsupported", "This dashboard cannot apply a source row or field access limit.")
|
| 67 |
allowed[table] = frozenset(str(field["key"]) for field in definition["fields"]
|
| 68 |
if isinstance(field, dict) and field.get("key"))
|
| 69 |
if "amzn_order_metrics" not in allowed:
|
|
@@ -86,14 +57,16 @@ class AdmittedReader(dashboard_module.MirrorReader):
|
|
| 86 |
|
| 87 |
|
| 88 |
def _app(session, app_id):
|
| 89 |
-
if not app_id:
|
| 90 |
-
if not session.admin:
|
| 91 |
-
raise err(403, "amazon_dashboard_app_required", "Open this dashboard from an App you can read.")
|
| 92 |
-
return
|
| 93 |
import application_registry
|
| 94 |
-
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
raise err(404, "amazon_dashboard_app_not_found", "That Amazon dashboard App does not exist.")
|
|
|
|
| 97 |
|
| 98 |
|
| 99 |
@router.get("/dashboard")
|
|
@@ -101,5 +74,7 @@ def read_dashboard(start: str | None = None, end: str | None = None, compare: st
|
|
| 101 |
app: str | None = None, session: Session = Depends(require_session)):
|
| 102 |
"""Serve the published Amazon route through canonical admission and figure validation."""
|
| 103 |
from routes_app_dashboard import read_dashboard as canonical_dashboard
|
|
|
|
|
|
|
| 104 |
return canonical_dashboard(kind="amazon_dashboard", app=app, start=start, end=end,
|
| 105 |
compare=compare, session=session)
|
|
|
|
| 13 |
router = APIRouter(prefix="/api/v1/app-kinds/amazon_dashboard", tags=["amazon-dashboard"])
|
| 14 |
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
def admit_sources(session):
|
| 17 |
"""Resolve source, recipient, field and BU walls before any mirror metadata or row I/O."""
|
| 18 |
+
import application_registry
|
| 19 |
+
from core import user_tables
|
| 20 |
runtime, user = session.runtime, session.user
|
| 21 |
if (not getattr(runtime, "key", None) or session.tenant != runtime.key
|
| 22 |
or user.get("tenant", session.tenant) != runtime.key):
|
| 23 |
raise err(403, "amazon_dashboard_forbidden", "The account does not belong to this workspace.")
|
|
|
|
|
|
|
| 24 |
allowed, refused = {}, {}
|
|
|
|
|
|
|
| 25 |
for table, key in dashboard_module.SOURCE_DATABASES.items():
|
| 26 |
# No production cost mapping is available, so this door never opens the Odoo cost mirror.
|
| 27 |
if table == "product_product":
|
| 28 |
refused[table] = "No proven Amazon SKU and Odoo unit mapping is available for this period."
|
| 29 |
continue
|
| 30 |
+
# R46: the shared check; explicit source restrictions bind aggregate reads, admin records too.
|
| 31 |
+
access = application_registry.app_access(user, runtime, None, (key,))
|
| 32 |
+
if access["code"] == "app_dashboard_scope":
|
| 33 |
+
raise err(403, "amazon_dashboard_scope_unsupported", access["message"])
|
| 34 |
+
if access["dashboard"] != "ready":
|
| 35 |
+
refused[table] = access["message"]
|
| 36 |
continue
|
| 37 |
definition = user_tables.get_definition(key, st=runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
allowed[table] = frozenset(str(field["key"]) for field in definition["fields"]
|
| 39 |
if isinstance(field, dict) and field.get("key"))
|
| 40 |
if "amzn_order_metrics" not in allowed:
|
|
|
|
| 57 |
|
| 58 |
|
| 59 |
def _app(session, app_id):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
import application_registry
|
| 61 |
+
record = (application_registry.load(session.runtime, strict=True)["applications"].get(app_id) if app_id else
|
| 62 |
+
application_registry.readable_app(session.runtime, session.tenant, session.uname,
|
| 63 |
+
session.admin, dashboard_module.KIND))
|
| 64 |
+
if (session.tenant != getattr(session.runtime, "key", None) or not isinstance(record, dict)
|
| 65 |
+
or record.get("tenant") != session.tenant or record.get("retired") is True
|
| 66 |
+
or record.get("templateKind") != dashboard_module.KIND
|
| 67 |
+
or not application_registry.may_read(record, session.uname, session.admin)):
|
| 68 |
raise err(404, "amazon_dashboard_app_not_found", "That Amazon dashboard App does not exist.")
|
| 69 |
+
return record["id"]
|
| 70 |
|
| 71 |
|
| 72 |
@router.get("/dashboard")
|
|
|
|
| 74 |
app: str | None = None, session: Session = Depends(require_session)):
|
| 75 |
"""Serve the published Amazon route through canonical admission and figure validation."""
|
| 76 |
from routes_app_dashboard import read_dashboard as canonical_dashboard
|
| 77 |
+
if not app:
|
| 78 |
+
app = _app(session, app)
|
| 79 |
return canonical_dashboard(kind="amazon_dashboard", app=app, start=start, end=end,
|
| 80 |
compare=compare, session=session)
|
api/routes_amazon_operator.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
"""Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
|
| 2 |
|
| 3 |
-
Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`
|
| 4 |
-
|
| 5 |
"""
|
| 6 |
from fastapi import APIRouter, Body, Depends, Query
|
| 7 |
|
|
@@ -77,13 +77,52 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 77 |
return session
|
| 78 |
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
@router.get("/today")
|
| 81 |
def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 82 |
app_cursor: str | None = Query(None), held_cursor: str | None = Query(None),
|
| 83 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 84 |
op = _module()
|
| 85 |
out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor,
|
| 86 |
-
app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q
|
|
|
|
| 87 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 88 |
|
| 89 |
|
|
@@ -110,7 +149,7 @@ def read_history(status: str | None = Query(None), cursor: str | None = Query(No
|
|
| 110 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 111 |
op = _module()
|
| 112 |
return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q,
|
| 113 |
-
st=session.runtime)
|
| 114 |
|
| 115 |
|
| 116 |
@router.get("/by-hour")
|
|
@@ -241,12 +280,16 @@ def approve_wasted(group_id: str, digest: str = Body(..., embed=True, min_length
|
|
| 241 |
@router.get("/results")
|
| 242 |
def read_results(proposal_id: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 243 |
op = _module()
|
| 244 |
-
return _call(op.results, _tenant(session), proposal_id, st=session.runtime
|
|
|
|
| 245 |
|
| 246 |
|
| 247 |
@router.post("/undo")
|
| 248 |
def undo(proposal_id: str = Body(..., embed=True, min_length=1), session: Session = Depends(_app_writer)):
|
| 249 |
op = _module()
|
|
|
|
|
|
|
|
|
|
| 250 |
return _call(op.propose_undo, _tenant(session), session.user, proposal_id, st=session.runtime)
|
| 251 |
|
| 252 |
|
|
@@ -286,12 +329,20 @@ def _weekly_context(st):
|
|
| 286 |
config = getattr(getattr(st, "tenant", None), "config", {}) or {}
|
| 287 |
timezone = config.get("timezone") or _module()._doc(st, "tenant_settings").get("timezone")
|
| 288 |
|
|
|
|
|
|
|
| 289 |
def may_read(app, owner):
|
| 290 |
-
|
| 291 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
and str(user.get("tenant") or getattr(st, "key", "")) == str(getattr(st, "key", ""))
|
| 293 |
and application_registry.may_read(
|
| 294 |
-
app, owner, is_admin=str(user.get("role") or "").lower() == "admin")
|
|
|
|
| 295 |
return {"apps": apps, "may_read": may_read, "timezone": timezone,
|
| 296 |
"base_url": os.environ.get("APP_BASE_URL") or ""}
|
| 297 |
|
|
@@ -302,7 +353,20 @@ def _weekly_delivery():
|
|
| 302 |
from notify_destinations import deliver
|
| 303 |
except ImportError:
|
| 304 |
return None
|
| 305 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 306 |
|
| 307 |
|
| 308 |
def _weekly_destinations(session):
|
|
@@ -352,6 +416,7 @@ def read_weekly(session: Session = Depends(_app_reader)):
|
|
| 352 |
if row.get("source_module") in op.PROPOSAL_SOURCES})}
|
| 353 |
summary = _call(weekly.compose, _tenant(session), session.uname, week, st=st, now=now,
|
| 354 |
source_state=source_state, **context)
|
|
|
|
| 355 |
return {**summary, "inbox_basis": "Open items in the saved Operator inbox.",
|
| 356 |
"destinations": _weekly_destinations(session)}
|
| 357 |
|
|
@@ -362,15 +427,38 @@ def send_weekly_now(owner: str | None = Body(None, embed=True), session: Session
|
|
| 362 |
if not session.admin:
|
| 363 |
raise err(403, "amazon_operator_admin_only", "Only an Administrator can send a weekly summary now.")
|
| 364 |
from modules import amazon_operator_weekly as weekly
|
| 365 |
-
|
| 366 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 367 |
|
| 368 |
|
| 369 |
@router.post("/approve")
|
| 370 |
-
def approve(item_ids: list[str] = Body(..., embed=True),
|
| 371 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 372 |
op = _module()
|
| 373 |
-
return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime
|
|
|
|
| 374 |
|
| 375 |
|
| 376 |
@router.post("/dismiss")
|
|
@@ -378,7 +466,7 @@ def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length=
|
|
| 378 |
session: Session = Depends(_app_writer)):
|
| 379 |
op = _module()
|
| 380 |
return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
|
| 381 |
-
st=session.runtime)
|
| 382 |
|
| 383 |
|
| 384 |
def _refresh_operator_tenant(tenant, st, *, chat=None):
|
|
@@ -394,7 +482,7 @@ def _refresh_operator_tenant(tenant, st, *, chat=None):
|
|
| 394 |
terms = wasted.wasted_terms(tenant, days, st=st)
|
| 395 |
before = wasted.saved(tenant, st=st)
|
| 396 |
if before.get("digest") != wasted.digest(terms):
|
| 397 |
-
grouped = ads_intent_groups.group(terms, chat=chat)
|
| 398 |
wasted.save(tenant, terms, grouped, days, st=st)
|
| 399 |
elif before.get("window_days") != days or "current_groups" not in before:
|
| 400 |
grouped = {**before, "groups": list(before["groups"].values())}
|
|
|
|
| 1 |
"""Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
|
| 2 |
|
| 3 |
+
Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`.
|
| 4 |
+
MCP approvals are refused. The router never calls the Ads client.
|
| 5 |
"""
|
| 6 |
from fastapi import APIRouter, Body, Depends, Query
|
| 7 |
|
|
|
|
| 77 |
return session
|
| 78 |
|
| 79 |
|
| 80 |
+
def _source_reader(user, st, *, weekly=False):
|
| 81 |
+
"""Cache Dashboard admission per source kind for one viewer and request."""
|
| 82 |
+
import application_registry
|
| 83 |
+
import app_templates
|
| 84 |
+
cache = {}
|
| 85 |
+
|
| 86 |
+
def allowed(source):
|
| 87 |
+
if str(source or "").startswith("modules.") and source not in _module().PROPOSAL_SOURCES:
|
| 88 |
+
return False
|
| 89 |
+
kind = str(source or "").removeprefix("modules.")
|
| 90 |
+
if kind not in cache:
|
| 91 |
+
cache[kind] = False
|
| 92 |
+
try:
|
| 93 |
+
template = app_templates.BY_KIND.get(kind)
|
| 94 |
+
if not isinstance(template, dict):
|
| 95 |
+
return False
|
| 96 |
+
required = template.get("sources")
|
| 97 |
+
optional = template.get("optional_sources", ())
|
| 98 |
+
if not isinstance(required, (list, tuple)) or not required or not isinstance(optional, (list, tuple)):
|
| 99 |
+
return False
|
| 100 |
+
sources = list(dict.fromkeys((*required, *optional)))
|
| 101 |
+
# Removal prose includes ship-to details even when these sources become optional.
|
| 102 |
+
if kind == "returns_refunds":
|
| 103 |
+
sources = list(dict.fromkeys((*sources, "ut_amzn_removal_orders", "ut_amzn_removal_shipments")))
|
| 104 |
+
if weekly:
|
| 105 |
+
sources = list(dict.fromkeys((*sources, "ut_amazon_sku_profit", "ut_amzn_inventory_planning")))
|
| 106 |
+
if any(not isinstance(key, str) or not key for key in sources):
|
| 107 |
+
return False
|
| 108 |
+
access = application_registry.app_access(user, st, {"templateKind": kind}, sources)
|
| 109 |
+
cache[kind] = access.get("dashboard") == "ready"
|
| 110 |
+
except Exception:
|
| 111 |
+
import logging
|
| 112 |
+
logging.getLogger(__name__).exception("Operator source access check failed for %s", kind)
|
| 113 |
+
return cache[kind]
|
| 114 |
+
|
| 115 |
+
return allowed
|
| 116 |
+
|
| 117 |
+
|
| 118 |
@router.get("/today")
|
| 119 |
def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
|
| 120 |
app_cursor: str | None = Query(None), held_cursor: str | None = Query(None),
|
| 121 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 122 |
op = _module()
|
| 123 |
out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor,
|
| 124 |
+
app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q,
|
| 125 |
+
source_allowed=_source_reader(session.user, session.runtime))
|
| 126 |
return {**out, "canEdit": _may_edit(session, _app(session, app))}
|
| 127 |
|
| 128 |
|
|
|
|
| 149 |
limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
|
| 150 |
op = _module()
|
| 151 |
return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q,
|
| 152 |
+
st=session.runtime, source_allowed=_source_reader(session.user, session.runtime))
|
| 153 |
|
| 154 |
|
| 155 |
@router.get("/by-hour")
|
|
|
|
| 280 |
@router.get("/results")
|
| 281 |
def read_results(proposal_id: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
|
| 282 |
op = _module()
|
| 283 |
+
return _call(op.results, _tenant(session), proposal_id, st=session.runtime,
|
| 284 |
+
source_allowed=_source_reader(session.user, session.runtime))
|
| 285 |
|
| 286 |
|
| 287 |
@router.post("/undo")
|
| 288 |
def undo(proposal_id: str = Body(..., embed=True, min_length=1), session: Session = Depends(_app_writer)):
|
| 289 |
op = _module()
|
| 290 |
+
allowed = _source_reader(session.user, session.runtime)
|
| 291 |
+
if not allowed(TEMPLATE_KIND):
|
| 292 |
+
return op.withheld(1)
|
| 293 |
return _call(op.propose_undo, _tenant(session), session.user, proposal_id, st=session.runtime)
|
| 294 |
|
| 295 |
|
|
|
|
| 329 |
config = getattr(getattr(st, "tenant", None), "config", {}) or {}
|
| 330 |
timezone = config.get("timezone") or _module()._doc(st, "tenant_settings").get("timezone")
|
| 331 |
|
| 332 |
+
readers = {}
|
| 333 |
+
|
| 334 |
def may_read(app, owner):
|
| 335 |
+
account = _module().account_get(owner, st=st) or {}
|
| 336 |
+
user = {**account, "username": owner} if account else {}
|
| 337 |
+
if owner not in readers:
|
| 338 |
+
readers[owner] = _source_reader(user, st, weekly=True)
|
| 339 |
+
return bool(isinstance(app, dict) and not app.get("retired")
|
| 340 |
+
and app.get("tenant") == str(getattr(st, "key", ""))
|
| 341 |
+
and user and user.get("active", True)
|
| 342 |
and str(user.get("tenant") or getattr(st, "key", "")) == str(getattr(st, "key", ""))
|
| 343 |
and application_registry.may_read(
|
| 344 |
+
app, owner, is_admin=str(user.get("role") or "").lower() == "admin")
|
| 345 |
+
and readers[owner](app.get("templateKind")))
|
| 346 |
return {"apps": apps, "may_read": may_read, "timezone": timezone,
|
| 347 |
"base_url": os.environ.get("APP_BASE_URL") or ""}
|
| 348 |
|
|
|
|
| 353 |
from notify_destinations import deliver
|
| 354 |
except ImportError:
|
| 355 |
return None
|
| 356 |
+
def guarded(st, kind, items, *, user, **kwargs):
|
| 357 |
+
context = _weekly_context(st)
|
| 358 |
+
apps = {str(app.get("id")): app for app in context["apps"]}
|
| 359 |
+
kept = []
|
| 360 |
+
for item in items:
|
| 361 |
+
source = item.get("source") or {}
|
| 362 |
+
ids = source.get("rowIds") or []
|
| 363 |
+
if (source.get("table") == "applications" and ids
|
| 364 |
+
and all(str(key) in apps and context["may_read"](apps[str(key)], user) for key in ids)):
|
| 365 |
+
kept.append(item)
|
| 366 |
+
if not kept:
|
| 367 |
+
return {"state": "empty", "withheldCount": len(items), "results": []}
|
| 368 |
+
return deliver(st, kind, kept, user=user, **kwargs)
|
| 369 |
+
return guarded
|
| 370 |
|
| 371 |
|
| 372 |
def _weekly_destinations(session):
|
|
|
|
| 416 |
if row.get("source_module") in op.PROPOSAL_SOURCES})}
|
| 417 |
summary = _call(weekly.compose, _tenant(session), session.uname, week, st=st, now=now,
|
| 418 |
source_state=source_state, **context)
|
| 419 |
+
# Compose independently validates the count for this recipient's owned items.
|
| 420 |
return {**summary, "inbox_basis": "Open items in the saved Operator inbox.",
|
| 421 |
"destinations": _weekly_destinations(session)}
|
| 422 |
|
|
|
|
| 427 |
if not session.admin:
|
| 428 |
raise err(403, "amazon_operator_admin_only", "Only an Administrator can send a weekly summary now.")
|
| 429 |
from modules import amazon_operator_weekly as weekly
|
| 430 |
+
result = _call(weekly.send_tenant, _tenant(session), session.runtime, force=True, owner=owner,
|
| 431 |
+
deliver=_weekly_delivery(), **_call(_weekly_context, session.runtime))
|
| 432 |
+
allowed = _source_reader(session.user, session.runtime, weekly=True)
|
| 433 |
+
rows = []
|
| 434 |
+
for row in result.get("rows", []):
|
| 435 |
+
row = dict(row)
|
| 436 |
+
row.pop("delivery", None)
|
| 437 |
+
if isinstance(row.get("summary"), dict):
|
| 438 |
+
summary = dict(row["summary"])
|
| 439 |
+
entries = summary.get("apps", [])
|
| 440 |
+
summary["apps"] = [app for app in entries if allowed(app.get("kind"))]
|
| 441 |
+
# Validation contains App names and figures too.
|
| 442 |
+
summary.pop("validation", None)
|
| 443 |
+
row["summary"] = summary
|
| 444 |
+
rows.append(row)
|
| 445 |
+
# Recipient counts survive caller text filtering without an all-owner recount.
|
| 446 |
+
return {**result, "rows": rows}
|
| 447 |
|
| 448 |
|
| 449 |
@router.post("/approve")
|
| 450 |
+
def approve(item_ids: list[str] = Body(..., embed=True), reviews: dict | None = Body(None, embed=True),
|
| 451 |
+
session: Session = Depends(_app_writer)):
|
| 452 |
+
"""Approve the ticked proposals as one batch. Returns each row's outcome.
|
| 453 |
+
|
| 454 |
+
`reviews` maps each removal's inbox id to the `review.reviewed` value shown in its drawer. Only a person
|
| 455 |
+
signed in to the App approves: an API key or OAuth session is refused before anything is stored.
|
| 456 |
+
"""
|
| 457 |
+
if (session.claims or {}).get("external"):
|
| 458 |
+
raise err(403, "amazon_operator_internal_approval", "Approve these inside the Operator inbox.")
|
| 459 |
op = _module()
|
| 460 |
+
return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime,
|
| 461 |
+
reviews=reviews, source_allowed=_source_reader(session.user, session.runtime))
|
| 462 |
|
| 463 |
|
| 464 |
@router.post("/dismiss")
|
|
|
|
| 466 |
session: Session = Depends(_app_writer)):
|
| 467 |
op = _module()
|
| 468 |
return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
|
| 469 |
+
st=session.runtime, source_allowed=_source_reader(session.user, session.runtime))
|
| 470 |
|
| 471 |
|
| 472 |
def _refresh_operator_tenant(tenant, st, *, chat=None):
|
|
|
|
| 482 |
terms = wasted.wasted_terms(tenant, days, st=st)
|
| 483 |
before = wasted.saved(tenant, st=st)
|
| 484 |
if before.get("digest") != wasted.digest(terms):
|
| 485 |
+
grouped = ads_intent_groups.group(terms, chat=chat, st=st)
|
| 486 |
wasted.save(tenant, terms, grouped, days, st=st)
|
| 487 |
elif before.get("window_days") != days or "current_groups" not in before:
|
| 488 |
grouped = {**before, "groups": list(before["groups"].values())}
|
api/routes_amazon_profit.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""Command Center doors: read the settings, switch the period, recompute, set the owner,
|
| 2 |
-
|
| 3 |
|
| 4 |
The table itself (`ut_amazon_sku_profit`) is read and edited through the ordinary table doors.
|
| 5 |
A refresh rewrites only computed columns, so owner, status and due date survive a period switch.
|
|
@@ -12,7 +12,7 @@ from typing import Any
|
|
| 12 |
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 13 |
|
| 14 |
from deps import Session, err, require_session
|
| 15 |
-
from modules import amazon_profit, amazon_profit_account
|
| 16 |
|
| 17 |
router = APIRouter(prefix="/api/v1")
|
| 18 |
|
|
@@ -96,6 +96,8 @@ def profit_status(session: Session = Depends(require_session)):
|
|
| 96 |
"linked": settings.get("linked"),
|
| 97 |
"verifiedMonth": settings.get("verifiedMonth"),
|
| 98 |
"refusal": settings.get("refusal"),
|
|
|
|
|
|
|
| 99 |
"defaultOwner": settings["defaultOwner"],
|
| 100 |
"periods": amazon_profit.period_choices(),
|
| 101 |
"statuses": list(amazon_profit.STATUSES),
|
|
@@ -109,6 +111,26 @@ def profit_status(session: Session = Depends(require_session)):
|
|
| 109 |
"canAdmin": bool(session.admin)}
|
| 110 |
|
| 111 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
@router.get("/amazon-profit/account")
|
| 113 |
def profit_account(period: str | None = None, session: Session = Depends(require_session)):
|
| 114 |
"""Read this tenant's account P&L without refreshing costs, caches or history."""
|
|
@@ -158,7 +180,8 @@ def profit_what_if(sku: str, price: float | None = None, landed_cost: float | No
|
|
| 158 |
|
| 159 |
|
| 160 |
def _recompute(session, period=None, thin=None):
|
| 161 |
-
"""Run one refresh through the tenant's mirror receipts and return 409 on a failed tie.
|
|
|
|
| 162 |
try:
|
| 163 |
if period is not None:
|
| 164 |
amazon_profit.period_bounds(period)
|
|
@@ -207,16 +230,6 @@ def profit_period(body: Any = Body(default=None), session: Session = Depends(req
|
|
| 207 |
return _recompute(session, period)
|
| 208 |
|
| 209 |
|
| 210 |
-
@router.post("/amazon-profit/projects/relink")
|
| 211 |
-
def projects_relink(session: Session = Depends(require_session)):
|
| 212 |
-
"""Write every project's row id onto its SKU, ASIN and parent ASIN rows; editors only."""
|
| 213 |
-
if not _may_edit(session):
|
| 214 |
-
raise err(403, "forbidden", "Only an editor of Command Center can relink projects.")
|
| 215 |
-
with amazon_profit._LOCK:
|
| 216 |
-
linked = amazon_profit.relink_projects(session.runtime)
|
| 217 |
-
return {"ok": True, "linked": linked}
|
| 218 |
-
|
| 219 |
-
|
| 220 |
@router.put("/amazon-profit/settings")
|
| 221 |
def profit_settings(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 222 |
"""Administrator settings: the default owner of new items and the thin margin."""
|
|
@@ -238,12 +251,13 @@ def profit_settings(body: Any = Body(default=None), session: Session = Depends(r
|
|
| 238 |
|
| 239 |
@router.post("/amazon-profit/matches/{sku:path}/{decision}")
|
| 240 |
def profit_match(sku: str, decision: str, session: Session = Depends(require_session)):
|
| 241 |
-
"""
|
| 242 |
|
| 243 |
-
Confirm copies the suggestion into the link and recomputes
|
| 244 |
-
|
|
|
|
| 245 |
import core.user_tables as ut
|
| 246 |
-
states = {"confirm": amazon_profit.
|
| 247 |
if decision not in states:
|
| 248 |
raise err(404, "not_found", "Choose confirm or reject.")
|
| 249 |
if not _may_edit(session):
|
|
@@ -266,12 +280,142 @@ def profit_match(sku: str, decision: str, session: Session = Depends(require_ses
|
|
| 266 |
"unitsPerSku": (result["row"] or {}).get("units_per_sku")}
|
| 267 |
if decision == "confirm":
|
| 268 |
try:
|
| 269 |
-
|
| 270 |
-
|
|
|
|
| 271 |
except HTTPException as exc:
|
| 272 |
out["refreshError"] = ((exc.detail or {}).get("error") or {}).get("message")
|
|
|
|
|
|
|
| 273 |
current = next((r for r in amazon_profit.table_rows(session.runtime).values()
|
| 274 |
if str(r.get("sku") or "").strip() == sku), {})
|
| 275 |
out.update(cogsPerUnit=current.get("cogs_per_unit"), netProfit=current.get("net_profit"),
|
| 276 |
margin=current.get("margin"), band=current.get("band"))
|
| 277 |
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""Command Center doors: read the settings, switch the period, recompute, set the owner,
|
| 2 |
+
link or reject a suggested Odoo match at once.
|
| 3 |
|
| 4 |
The table itself (`ut_amazon_sku_profit`) is read and edited through the ordinary table doors.
|
| 5 |
A refresh rewrites only computed columns, so owner, status and due date survive a period switch.
|
|
|
|
| 12 |
from fastapi import APIRouter, Body, Depends, HTTPException
|
| 13 |
|
| 14 |
from deps import Session, err, require_session
|
| 15 |
+
from modules import amazon_data_entry, amazon_profit, amazon_profit_account
|
| 16 |
|
| 17 |
router = APIRouter(prefix="/api/v1")
|
| 18 |
|
|
|
|
| 96 |
"linked": settings.get("linked"),
|
| 97 |
"verifiedMonth": settings.get("verifiedMonth"),
|
| 98 |
"refusal": settings.get("refusal"),
|
| 99 |
+
"verified": settings.get("verified"),
|
| 100 |
+
"lastRefusal": settings.get("lastRefusal"),
|
| 101 |
"defaultOwner": settings["defaultOwner"],
|
| 102 |
"periods": amazon_profit.period_choices(),
|
| 103 |
"statuses": list(amazon_profit.STATUSES),
|
|
|
|
| 111 |
"canAdmin": bool(session.admin)}
|
| 112 |
|
| 113 |
|
| 114 |
+
COMMAND_CENTER_KEYS = (amazon_profit.TABLE_KEY, amazon_profit.ASIN_KEY, amazon_profit.PARENT_KEY,
|
| 115 |
+
amazon_profit.PROJECTS_KEY, amazon_data_entry.TABLE_KEY)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
@router.get("/amazon-profit/tables")
|
| 119 |
+
def profit_tables(session: Session = Depends(require_session)):
|
| 120 |
+
"""Whether each Command Center table this session may open exists.
|
| 121 |
+
|
| 122 |
+
Each table is walled by its own `session.require`, never by `_account_access`, and no row is read."""
|
| 123 |
+
import core.user_tables as ut
|
| 124 |
+
tables = {}
|
| 125 |
+
for key in COMMAND_CENTER_KEYS:
|
| 126 |
+
try:
|
| 127 |
+
session.require(key)
|
| 128 |
+
except HTTPException:
|
| 129 |
+
continue
|
| 130 |
+
tables[key] = ut.get_definition(key, session.runtime) is not None
|
| 131 |
+
return {"tables": tables}
|
| 132 |
+
|
| 133 |
+
|
| 134 |
@router.get("/amazon-profit/account")
|
| 135 |
def profit_account(period: str | None = None, session: Session = Depends(require_session)):
|
| 136 |
"""Read this tenant's account P&L without refreshing costs, caches or history."""
|
|
|
|
| 180 |
|
| 181 |
|
| 182 |
def _recompute(session, period=None, thin=None):
|
| 183 |
+
"""Run one refresh through the tenant's mirror receipts and return 409 on a failed tie.
|
| 184 |
+
A 409 left the stored rows and period as they were."""
|
| 185 |
try:
|
| 186 |
if period is not None:
|
| 187 |
amazon_profit.period_bounds(period)
|
|
|
|
| 230 |
return _recompute(session, period)
|
| 231 |
|
| 232 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
@router.put("/amazon-profit/settings")
|
| 234 |
def profit_settings(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 235 |
"""Administrator settings: the default owner of new items and the thin margin."""
|
|
|
|
| 251 |
|
| 252 |
@router.post("/amazon-profit/matches/{sku:path}/{decision}")
|
| 253 |
def profit_match(sku: str, decision: str, session: Session = Depends(require_session)):
|
| 254 |
+
"""Link or reject the suggested Odoo product for one SKU; editors only.
|
| 255 |
|
| 256 |
+
Confirm copies the suggestion into the link and recomputes that SKU's cost and profit in the
|
| 257 |
+
same request (`recompute_sku`), so a refused refresh never blocks it. Reject clears the
|
| 258 |
+
suggestion, keeps the SKU unlinked and remembers the code."""
|
| 259 |
import core.user_tables as ut
|
| 260 |
+
states = {"confirm": amazon_profit.LINKED, "reject": amazon_profit.WRONG_MATCH}
|
| 261 |
if decision not in states:
|
| 262 |
raise err(404, "not_found", "Choose confirm or reject.")
|
| 263 |
if not _may_edit(session):
|
|
|
|
| 280 |
"unitsPerSku": (result["row"] or {}).get("units_per_sku")}
|
| 281 |
if decision == "confirm":
|
| 282 |
try:
|
| 283 |
+
linked = link_now(session.runtime, sku, by=session.uname)
|
| 284 |
+
if linked.get("refusal"):
|
| 285 |
+
out["refreshError"] = linked["refusal"]
|
| 286 |
except HTTPException as exc:
|
| 287 |
out["refreshError"] = ((exc.detail or {}).get("error") or {}).get("message")
|
| 288 |
+
except (amazon_profit.SourceUnavailable, amazon_profit.TableRefused) as exc:
|
| 289 |
+
out["refreshError"] = str(exc)
|
| 290 |
current = next((r for r in amazon_profit.table_rows(session.runtime).values()
|
| 291 |
if str(r.get("sku") or "").strip() == sku), {})
|
| 292 |
out.update(cogsPerUnit=current.get("cogs_per_unit"), netProfit=current.get("net_profit"),
|
| 293 |
margin=current.get("margin"), band=current.get("band"))
|
| 294 |
return out
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def _runtime_of(st):
|
| 298 |
+
"""The tenant runtime behind a lent store snapshot; the mirror and the writes need it."""
|
| 299 |
+
import core.user_tables as ut
|
| 300 |
+
lent = getattr(ut, "_Lent", None)
|
| 301 |
+
while lent is not None and isinstance(st, lent):
|
| 302 |
+
st = st._runtime
|
| 303 |
+
return st
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def link_now(runtime, sku, *, by=""):
|
| 307 |
+
"""Recompute one SKU's cost and profit from its Odoo link on this tenant's mirror."""
|
| 308 |
+
con = _mirror(runtime)
|
| 309 |
+
try:
|
| 310 |
+
result = amazon_profit.recompute_sku(runtime, con, sku, by=by, cost_reader=_cost_reader(runtime))
|
| 311 |
+
finally:
|
| 312 |
+
try:
|
| 313 |
+
con.close()
|
| 314 |
+
except Exception: # noqa: BLE001
|
| 315 |
+
pass
|
| 316 |
+
# S82-T71: a linked SKU that is now complete leaves Data Entry in the same request.
|
| 317 |
+
amazon_data_entry.sync(runtime, skus=[sku])
|
| 318 |
+
return result
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
#: Cell edits on the SKU table that change its Odoo link.
|
| 322 |
+
LINK_FIELDS = ("match_state", "odoo_product", "units_per_sku", "amazon_cogs")
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def link_hook(evt):
|
| 326 |
+
"""A person's Link status, Odoo product or Units per SKU edit links that SKU at once.
|
| 327 |
+
|
| 328 |
+
Registered on `core.user_tables.ROW_HOOKS`, which only the human write doors emit into.
|
| 329 |
+
Its own writes go through `update_with_activity`, so it never fires itself."""
|
| 330 |
+
if str(evt.get("type") or "") != "event_field" or str(evt.get("table") or "") != amazon_profit.TABLE_KEY:
|
| 331 |
+
return
|
| 332 |
+
field = str(evt.get("field") or "")
|
| 333 |
+
if field not in LINK_FIELDS or evt.get("st") is None:
|
| 334 |
+
return
|
| 335 |
+
runtime = _runtime_of(evt["st"])
|
| 336 |
+
row = amazon_profit.table_rows(runtime).get(str(evt.get("rowId") or "")) or {}
|
| 337 |
+
sku = str(row.get("sku") or "").strip()
|
| 338 |
+
if not sku:
|
| 339 |
+
return
|
| 340 |
+
by = str(evt.get("user") or "") or amazon_profit.WRITER
|
| 341 |
+
if field == "match_state":
|
| 342 |
+
state = str(evt.get("after") or "").strip()
|
| 343 |
+
state = amazon_profit.LEGACY_MATCH_STATES.get(state, state)
|
| 344 |
+
if state not in (amazon_profit.LINKED, amazon_profit.WRONG_MATCH):
|
| 345 |
+
return
|
| 346 |
+
with amazon_profit._LOCK:
|
| 347 |
+
result = amazon_profit.apply_decisions(runtime, by=by, sku=sku, state=state)
|
| 348 |
+
if state != amazon_profit.LINKED or not result.get("confirmed"):
|
| 349 |
+
return
|
| 350 |
+
link_now(runtime, sku, by=by)
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
# One registration per process: a reimport replaces the hook rather than adding a second one.
|
| 354 |
+
import core.user_tables as _ut_hooks # noqa: E402
|
| 355 |
+
|
| 356 |
+
_ut_hooks.ROW_HOOKS[:] = [h for h in _ut_hooks.ROW_HOOKS
|
| 357 |
+
if (getattr(h, "__module__", None), getattr(h, "__name__", None)) != (__name__, "link_hook")]
|
| 358 |
+
_ut_hooks.ROW_HOOKS.append(link_hook)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def data_entry_hook(evt):
|
| 362 |
+
"""A person's Data Entry cell edit writes that value to the SKU table in the same request.
|
| 363 |
+
|
| 364 |
+
Registered on `core.user_tables.ROW_HOOKS` and filtered on the Data Entry key. The write-back
|
| 365 |
+
goes through `patch_cells`, which emits no row event, so it never fires itself or `link_hook`.
|
| 366 |
+
A link edit then links the SKU at once, and the row is rebuilt: a complete SKU leaves (R8)."""
|
| 367 |
+
if (str(evt.get("type") or "") != "event_field"
|
| 368 |
+
or str(evt.get("table") or "") != amazon_data_entry.TABLE_KEY or evt.get("st") is None):
|
| 369 |
+
return
|
| 370 |
+
field = str(evt.get("field") or "")
|
| 371 |
+
if field not in amazon_data_entry.ENTRY_KEYS:
|
| 372 |
+
return
|
| 373 |
+
runtime = _runtime_of(evt["st"])
|
| 374 |
+
user = str(evt.get("user") or "")
|
| 375 |
+
result = amazon_data_entry.write_back(runtime, str(evt.get("rowId") or ""), field, evt.get("after"), user=user)
|
| 376 |
+
sku = result.get("sku")
|
| 377 |
+
if not sku:
|
| 378 |
+
return
|
| 379 |
+
try:
|
| 380 |
+
if result["written"] and field in (*amazon_data_entry.LINK_KEYS, "amazon_cogs"):
|
| 381 |
+
state = result["changes"].get("match_state")
|
| 382 |
+
if state in (amazon_profit.LINKED, amazon_profit.WRONG_MATCH):
|
| 383 |
+
with amazon_profit._LOCK:
|
| 384 |
+
amazon_profit.apply_decisions(runtime, by=user or amazon_profit.WRITER, sku=sku, state=state)
|
| 385 |
+
row = next((r for r in amazon_profit.table_rows(runtime).values()
|
| 386 |
+
if str(r.get("sku") or "").strip() == sku), {})
|
| 387 |
+
if str(row.get("odoo_product") or "").strip() or field == "amazon_cogs":
|
| 388 |
+
try:
|
| 389 |
+
link_now(runtime, sku, by=user)
|
| 390 |
+
except (HTTPException, amazon_profit.SourceUnavailable, amazon_profit.TableRefused) as exc:
|
| 391 |
+
print(f"[amazon-profit] Data Entry link for {sku} not recomputed: {exc}")
|
| 392 |
+
finally:
|
| 393 |
+
amazon_data_entry.sync(runtime, skus=[sku])
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
_ut_hooks.ROW_HOOKS[:] = [h for h in _ut_hooks.ROW_HOOKS
|
| 397 |
+
if (getattr(h, "__module__", None), getattr(h, "__name__", None)) != (__name__, "data_entry_hook")]
|
| 398 |
+
_ut_hooks.ROW_HOOKS.append(data_entry_hook)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
@router.get("/amazon-profit/fba-capacity")
|
| 402 |
+
def fba_capacity_read(session: Session = Depends(require_session)):
|
| 403 |
+
"""The one typed FBA capacity Command Center and Restock share."""
|
| 404 |
+
from core import amazon_inputs
|
| 405 |
+
session.require(amazon_profit.TABLE_KEY)
|
| 406 |
+
return {"fbaCapacity": amazon_inputs.fba_capacity(session.runtime), "units": list(amazon_inputs.UNITS),
|
| 407 |
+
"canEdit": _may_edit(session)}
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
@router.put("/amazon-profit/fba-capacity")
|
| 411 |
+
def fba_capacity_write(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 412 |
+
"""Save the typed FBA capacity `{value, unit}`, or clear it with null; editors only."""
|
| 413 |
+
from core import amazon_inputs
|
| 414 |
+
if not _may_edit(session):
|
| 415 |
+
raise err(403, "forbidden", "Only an editor of Command Center can change FBA capacity.")
|
| 416 |
+
body = body if isinstance(body, dict) else {}
|
| 417 |
+
try:
|
| 418 |
+
stored = amazon_inputs.set_fba_capacity(session.runtime, body.get("fbaCapacity"), by=session.uname)
|
| 419 |
+
except ValueError as exc:
|
| 420 |
+
raise err(400, "invalid_setting", str(exc)) from None
|
| 421 |
+
return {"fbaCapacity": stored}
|
api/routes_amazon_projects.py
ADDED
|
@@ -0,0 +1,703 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Projects doors: a row's projects, one project with its targets and activity, edits, and target search.
|
| 2 |
+
|
| 3 |
+
Editors of Command Center write and a viewer reads (R16, R17). Project rows live in the per-tenant
|
| 4 |
+
store (C6). Search and target details read the Amazon mirror, never `user_tables`. Each write
|
| 5 |
+
repaints the `projects` and `projects_label` cells of only the Command Center rows it touches.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from contextlib import contextmanager
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import sys
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
| 16 |
+
|
| 17 |
+
from core import amazon_projects as projects
|
| 18 |
+
from core import amazon_project_results as project_results
|
| 19 |
+
from core import users
|
| 20 |
+
from deps import Session, err, require_session
|
| 21 |
+
from modules import amazon_profit
|
| 22 |
+
from modules import sku_action_results
|
| 23 |
+
import routes_amazon_profit
|
| 24 |
+
import routes_sku_actions
|
| 25 |
+
|
| 26 |
+
router = APIRouter(prefix="/api/v1/amazon-projects")
|
| 27 |
+
|
| 28 |
+
FORBIDDEN = "Only an editor of Command Center can change projects."
|
| 29 |
+
NO_MIRROR = "This workspace has no Amazon data yet."
|
| 30 |
+
UNREADABLE = "Amazon data could not be read. Try again shortly."
|
| 31 |
+
NOT_FOUND = "That project does not exist."
|
| 32 |
+
#: Contract field name to store field.
|
| 33 |
+
FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "owner",
|
| 34 |
+
"startOn": "start_on", "targetOn": "target_on", "progress": "progress", "notes": "notes",
|
| 35 |
+
"completedOn": "completed_on", "resultConfigs": "result_configs"}
|
| 36 |
+
#: Project JSON keys a client may echo back. They are ignored, never written.
|
| 37 |
+
READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
|
| 38 |
+
"rollup", "impact", "via"))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _call(fn, *args, **kwargs):
|
| 42 |
+
try:
|
| 43 |
+
return fn(*args, **kwargs)
|
| 44 |
+
except HTTPException:
|
| 45 |
+
raise
|
| 46 |
+
except KeyError:
|
| 47 |
+
raise err(404, "project_not_found", NOT_FOUND) from None
|
| 48 |
+
except projects.BackendUnavailable:
|
| 49 |
+
raise err(503, "projects_store_unavailable", "Projects need the workspace database.") from None
|
| 50 |
+
except ValueError as exc:
|
| 51 |
+
raise err(400, "invalid_project", str(exc)) from None
|
| 52 |
+
except Exception as exc: # noqa: BLE001
|
| 53 |
+
print(f"[amazon-projects] {getattr(fn, '__name__', fn)} failed: {type(exc).__name__}: {exc}",
|
| 54 |
+
file=sys.stderr)
|
| 55 |
+
raise err(503, "projects_unavailable", "Projects are unavailable. Try again shortly.") from None
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _reader(session):
|
| 59 |
+
session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
|
| 60 |
+
return session.runtime
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _writer(session):
|
| 64 |
+
"""Editors and Administrators of Command Center, through the Command Center edit check."""
|
| 65 |
+
if not routes_amazon_profit._may_edit(session):
|
| 66 |
+
raise err(403, "forbidden", FORBIDDEN)
|
| 67 |
+
return session.runtime
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _by(session):
|
| 71 |
+
return session.uname or "unknown"
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@contextmanager
|
| 75 |
+
def _open_mirror(runtime):
|
| 76 |
+
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
|
| 77 |
+
from harness import runtime as R
|
| 78 |
+
if not Path(runtime.datastore_path).is_file():
|
| 79 |
+
yield None
|
| 80 |
+
return
|
| 81 |
+
cur = R.mirror_cursor(runtime, require_ready=False)
|
| 82 |
+
try:
|
| 83 |
+
yield cur
|
| 84 |
+
finally:
|
| 85 |
+
cur.close()
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _mirror(runtime, read, empty):
|
| 89 |
+
"""(read(cursor), None), or (empty, the cause) when the mirror is absent or unreadable."""
|
| 90 |
+
try:
|
| 91 |
+
with _open_mirror(runtime) as cur:
|
| 92 |
+
if cur is None:
|
| 93 |
+
return empty, NO_MIRROR
|
| 94 |
+
return read(cur), None
|
| 95 |
+
except Exception as exc: # noqa: BLE001
|
| 96 |
+
print(f"[amazon-projects] mirror read failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 97 |
+
return empty, UNREADABLE
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _pictures(runtime):
|
| 101 |
+
"""`{ASIN: picture ref}`; empty when no picture is stored or the index is unreadable."""
|
| 102 |
+
try:
|
| 103 |
+
from harness import amazon_pictures
|
| 104 |
+
return amazon_pictures.picture_refs(runtime)
|
| 105 |
+
except Exception: # noqa: BLE001
|
| 106 |
+
return {}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _project(record, today):
|
| 110 |
+
identity = projects.owner_identity(record["owner"], record.get("tenant"))
|
| 111 |
+
impact = dict(record.get("impact") or {})
|
| 112 |
+
impact.pop(projects.RESULT_CONFIGS_KEY, None)
|
| 113 |
+
return {"id": record["id"], "name": record["name"], "status": record["status"],
|
| 114 |
+
"priority": record["priority"], "owner": identity["name"] if identity else record["owner"],
|
| 115 |
+
"ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"],
|
| 116 |
+
"targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
|
| 117 |
+
"completedOn": record["completed_on"], "health": projects.health(record, today),
|
| 118 |
+
"targetCount": record.get("target_count", 0), "createdAt": record["created_at"],
|
| 119 |
+
"updatedAt": record["updated_at"], "createdBy": record["created_by"],
|
| 120 |
+
"rollup": record.get("rollup"), "impact": impact or None,
|
| 121 |
+
"resultConfigs": record.get("result_configs", [])}
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _item(grain, key, listing, pictures):
|
| 125 |
+
asin, parent = listing.get("asin"), listing.get("parent_asin")
|
| 126 |
+
return {"grain": grain, "key": key, "sku": listing.get("sku"), "asin": asin, "parentAsin": parent,
|
| 127 |
+
"title": listing.get("title"), "picture": pictures.get(asin) or pictures.get(parent)}
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _pair(grain, key):
|
| 131 |
+
try:
|
| 132 |
+
return projects.target(grain, key)
|
| 133 |
+
except (TypeError, ValueError):
|
| 134 |
+
raise err(400, "invalid_target", "Name a grain of sku, asin or parent_asin and its key.") from None
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _pairs(values):
|
| 138 |
+
if not isinstance(values, list) or not values or not all(isinstance(v, dict) for v in values):
|
| 139 |
+
raise err(400, "invalid_target", "Send targets as a list of grain and key.")
|
| 140 |
+
return [_pair(v.get("grain"), v.get("key")) for v in values]
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _fields(body, *, partial):
|
| 144 |
+
if not isinstance(body, dict):
|
| 145 |
+
raise err(400, "invalid_project", "Send the project fields as an object.")
|
| 146 |
+
unknown = sorted(set(body) - set(FIELDS) - READ_ONLY - {"targets", "ownerUsername"})
|
| 147 |
+
if unknown:
|
| 148 |
+
raise err(400, "invalid_project", f"Unknown project fields: {', '.join(unknown)}.")
|
| 149 |
+
values = {FIELDS[name]: body[name] for name in FIELDS if name in body}
|
| 150 |
+
if not partial and "name" not in values:
|
| 151 |
+
raise err(400, "invalid_project", "Name the project.")
|
| 152 |
+
return values
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _owner_candidates(session):
|
| 156 |
+
return _call(users.assignable_identities, session.tenant)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _owner_fields(body, session, values, previous=None):
|
| 160 |
+
"""Resolve new assignments against the current tenant roster on every write."""
|
| 161 |
+
if "ownerUsername" not in body:
|
| 162 |
+
if values.get("owner") is not None:
|
| 163 |
+
old = (previous or {}).get("owner")
|
| 164 |
+
if (values["owner"] != old or projects.owner_identity(old) is not None):
|
| 165 |
+
raise err(400, "invalid_project", "Select a current workspace user by identity.")
|
| 166 |
+
return values
|
| 167 |
+
username = body["ownerUsername"]
|
| 168 |
+
if username is None:
|
| 169 |
+
return {**values, "owner": None}
|
| 170 |
+
if not isinstance(username, str) or not username.strip():
|
| 171 |
+
raise err(400, "invalid_project", "Select a current workspace user.")
|
| 172 |
+
username = username.strip().lower()
|
| 173 |
+
identity = next((item for item in _owner_candidates(session) if item["username"] == username), None)
|
| 174 |
+
if identity is None:
|
| 175 |
+
old = projects.owner_identity((previous or {}).get("owner"), session.tenant)
|
| 176 |
+
if old and old["username"] == username:
|
| 177 |
+
return {**values, "owner": previous["owner"]}
|
| 178 |
+
raise err(400, "invalid_project", "Select a current workspace user.")
|
| 179 |
+
return {**values, "owner": _call(projects.owner_value, session.tenant, identity)}
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _targets(runtime, project_id):
|
| 183 |
+
"""The project's targets with their listing details, and a cause when the mirror is out."""
|
| 184 |
+
listed = _call(projects.targets, runtime, project_id)
|
| 185 |
+
pairs = [(t["grain"], t["key"]) for t in listed]
|
| 186 |
+
details, cause = _mirror(runtime, lambda cur: projects.describe(cur, pairs), {})
|
| 187 |
+
pictures = _pictures(runtime) if pairs else {}
|
| 188 |
+
out = [_item(grain, key, details.get((grain, key)) or {"sku": key if grain == "sku" else None,
|
| 189 |
+
"asin": key if grain == "asin" else None,
|
| 190 |
+
"parent_asin": key if grain == "parent_asin" else None}, pictures)
|
| 191 |
+
for grain, key in pairs]
|
| 192 |
+
return out, cause
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def _with_cause(body, cause):
|
| 196 |
+
if cause:
|
| 197 |
+
body["cause"] = cause
|
| 198 |
+
return body
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
# --------------------------------------------------------------------------- Command Center cells
|
| 202 |
+
|
| 203 |
+
PROGRESS_KEY, LABEL_KEY = "projects", "projects_label"
|
| 204 |
+
#: (table, the grains one row counts, in order): a SKU row counts its ASIN's and parent's projects too.
|
| 205 |
+
CELL_TABLES = ((amazon_profit.TABLE_KEY, ("sku", "asin", "parent_asin")),
|
| 206 |
+
(amazon_profit.ASIN_KEY, ("asin", "parent_asin")),
|
| 207 |
+
(amazon_profit.PARENT_KEY, ("parent_asin",)))
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def _distinct(found, pairs):
|
| 211 |
+
"""`[(via pair, project)]` with each project once, in the order of `pairs`."""
|
| 212 |
+
seen, out = set(), []
|
| 213 |
+
for via in pairs:
|
| 214 |
+
for record in found.get(via, []):
|
| 215 |
+
if record["id"] not in seen:
|
| 216 |
+
seen.add(record["id"])
|
| 217 |
+
out.append((via, record))
|
| 218 |
+
return out
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def cell_values(records):
|
| 222 |
+
"""Store average progress with Completed at 100 and the completion count, blank when none count."""
|
| 223 |
+
counted = projects.summary(records)
|
| 224 |
+
if not counted["total"]:
|
| 225 |
+
return {PROGRESS_KEY: None, LABEL_KEY: None}
|
| 226 |
+
return {PROGRESS_KEY: amazon_profit.stored_cell({"type": "pct"}, counted["average"] / 100),
|
| 227 |
+
LABEL_KEY: f"{counted['completed']} of {counted['total']} completed"}
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _row_pairs(row, grains):
|
| 231 |
+
out = []
|
| 232 |
+
for grain in grains:
|
| 233 |
+
value = str(row.get(grain) or "").strip()
|
| 234 |
+
if value:
|
| 235 |
+
pair = projects.target(grain, value)
|
| 236 |
+
if pair not in out:
|
| 237 |
+
out.append(pair)
|
| 238 |
+
return out
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _cells_store(runtime):
|
| 242 |
+
"""The store that holds the Command Center tables: the tenant runtime itself."""
|
| 243 |
+
return runtime
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def _row_ids(cells):
|
| 247 |
+
"""The write names only the rows whose cells change."""
|
| 248 |
+
return sorted(cells)
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def _write_cells(st, table_key, cells):
|
| 252 |
+
import core.user_tables as ut
|
| 253 |
+
|
| 254 |
+
def _set(cur):
|
| 255 |
+
rows = ((cur or {}).get(table_key) or {}).get("rows") or {}
|
| 256 |
+
for rid, values in cells.items():
|
| 257 |
+
row = rows.get(rid)
|
| 258 |
+
if row is None:
|
| 259 |
+
continue
|
| 260 |
+
for key, value in values.items():
|
| 261 |
+
if value is None:
|
| 262 |
+
row.pop(key, None)
|
| 263 |
+
else:
|
| 264 |
+
row[key] = value
|
| 265 |
+
return cur
|
| 266 |
+
|
| 267 |
+
ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA,
|
| 268 |
+
row_ids=_row_ids(cells))
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def paint(runtime, pairs=None):
|
| 272 |
+
"""Rewrite the progress cells of the rows that count a touched pair; every row when `pairs` is None.
|
| 273 |
+
|
| 274 |
+
Returns `{table: [row id]}` for the rows whose cells changed. Never runs under `amazon_profit._LOCK`."""
|
| 275 |
+
touched = None if pairs is None else {projects.target(*pair) for pair in pairs}
|
| 276 |
+
if touched is not None and not touched:
|
| 277 |
+
return {}
|
| 278 |
+
st = _cells_store(runtime)
|
| 279 |
+
source = {grains[0]: amazon_profit.table_rows(st, key) for key, grains in CELL_TABLES}
|
| 280 |
+
own = {g: {rid: {g: row.get(g)} for rid, row in rows.items()} for g, rows in source.items()}
|
| 281 |
+
lineage, _cause = _mirror(runtime, lambda cur: amazon_profit.M.lineage_rows(cur, source), own)
|
| 282 |
+
plan = []
|
| 283 |
+
for table_key, grains in CELL_TABLES:
|
| 284 |
+
rows = {}
|
| 285 |
+
for rid, row in source[grains[0]].items():
|
| 286 |
+
chain = _row_pairs(lineage[grains[0]][rid], grains)
|
| 287 |
+
if touched is None or touched.intersection(chain + _row_pairs(row, grains)):
|
| 288 |
+
rows[str(rid)] = (chain, row)
|
| 289 |
+
plan.append((table_key, rows))
|
| 290 |
+
wanted = sorted({pair for _key, rows in plan for chain, _row in rows.values() for pair in chain})
|
| 291 |
+
found = projects.projects_for(runtime, wanted) if wanted else {}
|
| 292 |
+
written = {}
|
| 293 |
+
for table_key, rows in plan:
|
| 294 |
+
cells = {}
|
| 295 |
+
for rid, (chain, row) in rows.items():
|
| 296 |
+
values = cell_values([record for _via, record in _distinct(found, chain)])
|
| 297 |
+
if any(row.get(key) != value for key, value in values.items()):
|
| 298 |
+
cells[rid] = values
|
| 299 |
+
if cells:
|
| 300 |
+
_write_cells(st, table_key, cells)
|
| 301 |
+
written[table_key] = sorted(cells)
|
| 302 |
+
return written
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def _repaint(runtime, pairs):
|
| 306 |
+
"""Paint after a committed project write. A failure is logged: the write itself stands."""
|
| 307 |
+
try:
|
| 308 |
+
return paint(runtime, pairs)
|
| 309 |
+
except Exception as exc: # noqa: BLE001
|
| 310 |
+
print(f"[amazon-projects] cell repaint failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 311 |
+
return None
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def _target_pairs(runtime, project_id):
|
| 315 |
+
return [(t["grain"], t["key"]) for t in _call(projects.targets, runtime, project_id)]
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def _result_metadata(metric, currency=None):
|
| 319 |
+
return {"label": sku_action_results.LABELS[sku_action_results.METRICS.index(metric)],
|
| 320 |
+
"unit": (currency or "currency") if metric == "sales" else
|
| 321 |
+
"percent" if metric.endswith("_rate") else "ratio" if metric == "roas" else "units",
|
| 322 |
+
"basis": "Shipped units" if metric.endswith("_rate") else
|
| 323 |
+
"Net units sold" if metric == "units" else "Mirror observations"}
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def _result_sources(session, targets):
|
| 327 |
+
"""Apply source and membership grants before opening the tenant mirror."""
|
| 328 |
+
available = routes_sku_actions._results_sources(session)
|
| 329 |
+
if not any(t["grain"] != "sku" for t in targets):
|
| 330 |
+
return available
|
| 331 |
+
from core import perm_scope, user_tables
|
| 332 |
+
definitions = user_tables.lend_defs(session.runtime, schema_only=True)
|
| 333 |
+
for source in (projects.LISTINGS, projects.PRICING):
|
| 334 |
+
database = "ut_" + source
|
| 335 |
+
definition = user_tables.get_one(database, st=definitions)
|
| 336 |
+
if definition is None or not perm_scope.may_read(session.user, database, st=definitions):
|
| 337 |
+
raise err(403, "project_results_source_forbidden", f"Results require access to {database}.")
|
| 338 |
+
team, agent = perm_scope.derive_pool_scope(session.user, database)
|
| 339 |
+
if (perm_scope.wall_declared(session.user, database, st=definitions, defn=definition)
|
| 340 |
+
or team is not None or agent is not None):
|
| 341 |
+
raise err(409, "project_results_scope_not_applied",
|
| 342 |
+
f"Results cannot apply the source restriction on {database}.")
|
| 343 |
+
return available
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
class ProjectMirror(sku_action_results.Mirror):
|
| 347 |
+
"""Read a deduplicated project SKU union and retain source evidence for every target day."""
|
| 348 |
+
|
| 349 |
+
def __init__(self, con, skus, available):
|
| 350 |
+
super().__init__(con, {}, available)
|
| 351 |
+
self.skus, self.coverage = tuple(skus), []
|
| 352 |
+
|
| 353 |
+
def entity(self):
|
| 354 |
+
self.field(sku_action_results.ECONOMICS, "msku")
|
| 355 |
+
return '"msku" IN (' + ", ".join("?" for _ in self.skus) + ")", self.skus
|
| 356 |
+
|
| 357 |
+
def raw(self, table, days, values, **kwargs):
|
| 358 |
+
plan = super().raw(table, days, values, **kwargs)
|
| 359 |
+
sku = self.field(table, *kwargs["sku_fields"])
|
| 360 |
+
head, tail = plan.sql.split(" FROM ", 1)
|
| 361 |
+
coverage = sku_action_results.Plan(
|
| 362 |
+
head + ", " + sku_action_results._q(sku) + ' AS "__project_member" FROM ' + tail,
|
| 363 |
+
plan.args, plan.sources)
|
| 364 |
+
self.coverage.append((table, tuple(days), coverage))
|
| 365 |
+
return plan
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
def _result_figure(con, config, bounds, skus, available):
|
| 369 |
+
"""Reuse daily metric definitions and reconcile their union against separate source totals."""
|
| 370 |
+
if not skus:
|
| 371 |
+
return {"state": "unavailable", "reason": "Assign an Amazon target before measuring results."}
|
| 372 |
+
|
| 373 |
+
mirror = ProjectMirror(con, skus, available)
|
| 374 |
+
try:
|
| 375 |
+
mirror.select_scope(bounds)
|
| 376 |
+
except sku_action_results.SourceUnavailable as exc:
|
| 377 |
+
return {"state": "unavailable", "reason": str(exc)}
|
| 378 |
+
plans = {name: mirror.plans(bounds[name]) for name in ("current", "comparison")}
|
| 379 |
+
metric = config["metric"]
|
| 380 |
+
sources = list(dict.fromkeys(source for plan_set in plans.values()
|
| 381 |
+
for plan in sku_action_results._specs(plan_set)[metric][0]
|
| 382 |
+
if isinstance(plan, sku_action_results.Plan) for source in plan.sources))
|
| 383 |
+
figure = {"id": metric, **_result_metadata(metric, mirror.scope[1]), "source_table": sources}
|
| 384 |
+
for name in ("current", "comparison"):
|
| 385 |
+
figure[name] = sku_action_results._window(con, plans[name], bounds[name], metric,
|
| 386 |
+
comparison=name == "comparison")
|
| 387 |
+
figure.update(value=figure["current"]["value"], series=figure["current"]["series"])
|
| 388 |
+
sku_action_results.validate(con, figure, plans, bounds)
|
| 389 |
+
if any(figure[name]["value"] is not None and not figure[name].get("validated")
|
| 390 |
+
for name in ("current", "comparison")):
|
| 391 |
+
raise sku_action_results.ValidationFailed("A project result lacks source validation.")
|
| 392 |
+
complete_windows = tuple(name for name in ("current", "comparison") if figure[name]["status"] == "available")
|
| 393 |
+
if (complete_windows
|
| 394 |
+
and not _complete_result_membership(con, mirror.coverage, bounds, sources, skus, complete_windows)):
|
| 395 |
+
return {"state": "partial_data", "reason": "Some target source days are unavailable."}
|
| 396 |
+
return {**figure, "scope": {"marketplace_id": mirror.scope[0], "currency": mirror.scope[1]}}
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
def _complete_result_membership(con, coverage, bounds, sources, skus, names=("current", "comparison")):
|
| 400 |
+
"""Require source evidence for every target day so a sibling cannot hide a missing observation."""
|
| 401 |
+
expected = len(skus) * bounds["days"]
|
| 402 |
+
for name in names:
|
| 403 |
+
for source in sources:
|
| 404 |
+
plan = next((plan for table, days, plan in coverage
|
| 405 |
+
if table == source and days == tuple(bounds[name])), None)
|
| 406 |
+
if plan is None:
|
| 407 |
+
return False
|
| 408 |
+
count = con.execute('SELECT count(*) FROM (SELECT DISTINCT day, "__project_member" '
|
| 409 |
+
f'FROM ({plan.sql}) source_days) covered', plan.args).fetchone()[0]
|
| 410 |
+
if count != expected:
|
| 411 |
+
return False
|
| 412 |
+
return True
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
_SERIES_UNAVAILABLE = {
|
| 416 |
+
"margin": "Margin requires daily net profit with verified Odoo or typed COGS. "
|
| 417 |
+
"amzn_economics alone cannot reconcile that cost.",
|
| 418 |
+
"returns": "Returns require FBA and MFN customer return events. "
|
| 419 |
+
"amzn_economics has no independent returned-unit aggregate.",
|
| 420 |
+
"returns_pct": "Returns % requires returned units over shipped units. "
|
| 421 |
+
"amzn_economics cannot independently reconcile those quantities.",
|
| 422 |
+
"refunds_pct": "Refunds % requires deduplicated refund units over shipped units. "
|
| 423 |
+
"amzn_economics cannot independently reconcile those quantities.",
|
| 424 |
+
"roas": "ROAS requires attributed ad sales over ad spend. "
|
| 425 |
+
"amzn_economics has no independent attributed ad-sales aggregate.",
|
| 426 |
+
}
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def _series_figure(key, figure, bounds):
|
| 430 |
+
"""Adapt existing sales and units definitions without publishing partial source windows."""
|
| 431 |
+
if key in _SERIES_UNAVAILABLE:
|
| 432 |
+
return {"reason": _SERIES_UNAVAILABLE[key]}
|
| 433 |
+
if figure.get("reason"):
|
| 434 |
+
return {"reason": figure["reason"]}
|
| 435 |
+
current = figure.get("current")
|
| 436 |
+
if not current or current["status"] != "available":
|
| 437 |
+
return {"reason": (current or {}).get("reason") or "Some target source days are unavailable."}
|
| 438 |
+
if key != "sales_yoy":
|
| 439 |
+
return {"total": current["value"], "points": current["series"], "unit": figure["unit"],
|
| 440 |
+
"reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None}
|
| 441 |
+
prior = figure["comparison"]
|
| 442 |
+
if prior["status"] != "available" or prior["value"] is None or prior["value"] <= 0:
|
| 443 |
+
return {"reason": prior.get("reason") or "Sales YoY requires positive sales in the matching prior window."}
|
| 444 |
+
points = []
|
| 445 |
+
for now, before in zip(current["series"], prior["series"]):
|
| 446 |
+
if before["value"] is None or before["value"] <= 0:
|
| 447 |
+
return {"reason": "Sales YoY requires positive sales on every matching prior day."}
|
| 448 |
+
points.append({"date": now["date"], "value": 100 * (now["value"] - before["value"]) / before["value"]})
|
| 449 |
+
return {"total": 100 * (current["value"] - prior["value"]) / prior["value"], "points": points,
|
| 450 |
+
"reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None}
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
def _series_economics_total(con, key, bounds, skus, available):
|
| 454 |
+
"""Aggregate original economics rows without the display plan, daily rollup or display total."""
|
| 455 |
+
if key in _SERIES_UNAVAILABLE:
|
| 456 |
+
return {"value": None, "reason": _SERIES_UNAVAILABLE[key]}
|
| 457 |
+
if not skus or not bounds["days"]:
|
| 458 |
+
return {"value": None, "reason": "No completed target days are available in this range."}
|
| 459 |
+
mirror = ProjectMirror(con, skus, available)
|
| 460 |
+
try:
|
| 461 |
+
mirror.select_scope(bounds)
|
| 462 |
+
field = mirror.field(sku_action_results.ECONOMICS,
|
| 463 |
+
"sales_net_units_sold" if key == "units" else "sales_net_product_sales_amount")
|
| 464 |
+
number = f'TRY_CAST({sku_action_results._q(field)} AS DOUBLE)'
|
| 465 |
+
day = sku_action_results._day("start_date")
|
| 466 |
+
marks = ", ".join("?" for _ in skus)
|
| 467 |
+
|
| 468 |
+
def total(days):
|
| 469 |
+
row = con.execute(
|
| 470 |
+
f'SELECT SUM({number}), count(*), count({number}), count(DISTINCT (msku, {day})) '
|
| 471 |
+
f'FROM amzn_economics WHERE msku IN ({marks}) AND marketplace_id = ? '
|
| 472 |
+
f'AND sales_net_product_sales_currency_code = ? AND {day} >= ? AND {day} < ?',
|
| 473 |
+
(*skus, *mirror.scope, *days)).fetchone()
|
| 474 |
+
if (not row[1] or row[1] != row[2] or row[3] != len(skus) * bounds["days"]
|
| 475 |
+
or not project_results._finite(row[0])):
|
| 476 |
+
raise sku_action_results.SourceUnavailable("amzn_economics: some target values or days are unavailable.")
|
| 477 |
+
return float(row[0])
|
| 478 |
+
|
| 479 |
+
value = total(bounds["current"])
|
| 480 |
+
if key == "sales_yoy":
|
| 481 |
+
prior = total(bounds["comparison"])
|
| 482 |
+
if prior <= 0:
|
| 483 |
+
raise sku_action_results.SourceUnavailable("Sales YoY requires positive prior sales.")
|
| 484 |
+
value = 100 * (value - prior) / prior
|
| 485 |
+
return {"value": value, "reason": None}
|
| 486 |
+
except sku_action_results.SourceUnavailable as exc:
|
| 487 |
+
return {"value": None, "reason": str(exc)}
|
| 488 |
+
|
| 489 |
+
|
| 490 |
+
@router.get("")
|
| 491 |
+
def row_projects(grain: str | None = None, key: str | None = None,
|
| 492 |
+
session: Session = Depends(require_session)):
|
| 493 |
+
"""A row's projects with its ASIN's and Parent ASIN's, each once; every project without a row."""
|
| 494 |
+
runtime = _reader(session)
|
| 495 |
+
can_edit = bool(routes_amazon_profit._may_edit(session))
|
| 496 |
+
today = projects._today()
|
| 497 |
+
if grain is None and key is None:
|
| 498 |
+
records = _call(projects.list_projects, runtime)
|
| 499 |
+
return {"grain": None, "key": None, "canEdit": can_edit,
|
| 500 |
+
"projects": [_project(r, today) for r in records], "summary": projects.summary(records)}
|
| 501 |
+
pair = _pair(grain, key)
|
| 502 |
+
above, cause = _mirror(runtime, lambda cur: projects.lineage(cur, *pair), [])
|
| 503 |
+
pairs = [pair] + [p for p in above if p != pair]
|
| 504 |
+
listed = _distinct(_call(projects.projects_for, runtime, pairs), pairs)
|
| 505 |
+
out = [{**_project(record, today), "via": {"grain": via[0], "key": via[1]}} for via, record in listed]
|
| 506 |
+
return _with_cause({"grain": pair[0], "key": pair[1], "canEdit": can_edit, "projects": out,
|
| 507 |
+
"summary": projects.summary([record for _via, record in listed])}, cause)
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
@router.get("/search")
|
| 511 |
+
def search_targets(q: str = "", limit: int = 25, session: Session = Depends(require_session), offset: int = 0):
|
| 512 |
+
"""A SKU-ordered page from the Amazon mirror; total counts every match, including past the end."""
|
| 513 |
+
runtime = _reader(session)
|
| 514 |
+
if not 1 <= limit <= projects.SEARCH_MAX:
|
| 515 |
+
raise err(400, "invalid_limit", f"Limit must be from 1 to {projects.SEARCH_MAX}.")
|
| 516 |
+
if type(offset) is not int or offset < 0:
|
| 517 |
+
raise err(400, "invalid_offset", "Offset must be a nonnegative whole number.")
|
| 518 |
+
(listings, total), cause = _mirror(runtime, lambda cur: projects.search(cur, q, limit, offset), ([], 0))
|
| 519 |
+
pictures = _pictures(runtime) if listings else {}
|
| 520 |
+
results = [_item("sku", item["sku"], item, pictures) for item in listings]
|
| 521 |
+
return _with_cause({"results": results, "total": total}, cause)
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
@router.post("")
|
| 525 |
+
def create_project(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 526 |
+
"""Create one project, with targets when the body names them."""
|
| 527 |
+
runtime = _writer(session)
|
| 528 |
+
values = _owner_fields(body, session, _fields(body, partial=False))
|
| 529 |
+
targets = _pairs(body["targets"]) if body.get("targets") else []
|
| 530 |
+
record = _call(projects.create, runtime, targets=targets, by=_by(session), **values)
|
| 531 |
+
_repaint(runtime, targets)
|
| 532 |
+
return {"project": _project(record, projects._today())}
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
@router.get("/owners")
|
| 536 |
+
def project_owners(session: Session = Depends(require_session)):
|
| 537 |
+
"""Offer active canonical identities from the caller's own tenant."""
|
| 538 |
+
_reader(session)
|
| 539 |
+
return {"owners": _owner_candidates(session)}
|
| 540 |
+
|
| 541 |
+
|
| 542 |
+
@router.get("/{project_id}")
|
| 543 |
+
def one_project(project_id: str, session: Session = Depends(require_session)):
|
| 544 |
+
"""One project with its targets and its activity, newest first."""
|
| 545 |
+
runtime = _reader(session)
|
| 546 |
+
record = _call(projects.get, runtime, project_id)
|
| 547 |
+
targets, cause = _targets(runtime, project_id)
|
| 548 |
+
activity = _call(projects.activity, runtime, project_id)
|
| 549 |
+
return _with_cause({"canEdit": bool(routes_amazon_profit._may_edit(session)),
|
| 550 |
+
"project": _project(record, projects._today()), "targets": targets,
|
| 551 |
+
"activity": activity}, cause)
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
@router.get("/{project_id}/results")
|
| 555 |
+
def project_result_figures(project_id: str, session: Session = Depends(require_session)):
|
| 556 |
+
"""Serve current target results only after effective grants and independent total validation."""
|
| 557 |
+
runtime = _reader(session)
|
| 558 |
+
record = _call(projects.get, runtime, project_id)
|
| 559 |
+
targets = _call(projects.targets, runtime, project_id)
|
| 560 |
+
today = projects._today()
|
| 561 |
+
con = None
|
| 562 |
+
try:
|
| 563 |
+
configs = project_results.configurations(record["result_configs"])
|
| 564 |
+
if not any(project_results.windows(record, c, today)["state"] == "ready" for c in configs):
|
| 565 |
+
return project_results.results(record, targets, read=None, metadata=_result_metadata, today=today)
|
| 566 |
+
available = _result_sources(session, targets)
|
| 567 |
+
con = routes_sku_actions._results_mirror(runtime)
|
| 568 |
+
try:
|
| 569 |
+
skus = projects.resolve_target_skus(con, targets)
|
| 570 |
+
except projects.MirrorUnreadable as exc:
|
| 571 |
+
return project_results.results(record, targets,
|
| 572 |
+
read=lambda *_: {"state": "unavailable", "reason": str(exc)},
|
| 573 |
+
metadata=_result_metadata, today=today)
|
| 574 |
+
return project_results.results(record, targets,
|
| 575 |
+
read=lambda config, bounds: _result_figure(con, config, bounds, skus, available),
|
| 576 |
+
metadata=_result_metadata, today=today)
|
| 577 |
+
except HTTPException:
|
| 578 |
+
raise
|
| 579 |
+
except sku_action_results.ValidationFailed:
|
| 580 |
+
raise err(409, "project_results_not_validated",
|
| 581 |
+
"A result total does not tie to its Amazon source.") from None
|
| 582 |
+
except ValueError as exc:
|
| 583 |
+
raise err(400, "invalid_project_results", str(exc)) from None
|
| 584 |
+
except Exception as exc: # noqa: BLE001
|
| 585 |
+
print(f"[amazon-projects] results failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 586 |
+
raise err(503, "project_results_unavailable", "Project result sources are unavailable.") from None
|
| 587 |
+
finally:
|
| 588 |
+
if con is not None:
|
| 589 |
+
con.close()
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
@router.get("/{project_id}/series")
|
| 593 |
+
def project_result_series(project_id: str, start: str | None = Query(None, alias="from"),
|
| 594 |
+
end: str | None = Query(None, alias="to"), scope: str = Query("all"),
|
| 595 |
+
session: Session = Depends(require_session)):
|
| 596 |
+
"""Serve the preset daily charts within the intersection of project targets and the modal filter."""
|
| 597 |
+
runtime = _reader(session)
|
| 598 |
+
record = _call(projects.get, runtime, project_id)
|
| 599 |
+
targets = _call(projects.targets, runtime, project_id)
|
| 600 |
+
con = None
|
| 601 |
+
try:
|
| 602 |
+
bounds = project_results.series_window(record, start, end, today=projects._today())
|
| 603 |
+
selected = project_results.series_scope(scope)
|
| 604 |
+
selected_targets = ([{"grain": "parent_asin" if selected["grain"] == "parent" else selected["grain"],
|
| 605 |
+
"key": key} for key in selected["keys"]] if selected else [])
|
| 606 |
+
available = _result_sources(session, targets + selected_targets)
|
| 607 |
+
con = routes_sku_actions._results_mirror(runtime)
|
| 608 |
+
reason = None
|
| 609 |
+
try:
|
| 610 |
+
skus = projects.resolve_target_skus(con, targets)
|
| 611 |
+
if selected:
|
| 612 |
+
selected_skus = set(projects.resolve_target_skus(con, selected_targets))
|
| 613 |
+
skus = [sku for sku in skus if sku in selected_skus]
|
| 614 |
+
if not skus:
|
| 615 |
+
reason = "No project targets match the selected scope."
|
| 616 |
+
elif not bounds["days"]:
|
| 617 |
+
reason = "Today is incomplete. No completed days are available in this range."
|
| 618 |
+
except projects.MirrorUnreadable as exc:
|
| 619 |
+
skus, reason = [], str(exc)
|
| 620 |
+
figures = {}
|
| 621 |
+
|
| 622 |
+
def read(key):
|
| 623 |
+
if reason:
|
| 624 |
+
return {"reason": reason}
|
| 625 |
+
metric = "sales" if key == "sales_yoy" else key
|
| 626 |
+
if metric not in _SERIES_UNAVAILABLE and metric not in figures:
|
| 627 |
+
figures[metric] = _result_figure(con, {"metric": metric}, bounds, skus, available)
|
| 628 |
+
return _series_figure(key, figures.get(metric, {}), bounds)
|
| 629 |
+
|
| 630 |
+
return project_results.series(record, bounds=bounds, read=read,
|
| 631 |
+
independent=lambda key: _series_economics_total(con, key, bounds, skus, available))
|
| 632 |
+
except HTTPException:
|
| 633 |
+
raise
|
| 634 |
+
except (project_results.ValidationFailed, sku_action_results.ValidationFailed):
|
| 635 |
+
raise err(409, "project_results_not_validated",
|
| 636 |
+
"A series total does not tie to its independent Amazon economics aggregate.") from None
|
| 637 |
+
except ValueError as exc:
|
| 638 |
+
raise err(400, "invalid_project_results", str(exc)) from None
|
| 639 |
+
except Exception as exc: # noqa: BLE001
|
| 640 |
+
print(f"[amazon-projects] series failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
| 641 |
+
raise err(503, "project_results_unavailable", "Project result sources are unavailable.") from None
|
| 642 |
+
finally:
|
| 643 |
+
if con is not None:
|
| 644 |
+
con.close()
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
@router.patch("/{project_id}")
|
| 648 |
+
def edit_project(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 649 |
+
"""Change the named fields. Targets named here are added."""
|
| 650 |
+
runtime = _writer(session)
|
| 651 |
+
values = _fields(body, partial=True)
|
| 652 |
+
previous = _call(projects.get, runtime, project_id) if {"ownerUsername", "owner"} & set(body) else None
|
| 653 |
+
values = _owner_fields(body, session, values, previous)
|
| 654 |
+
targets = _pairs(body["targets"]) if body.get("targets") else []
|
| 655 |
+
if not values and not targets:
|
| 656 |
+
raise err(400, "invalid_project", "Change at least one project field.")
|
| 657 |
+
if values:
|
| 658 |
+
record = _call(projects.update, runtime, project_id, values, by=_by(session))
|
| 659 |
+
if targets:
|
| 660 |
+
record = _call(projects.add_targets, runtime, project_id, targets, by=_by(session))["project"]
|
| 661 |
+
# A status or progress edit repaints every assigned target, including when a project reopens.
|
| 662 |
+
if {"status", "progress"} & set(values):
|
| 663 |
+
_repaint(runtime, _target_pairs(runtime, project_id))
|
| 664 |
+
elif targets:
|
| 665 |
+
_repaint(runtime, targets)
|
| 666 |
+
return {"project": _project(record, projects._today())}
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
@router.delete("/{project_id}")
|
| 670 |
+
def delete_project(project_id: str, session: Session = Depends(require_session)):
|
| 671 |
+
"""Delete one project and its targets."""
|
| 672 |
+
runtime = _writer(session)
|
| 673 |
+
pairs = _target_pairs(runtime, project_id)
|
| 674 |
+
_call(projects.delete, runtime, project_id, by=_by(session))
|
| 675 |
+
_repaint(runtime, pairs)
|
| 676 |
+
return {"ok": True, "id": project_id}
|
| 677 |
+
|
| 678 |
+
|
| 679 |
+
def _target_answer(runtime, project_id, result):
|
| 680 |
+
targets, cause = _targets(runtime, project_id)
|
| 681 |
+
return _with_cause({"project": _project(result["project"], projects._today()), "targets": targets}, cause)
|
| 682 |
+
|
| 683 |
+
|
| 684 |
+
@router.post("/{project_id}/targets")
|
| 685 |
+
def add_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 686 |
+
"""Assign the project to more SKUs, ASINs or Parent ASINs."""
|
| 687 |
+
runtime = _writer(session)
|
| 688 |
+
pairs = _pairs((body or {}).get("targets") if isinstance(body, dict) else None)
|
| 689 |
+
result = _call(projects.add_targets, runtime, project_id, pairs, by=_by(session))
|
| 690 |
+
_repaint(runtime, pairs)
|
| 691 |
+
return _target_answer(runtime, project_id, result)
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
@router.post("/{project_id}/targets/remove")
|
| 695 |
+
def remove_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 696 |
+
"""Unassign one SKU, ASIN or Parent ASIN from the project."""
|
| 697 |
+
runtime = _writer(session)
|
| 698 |
+
if not isinstance(body, dict):
|
| 699 |
+
raise err(400, "invalid_target", "Name the grain and key to remove.")
|
| 700 |
+
pair = _pair(body.get("grain"), body.get("key"))
|
| 701 |
+
result = _call(projects.remove_targets, runtime, project_id, [pair], by=_by(session))
|
| 702 |
+
_repaint(runtime, [pair])
|
| 703 |
+
return _target_answer(runtime, project_id, result)
|
api/routes_app_dashboard.py
CHANGED
|
@@ -3,21 +3,83 @@ from __future__ import annotations
|
|
| 3 |
|
| 4 |
from datetime import date, timedelta
|
| 5 |
from importlib import import_module
|
|
|
|
| 6 |
|
| 7 |
from fastapi import APIRouter, Depends, HTTPException, Query
|
| 8 |
|
| 9 |
from deps import Session, err, require_session
|
| 10 |
-
from core.app_dashboard import checked_dashboard
|
| 11 |
from core.request_timing import raise_if_resource_error, remaining_budget
|
| 12 |
|
| 13 |
router = APIRouter(prefix="/api/v1/app-kinds", tags=["app-dashboard"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
|
| 16 |
def admit(session, kind, app_id):
|
| 17 |
"""Refuse global saved aggregates when any source has a row or field restriction."""
|
| 18 |
import app_templates
|
| 19 |
import application_registry
|
| 20 |
-
from core import perm_scope, perms, route_system, user_tables
|
| 21 |
|
| 22 |
runtime, user = session.runtime, session.user
|
| 23 |
if (not session.tenant or getattr(runtime, "key", None) != session.tenant
|
|
@@ -32,28 +94,10 @@ def admit(session, kind, app_id):
|
|
| 32 |
or app.get("retired") is True or app.get("templateKind") != kind
|
| 33 |
or not application_registry.may_read(app, session.uname, session.admin)):
|
| 34 |
raise err(404, "app_dashboard_app", "This App is unavailable or is not shared with you.")
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
raise err(503, "app_dashboard_sources", "The dashboard source descriptor is unavailable.")
|
| 40 |
-
config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
|
| 41 |
-
enabled = config.get("modules", "all")
|
| 42 |
-
for source in sources:
|
| 43 |
-
remaining_budget()
|
| 44 |
-
if ((enabled != "all" and source not in (enabled or ()))
|
| 45 |
-
or not perm_scope.may_read(user, source, st=runtime)):
|
| 46 |
-
raise err(403, "app_dashboard_source", "Your account cannot read a required dashboard source.")
|
| 47 |
-
definition = user_tables.get_definition(source, st=runtime)
|
| 48 |
-
if not isinstance(definition, dict) or not definition.get("fields"):
|
| 49 |
-
raise err(503, "app_dashboard_source", "A dashboard source is unavailable.")
|
| 50 |
-
declared = perm_scope.entry(user, source) or {}
|
| 51 |
-
if (declared.get("access") is False or declared.get("filter") or declared.get("hiddenFields")
|
| 52 |
-
or perm_scope.wall_declared(user, source, st=runtime, defn=definition)
|
| 53 |
-
or perm_scope.row_scope_applies(user, source, st=runtime, defn=definition)
|
| 54 |
-
or perm_scope.hidden_keys(user, source, definition["fields"], st=runtime)
|
| 55 |
-
or route_system.is_managed(definition) or route_system.is_managed(source)):
|
| 56 |
-
raise err(403, "app_dashboard_scope", "This dashboard needs unrestricted source access.")
|
| 57 |
return dict(template)
|
| 58 |
|
| 59 |
|
|
@@ -78,6 +122,7 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
|
|
| 78 |
raise ValueError
|
| 79 |
except ValueError:
|
| 80 |
raise err(400, "app_dashboard_period", "Choose valid dates and a supported comparison.") from None
|
|
|
|
| 81 |
try:
|
| 82 |
remaining_budget()
|
| 83 |
descriptor = admit(session, kind, app)
|
|
@@ -86,7 +131,7 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
|
|
| 86 |
options = {}
|
| 87 |
if kind == "amazon_dashboard":
|
| 88 |
options.update(compare=compare, allowed_sources=tuple(descriptor["sources"]), cost_links=None)
|
| 89 |
-
if kind in ("payouts", "reimbursements"):
|
| 90 |
options["user"] = dict(session.user)
|
| 91 |
if kind == "opportunity_finder":
|
| 92 |
options.update(opener=_opener(session.runtime, "ut_amzn_brand_analytics"),
|
|
@@ -99,4 +144,5 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
|
|
| 99 |
raise_if_resource_error(exc)
|
| 100 |
if isinstance(exc, HTTPException):
|
| 101 |
raise
|
| 102 |
-
|
|
|
|
|
|
| 3 |
|
| 4 |
from datetime import date, timedelta
|
| 5 |
from importlib import import_module
|
| 6 |
+
import logging
|
| 7 |
|
| 8 |
from fastapi import APIRouter, Depends, HTTPException, Query
|
| 9 |
|
| 10 |
from deps import Session, err, require_session
|
| 11 |
+
from core.app_dashboard import checked_dashboard, log_failure, plain_cause
|
| 12 |
from core.request_timing import raise_if_resource_error, remaining_budget
|
| 13 |
|
| 14 |
router = APIRouter(prefix="/api/v1/app-kinds", tags=["app-dashboard"])
|
| 15 |
+
_log = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _app_source_admission(user, runtime, app_id, source):
|
| 19 |
+
"""Bind core delegation to a fresh readable App and its declared sources."""
|
| 20 |
+
import application_registry
|
| 21 |
+
import app_templates
|
| 22 |
+
from core import perms
|
| 23 |
+
tenant = getattr(runtime, 'key', None)
|
| 24 |
+
if not tenant or user.get('tenant') != tenant or not user.get('username'):
|
| 25 |
+
return False
|
| 26 |
+
enabled = (getattr(getattr(runtime, 'tenant', None), 'config', None) or {}).get('modules', 'all')
|
| 27 |
+
if enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset)) or source not in enabled):
|
| 28 |
+
return False
|
| 29 |
+
app = application_registry.load(runtime, strict=True)['applications'].get(app_id)
|
| 30 |
+
if (not isinstance(app, dict) or app.get('id') != app_id or app.get('tenant') != tenant
|
| 31 |
+
or app.get('retired') is True
|
| 32 |
+
or not application_registry.may_read(app, user['username'], perms.is_admin(user))):
|
| 33 |
+
return False
|
| 34 |
+
template = app_templates.BY_KIND.get(app.get('templateKind'))
|
| 35 |
+
return bool(template and not app_templates.held(app.get('templateKind'))
|
| 36 |
+
and source in (*template['sources'], *template.get('optional_sources', ())))
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
from core.perm_scope import register_app_source_admission
|
| 40 |
+
register_app_source_admission(_app_source_admission)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def source_scope_for(session, app):
|
| 44 |
+
"""Bind a provider callback to the live App's tenant, principal and declared sources."""
|
| 45 |
+
import application_registry
|
| 46 |
+
|
| 47 |
+
def scope(source, records=None):
|
| 48 |
+
return application_registry.app_source_scope(session.user, session.runtime, source,
|
| 49 |
+
app=app, records=records)
|
| 50 |
+
|
| 51 |
+
return scope
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def admit_reader(session, kind, app=None, *, sources=None):
|
| 55 |
+
"""Admit an App holder while unsupported source narrowing remains refused."""
|
| 56 |
+
import application_registry
|
| 57 |
+
import app_templates
|
| 58 |
+
descriptor = app_templates.BY_KIND[kind]
|
| 59 |
+
if app is None:
|
| 60 |
+
app = application_registry.readable_app(session.runtime, session.tenant, session.uname,
|
| 61 |
+
session.admin, kind)
|
| 62 |
+
if not isinstance(app, dict) or app.get('templateKind') != kind:
|
| 63 |
+
raise err(404, kind + '_app_not_found', 'This App is unavailable or is not shared with you.')
|
| 64 |
+
required = tuple(descriptor['sources'] if sources is None else sources)
|
| 65 |
+
checked = tuple(dict.fromkeys((*required, *descriptor.get('optional_sources', ()))))
|
| 66 |
+
access = application_registry.admit_app(session, app, checked, code=kind)
|
| 67 |
+
from core import user_tables
|
| 68 |
+
enabled = (getattr(getattr(session.runtime, 'tenant', None), 'config', None) or {}).get('modules', 'all')
|
| 69 |
+
for source in required:
|
| 70 |
+
if not _app_source_admission(session.user, session.runtime, app.get('id'), source):
|
| 71 |
+
raise err(403, kind + '_source_unavailable', 'This App cannot read that source.')
|
| 72 |
+
if enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset)) or source not in enabled):
|
| 73 |
+
raise err(403, kind + '_source_unavailable', 'This App source is unavailable in this workspace.')
|
| 74 |
+
if not (user_tables.get_definition(source, st=session.runtime) or {}).get('fields'):
|
| 75 |
+
raise err(503, kind + '_source_contract', f'The field contract for {source} is unavailable. Sync its connector.')
|
| 76 |
+
return access
|
| 77 |
|
| 78 |
|
| 79 |
def admit(session, kind, app_id):
|
| 80 |
"""Refuse global saved aggregates when any source has a row or field restriction."""
|
| 81 |
import app_templates
|
| 82 |
import application_registry
|
|
|
|
| 83 |
|
| 84 |
runtime, user = session.runtime, session.user
|
| 85 |
if (not session.tenant or getattr(runtime, "key", None) != session.tenant
|
|
|
|
| 94 |
or app.get("retired") is True or app.get("templateKind") != kind
|
| 95 |
or not application_registry.may_read(app, session.uname, session.admin)):
|
| 96 |
raise err(404, "app_dashboard_app", "This App is unavailable or is not shared with you.")
|
| 97 |
+
# R46: the source walls live once, in the check availability also reads.
|
| 98 |
+
access = application_registry.app_access(user, runtime, app, template.get("sources"))
|
| 99 |
+
if access["dashboard"] != "ready":
|
| 100 |
+
raise err(access["status"], access["code"], access["message"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
return dict(template)
|
| 102 |
|
| 103 |
|
|
|
|
| 122 |
raise ValueError
|
| 123 |
except ValueError:
|
| 124 |
raise err(400, "app_dashboard_period", "Choose valid dates and a supported comparison.") from None
|
| 125 |
+
descriptor = {}
|
| 126 |
try:
|
| 127 |
remaining_budget()
|
| 128 |
descriptor = admit(session, kind, app)
|
|
|
|
| 131 |
options = {}
|
| 132 |
if kind == "amazon_dashboard":
|
| 133 |
options.update(compare=compare, allowed_sources=tuple(descriptor["sources"]), cost_links=None)
|
| 134 |
+
if kind in ("payouts", "reimbursements", "returns_refunds", "repeat_buyers"):
|
| 135 |
options["user"] = dict(session.user)
|
| 136 |
if kind == "opportunity_finder":
|
| 137 |
options.update(opener=_opener(session.runtime, "ut_amzn_brand_analytics"),
|
|
|
|
| 144 |
raise_if_resource_error(exc)
|
| 145 |
if isinstance(exc, HTTPException):
|
| 146 |
raise
|
| 147 |
+
log_failure(_log, exc, "read", kind, session.tenant)
|
| 148 |
+
raise err(503, "app_dashboard_unavailable", plain_cause(exc, "read", descriptor.get("sources"))) from None
|
api/routes_applications.py
CHANGED
|
@@ -311,7 +311,7 @@ def get_customer_card(app_id: str, customer_id: int,
|
|
| 311 |
if section != "customer":
|
| 312 |
_customer_card_admit(session, section, records)
|
| 313 |
|
| 314 |
-
result = customer_card.card(customer_id, cursor, rt=session.runtime, team_id=team_id,
|
| 315 |
invoice_limit=limit, invoice_offset=offset, authorize=authorize,
|
| 316 |
include_agents=agent_field in visible)
|
| 317 |
result["customer"]["name"] = selected.get(app["fieldMap"]["name"])
|
|
|
|
| 311 |
if section != "customer":
|
| 312 |
_customer_card_admit(session, section, records)
|
| 313 |
|
| 314 |
+
result = customer_card.card(customer_id, cursor, rt=session.runtime, user=session.user, team_id=team_id,
|
| 315 |
invoice_limit=limit, invoice_offset=offset, authorize=authorize,
|
| 316 |
include_agents=agent_field in visible)
|
| 317 |
result["customer"]["name"] = selected.get(app["fieldMap"]["name"])
|
api/routes_assets.py
CHANGED
|
@@ -145,6 +145,18 @@ def code_access(session, application_id):
|
|
| 145 |
return True
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
# Tokens and the embedded Inter font come from the same stylesheet as the shell.
|
| 149 |
_CODE_KIT_COMPONENTS = """
|
| 150 |
:root {
|
|
@@ -428,7 +440,8 @@ def delete_application(application_id: str, body: dict = Body(default=None),
|
|
| 428 |
session: Session = Depends(require_session)):
|
| 429 |
try:
|
| 430 |
# A database-free App still needs its own read and edit permissions.
|
| 431 |
-
if finance_access(session, application_id) or code_access(session, application_id)
|
|
|
|
| 432 |
allowed = set()
|
| 433 |
else:
|
| 434 |
current = _application_databases(session.runtime, application_id)
|
|
@@ -894,14 +907,32 @@ def amazon_picture_bytes(session, ref):
|
|
| 894 |
return bytes(blob) if blob else None
|
| 895 |
|
| 896 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 897 |
@router.get("/assets/amazon/{ref}")
|
| 898 |
def amazon_asset(ref: str, q: str = "web",
|
| 899 |
session: Session = Depends(require_session_or_bearer)):
|
| 900 |
-
"""Serve one stored Amazon listing picture for THIS session's tenant
|
| 901 |
-
|
| 902 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 903 |
if blob is None:
|
| 904 |
-
# One answer for a bad ref, a missing picture and another tenant's picture.
|
| 905 |
raise err(404, "asset_not_found", "no image for that reference")
|
| 906 |
from harness import amazon_pictures
|
| 907 |
# Only a sha-stamped ref is immutable: the sha changes when the picture changes.
|
|
|
|
| 145 |
return True
|
| 146 |
|
| 147 |
|
| 148 |
+
def alerts_access(session, application_id):
|
| 149 |
+
"""Admit a source-free Alerts App through its registry and Automation permission walls."""
|
| 150 |
+
app = applications.load(session.runtime)["applications"].get(application_id)
|
| 151 |
+
if not isinstance(app, dict) or app.get("templateKind") != "alerts" or not applications._database_free(app):
|
| 152 |
+
return False
|
| 153 |
+
if (app.get("tenant") != session.tenant or session.tenant != session.runtime.key
|
| 154 |
+
or app.get("retired") is True or not applications.may_read(app, session.uname, session.admin)
|
| 155 |
+
or not _session_wall(session)("automation")):
|
| 156 |
+
raise applications.ApplicationError("application does not exist")
|
| 157 |
+
return True
|
| 158 |
+
|
| 159 |
+
|
| 160 |
# Tokens and the embedded Inter font come from the same stylesheet as the shell.
|
| 161 |
_CODE_KIT_COMPONENTS = """
|
| 162 |
:root {
|
|
|
|
| 440 |
session: Session = Depends(require_session)):
|
| 441 |
try:
|
| 442 |
# A database-free App still needs its own read and edit permissions.
|
| 443 |
+
if (finance_access(session, application_id) or code_access(session, application_id)
|
| 444 |
+
or alerts_access(session, application_id)):
|
| 445 |
allowed = set()
|
| 446 |
else:
|
| 447 |
current = _application_databases(session.runtime, application_id)
|
|
|
|
| 907 |
return bytes(blob) if blob else None
|
| 908 |
|
| 909 |
|
| 910 |
+
def amazon_full_picture_bytes(session, ref):
|
| 911 |
+
"""`(status, jpeg)` for the full-size copy of `<ASIN>.<sha>` in THIS session's tenant store."""
|
| 912 |
+
from harness import amazon_pictures
|
| 913 |
+
parsed = amazon_pictures.parse_ref(ref)
|
| 914 |
+
if parsed is None:
|
| 915 |
+
return "missing", None
|
| 916 |
+
try:
|
| 917 |
+
return amazon_pictures.full_picture(session.runtime, parsed[0])
|
| 918 |
+
except Exception: # noqa: BLE001
|
| 919 |
+
return "failed", None
|
| 920 |
+
|
| 921 |
+
|
| 922 |
@router.get("/assets/amazon/{ref}")
|
| 923 |
def amazon_asset(ref: str, q: str = "web",
|
| 924 |
session: Session = Depends(require_session_or_bearer)):
|
| 925 |
+
"""Serve one stored Amazon listing picture for THIS session's tenant: the 320 px thumbnail,
|
| 926 |
+
or with `q=full` the large copy the server fetched once and stored. The page never loads an
|
| 927 |
+
Amazon host."""
|
| 928 |
+
if q == "full":
|
| 929 |
+
status, blob = amazon_full_picture_bytes(session, ref)
|
| 930 |
+
if status == "failed":
|
| 931 |
+
raise err(502, "picture_unavailable", "the full picture could not be loaded, try again later")
|
| 932 |
+
else:
|
| 933 |
+
blob = amazon_picture_bytes(session, ref)
|
| 934 |
if blob is None:
|
| 935 |
+
# One answer for a bad ref, a missing picture, a refused link and another tenant's picture.
|
| 936 |
raise err(404, "asset_not_found", "no image for that reference")
|
| 937 |
from harness import amazon_pictures
|
| 938 |
# Only a sha-stamped ref is immutable: the sha changes when the picture changes.
|
api/routes_auth.py
CHANGED
|
@@ -132,6 +132,7 @@ def _public_user(user):
|
|
| 132 |
except Exception:
|
| 133 |
# A missing display name must not prevent a proved account from signing in.
|
| 134 |
pass
|
|
|
|
| 135 |
return {"username": user.get("username", ""), "name": user.get("name", ""),
|
| 136 |
"role": user.get("role", "user"),
|
| 137 |
# Wave 18 (C1-TENANT, R1): the company this session is bound to — the client shows
|
|
@@ -139,7 +140,7 @@ def _public_user(user):
|
|
| 139 |
"tenant": tenant,
|
| 140 |
"workspaceName": workspace_name,
|
| 141 |
"bus": perms.allowed_bu_labels(user),
|
| 142 |
-
"team_id":
|
| 143 |
"agent": perms.scope_agent(user),
|
| 144 |
"modules": sorted(perms.allowed_modules(user) or []) or "all",
|
| 145 |
"landing": perms.landing_page(user),
|
|
@@ -311,7 +312,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 311 |
raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.")
|
| 312 |
# TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's
|
| 313 |
# ordinary code already proves the factor, and never reset the guess budget on password.
|
| 314 |
-
factor_valid = bool(remembered) or users.totp_valid(rec.get("mfa_secret"), code
|
|
|
|
| 315 |
if not factor_valid:
|
| 316 |
recovery_code_used = users.consume_recovery_code(user["username"], code)
|
| 317 |
if not (factor_valid or recovery_code_used or remembered):
|
|
@@ -321,6 +323,9 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 321 |
except audit.AuditRefused as exc:
|
| 322 |
print(f"[aios-api] {exc}")
|
| 323 |
raise err(401, "mfa_invalid", _MFA_INVALID)
|
|
|
|
|
|
|
|
|
|
| 324 |
|
| 325 |
# ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
|
| 326 |
# account already holds, so a forgotten sign-in on someone else's computer cannot lock the
|
|
|
|
| 132 |
except Exception:
|
| 133 |
# A missing display name must not prevent a proved account from signing in.
|
| 134 |
pass
|
| 135 |
+
team_id = perms.scope_team_id(user) if users.bu_labels(user) else None
|
| 136 |
return {"username": user.get("username", ""), "name": user.get("name", ""),
|
| 137 |
"role": user.get("role", "user"),
|
| 138 |
# Wave 18 (C1-TENANT, R1): the company this session is bound to — the client shows
|
|
|
|
| 140 |
"tenant": tenant,
|
| 141 |
"workspaceName": workspace_name,
|
| 142 |
"bus": perms.allowed_bu_labels(user),
|
| 143 |
+
"team_id": team_id,
|
| 144 |
"agent": perms.scope_agent(user),
|
| 145 |
"modules": sorted(perms.allowed_modules(user) or []) or "all",
|
| 146 |
"landing": perms.landing_page(user),
|
|
|
|
| 312 |
raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.")
|
| 313 |
# TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's
|
| 314 |
# ordinary code already proves the factor, and never reset the guess budget on password.
|
| 315 |
+
factor_valid = bool(remembered) or users.totp_valid(rec.get("mfa_secret"), code,
|
| 316 |
+
username=user["username"])
|
| 317 |
if not factor_valid:
|
| 318 |
recovery_code_used = users.consume_recovery_code(user["username"], code)
|
| 319 |
if not (factor_valid or recovery_code_used or remembered):
|
|
|
|
| 323 |
except audit.AuditRefused as exc:
|
| 324 |
print(f"[aios-api] {exc}")
|
| 325 |
raise err(401, "mfa_invalid", _MFA_INVALID)
|
| 326 |
+
if factor_valid and not remembered:
|
| 327 |
+
# S82-T134: a verified plaintext secret is sealed in place. Fail silent, no epoch bump.
|
| 328 |
+
users.reseal_mfa(user["username"], rec.get("mfa_secret"))
|
| 329 |
|
| 330 |
# ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
|
| 331 |
# account already holds, so a forgotten sign-in on someone else's computer cannot lock the
|
api/routes_automation.py
CHANGED
|
@@ -61,6 +61,100 @@ MODULE = "automation"
|
|
| 61 |
_GATE = module_gate(MODULE)
|
| 62 |
|
| 63 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
def _item_rights(session, item):
|
| 65 |
"""Use the item's owner for writes, with Administrator authority and system exclusions."""
|
| 66 |
if item and item.get("id") == ODOO_SYNC_ID:
|
|
@@ -2768,6 +2862,30 @@ import core.user_tables as _ut_hooks # noqa: E402
|
|
| 2768 |
if engine.grid_hook not in _ut_hooks.ROW_HOOKS:
|
| 2769 |
_ut_hooks.ROW_HOOKS.append(engine.grid_hook)
|
| 2770 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2771 |
# ⭐⭐ W31 QA — DECLARE THE MACHINE-OWNED CHILD DATABASES, for the same reason and in the same
|
| 2772 |
# place as the hook above: this module already imports both sides, so neither the engine nor
|
| 2773 |
# `platform/core` grows a dependency on the other. Idempotent by construction (a set).
|
|
|
|
| 61 |
_GATE = module_gate(MODULE)
|
| 62 |
|
| 63 |
|
| 64 |
+
def _alert_gate(session: Session = Depends(require_session)):
|
| 65 |
+
"""Bind Alerts to its authenticated workspace and current Automation grant."""
|
| 66 |
+
session.require(MODULE)
|
| 67 |
+
if session.tenant != getattr(session.runtime, 'key', None):
|
| 68 |
+
raise err(403, 'alert_workspace', 'This alert belongs to another workspace.')
|
| 69 |
+
return session
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _own_alert(session, aid):
|
| 73 |
+
definition = engine.all_definitions(session.runtime).get(aid)
|
| 74 |
+
if not definition or not definition.get('alert') or definition.get('createdBy') != session.uname:
|
| 75 |
+
raise err(404, 'unknown_alert', 'This alert is unavailable.')
|
| 76 |
+
return definition
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@router.get('/alerts/options')
|
| 80 |
+
def alert_options(session: Session = Depends(_alert_gate)):
|
| 81 |
+
from core import keychain, notify_destinations, user_tables
|
| 82 |
+
tables = []
|
| 83 |
+
definitions = dict(user_tables.all_defs(st=session.runtime, schema_only=True))
|
| 84 |
+
for key, label in (('customer_data', 'Customers'), ('product_data', 'Products')):
|
| 85 |
+
definitions.setdefault(key, {'label': label})
|
| 86 |
+
for key, definition in definitions.items():
|
| 87 |
+
try:
|
| 88 |
+
fields = engine.alert_schema(session.runtime, session.user, key)
|
| 89 |
+
if fields:
|
| 90 |
+
tables.append({'key': key, 'label': definition.get('label') or key,
|
| 91 |
+
'fields': [{'key': f['key'], 'label': f.get('label') or f['key']}
|
| 92 |
+
for f in fields if f.get('key')]})
|
| 93 |
+
except (PermissionError, ValueError):
|
| 94 |
+
continue
|
| 95 |
+
destinations = []
|
| 96 |
+
preference = notify_destinations.get(session.runtime, session.uname, 'automation_run')
|
| 97 |
+
destinations.extend({'kind': 'slack', 'id': channel, 'label': f'Slack {channel}'}
|
| 98 |
+
for channel in preference['slackChannels'])
|
| 99 |
+
for entry in keychain.list_entries(session.runtime, 'generic'):
|
| 100 |
+
reference = {'kind': 'webhook', 'id': entry['id']}
|
| 101 |
+
try:
|
| 102 |
+
engine.alert_destination(session.runtime, session.user, reference)
|
| 103 |
+
destinations.append({**reference, 'label': entry['label']})
|
| 104 |
+
except (PermissionError, ValueError):
|
| 105 |
+
continue
|
| 106 |
+
return {'databases': tables, 'destinations': destinations}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
@router.get('/alerts/rules')
|
| 110 |
+
def list_alerts(session: Session = Depends(_alert_gate)):
|
| 111 |
+
visible = []
|
| 112 |
+
for definition in engine.all_definitions(session.runtime).values():
|
| 113 |
+
if not definition.get('alert') or definition.get('createdBy') != session.uname:
|
| 114 |
+
continue
|
| 115 |
+
try:
|
| 116 |
+
engine.clean_alert(session.runtime, session.user, definition['alert'])
|
| 117 |
+
visible.append(engine.alert_wire(definition))
|
| 118 |
+
except (PermissionError, ValueError):
|
| 119 |
+
continue
|
| 120 |
+
return {'alerts': visible}
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _save_alert(session, body, aid=None):
|
| 124 |
+
try:
|
| 125 |
+
if aid:
|
| 126 |
+
_own_alert(session, aid)
|
| 127 |
+
definition = engine.save_alert(session.runtime, session.user, body, aid)
|
| 128 |
+
return {'alert': engine.alert_wire(definition)}
|
| 129 |
+
except PermissionError:
|
| 130 |
+
raise err(403, 'alert_access', 'This alert uses a database, field or destination you cannot access.')
|
| 131 |
+
except ValueError as exc:
|
| 132 |
+
raise err(400, 'invalid_alert', str(exc))
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
@router.post('/alerts/rules')
|
| 136 |
+
def create_alert(body: dict = Body(...), session: Session = Depends(_alert_gate)):
|
| 137 |
+
return _save_alert(session, body)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
@router.patch('/alerts/rules/{aid}')
|
| 141 |
+
def patch_alert(aid: str, body: dict = Body(...), session: Session = Depends(_alert_gate)):
|
| 142 |
+
prior = engine.alert_wire(_own_alert(session, aid))
|
| 143 |
+
return _save_alert(session, {**prior, **body}, aid)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
@router.delete('/alerts/rules/{aid}')
|
| 147 |
+
def remove_alert(aid: str, session: Session = Depends(_alert_gate)):
|
| 148 |
+
_own_alert(session, aid)
|
| 149 |
+
def remove(doc):
|
| 150 |
+
current = doc.get(aid)
|
| 151 |
+
if current and current.get('createdBy') == session.uname and current.get('alert'):
|
| 152 |
+
doc.pop(aid)
|
| 153 |
+
return doc
|
| 154 |
+
engine._store_update(session.runtime, remove)
|
| 155 |
+
return {'removed': True}
|
| 156 |
+
|
| 157 |
+
|
| 158 |
def _item_rights(session, item):
|
| 159 |
"""Use the item's owner for writes, with Administrator authority and system exclusions."""
|
| 160 |
if item and item.get("id") == ODOO_SYNC_ID:
|
|
|
|
| 2862 |
if engine.grid_hook not in _ut_hooks.ROW_HOOKS:
|
| 2863 |
_ut_hooks.ROW_HOOKS.append(engine.grid_hook)
|
| 2864 |
|
| 2865 |
+
|
| 2866 |
+
def _received_projection(connector):
|
| 2867 |
+
"""Reuse each connected grid's canonical identities and field projection at publication."""
|
| 2868 |
+
def resolve(rt, cursor):
|
| 2869 |
+
from core import user_tables
|
| 2870 |
+
import routes_odoo_tables
|
| 2871 |
+
result = {}
|
| 2872 |
+
prefix = 'ut_amzn_' if connector == 'amazon' else 'ut_odoo_'
|
| 2873 |
+
for key in user_tables.all_defs(st=rt, schema_only=True):
|
| 2874 |
+
if not str(key).startswith(prefix):
|
| 2875 |
+
continue
|
| 2876 |
+
spec = routes_odoo_tables._source_for(cursor, key, allow_lookups=False)
|
| 2877 |
+
if spec and not spec.get('missing'):
|
| 2878 |
+
result[key] = spec
|
| 2879 |
+
return result
|
| 2880 |
+
return resolve
|
| 2881 |
+
|
| 2882 |
+
|
| 2883 |
+
from core import received_changes as _received_changes
|
| 2884 |
+
_received_changes.register_projection('amazon', _received_projection('amazon'))
|
| 2885 |
+
_received_changes.register_projection('odoo', _received_projection('odoo'))
|
| 2886 |
+
_received_changes.register_hooks()
|
| 2887 |
+
_received_changes.on_change(engine.received_change_hook)
|
| 2888 |
+
|
| 2889 |
# ⭐⭐ W31 QA — DECLARE THE MACHINE-OWNED CHILD DATABASES, for the same reason and in the same
|
| 2890 |
# place as the hook above: this module already imports both sides, so neither the engine nor
|
| 2891 |
# `platform/core` grows a dependency on the other. Idempotent by construction (a set).
|
api/routes_competitor_watch.py
CHANGED
|
@@ -1,4 +1,6 @@
|
|
| 1 |
"""Competitor watch App routes: watched terms, term detail, settings and owned alerts."""
|
|
|
|
|
|
|
| 2 |
from fastapi import APIRouter, Body, Depends, Query
|
| 3 |
|
| 4 |
from deps import Session, err, require_session
|
|
@@ -13,11 +15,11 @@ def _module():
|
|
| 13 |
return competitor_watch
|
| 14 |
|
| 15 |
|
| 16 |
-
def _reader(session: Session = Depends(require_session)) -> Session:
|
| 17 |
-
"""
|
| 18 |
-
from
|
| 19 |
-
|
| 20 |
-
|
| 21 |
return session
|
| 22 |
|
| 23 |
|
|
@@ -33,19 +35,20 @@ def _tenant(session):
|
|
| 33 |
|
| 34 |
|
| 35 |
def _seller_alert_access(session):
|
| 36 |
-
"""Cached seller alerts
|
| 37 |
-
|
| 38 |
try:
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
|
|
|
| 42 |
except Exception: # noqa: BLE001
|
| 43 |
return False
|
| 44 |
|
| 45 |
|
| 46 |
def _call(fn, *args, **kwargs):
|
| 47 |
try:
|
| 48 |
-
return fn(*args, **kwargs)
|
| 49 |
except ValueError as exc:
|
| 50 |
raise err(400, "competitor_watch_invalid", str(exc)) from exc
|
| 51 |
except KeyError as exc:
|
|
@@ -71,6 +74,21 @@ def _call(fn, *args, **kwargs):
|
|
| 71 |
|
| 72 |
# --------------------------------------------------------------------------- S78-T17 App binding
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
TEMPLATE_KIND = "competitor_watch"
|
| 75 |
|
| 76 |
|
|
@@ -81,9 +99,12 @@ def _app(session, app_id):
|
|
| 81 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 82 |
except Exception as exc: # noqa: BLE001
|
| 83 |
raise err(503, "competitor_watch_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 84 |
-
if (not isinstance(app, dict) or app.get("
|
|
|
|
| 85 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 86 |
raise err(404, "competitor_watch_app_not_found", "That Competitor watch App does not exist.")
|
|
|
|
|
|
|
| 87 |
return app
|
| 88 |
|
| 89 |
|
|
@@ -108,9 +129,11 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 108 |
|
| 109 |
@router.get("/terms")
|
| 110 |
def list_terms(q: str = "", offset: int = 0, limit: int = 50,
|
|
|
|
| 111 |
session: Session = Depends(_app_reader)):
|
| 112 |
cw = _module()
|
| 113 |
-
return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit
|
|
|
|
| 114 |
|
| 115 |
|
| 116 |
@router.get("/terms/detail")
|
|
@@ -122,13 +145,19 @@ def term_detail(term: str = Query(..., min_length=1), session: Session = Depends
|
|
| 122 |
@router.post("/terms")
|
| 123 |
def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 124 |
cw = _module()
|
| 125 |
-
return {"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
|
| 127 |
|
| 128 |
@router.post("/terms/remove")
|
| 129 |
def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 130 |
cw = _module()
|
| 131 |
-
return {"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
|
| 132 |
|
| 133 |
|
| 134 |
@router.get("/settings")
|
|
@@ -188,6 +217,33 @@ def _not_ready_message(cw, periods):
|
|
| 188 |
return cw.NOT_READY
|
| 189 |
|
| 190 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
| 192 |
"""Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
|
| 193 |
periods = list(detail.get("periods") or ())
|
|
@@ -231,7 +287,13 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
|
| 231 |
comparison = detail.get("comparison") or {}
|
| 232 |
ready = comparison.get("ready") is True
|
| 233 |
return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
"termsReady": (page or {}).get("termsReady", True),
|
|
|
|
|
|
|
| 235 |
"week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
|
| 236 |
"previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
|
| 237 |
"comparisonReady": ready,
|
|
@@ -244,13 +306,15 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
|
| 244 |
@router.get("/screen")
|
| 245 |
def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
|
| 246 |
q: str = "", offset: int = 0, limit: int = 50,
|
|
|
|
| 247 |
session: Session = Depends(_app_reader)):
|
| 248 |
"""S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
|
| 249 |
cw = _module()
|
| 250 |
tenant = _tenant(session)
|
| 251 |
record = _app(session, app)
|
| 252 |
page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
|
| 253 |
-
limit=limit, selected=term)
|
| 254 |
terms, picked = page["terms"], page["term"]
|
|
|
|
| 255 |
detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
|
| 256 |
return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
|
|
|
|
| 1 |
"""Competitor watch App routes: watched terms, term detail, settings and owned alerts."""
|
| 2 |
+
from typing import Literal
|
| 3 |
+
|
| 4 |
from fastapi import APIRouter, Body, Depends, Query
|
| 5 |
|
| 6 |
from deps import Session, err, require_session
|
|
|
|
| 15 |
return competitor_watch
|
| 16 |
|
| 17 |
|
| 18 |
+
def _reader(session: Session = Depends(require_session), app: str | None = Query(None)) -> Session:
|
| 19 |
+
"""Bind the requested App before admitting its declared source contracts and limits."""
|
| 20 |
+
from routes_app_dashboard import admit_reader
|
| 21 |
+
record = _app(session, app) if isinstance(app, str) and app else None
|
| 22 |
+
admit_reader(session, 'competitor_watch', record)
|
| 23 |
return session
|
| 24 |
|
| 25 |
|
|
|
|
| 35 |
|
| 36 |
|
| 37 |
def _seller_alert_access(session):
|
| 38 |
+
"""Cached seller alerts use the App grant and refuse offer or ownership limits."""
|
| 39 |
+
import application_registry
|
| 40 |
try:
|
| 41 |
+
record = application_registry.readable_app(session.runtime, _tenant(session), session.uname,
|
| 42 |
+
session.admin, TEMPLATE_KIND)
|
| 43 |
+
access = application_registry.admit_app(session, record, SELLER_SOURCES, code="competitor_watch")
|
| 44 |
+
return not any(wall["rows"] or wall["hidden"] for wall in access["narrow"].values())
|
| 45 |
except Exception: # noqa: BLE001
|
| 46 |
return False
|
| 47 |
|
| 48 |
|
| 49 |
def _call(fn, *args, **kwargs):
|
| 50 |
try:
|
| 51 |
+
return _term_labels(fn(*args, **kwargs))
|
| 52 |
except ValueError as exc:
|
| 53 |
raise err(400, "competitor_watch_invalid", str(exc)) from exc
|
| 54 |
except KeyError as exc:
|
|
|
|
| 74 |
|
| 75 |
# --------------------------------------------------------------------------- S78-T17 App binding
|
| 76 |
|
| 77 |
+
def _term_labels(value):
|
| 78 |
+
"""Attach decoded labels while retaining stored term keys in every route payload."""
|
| 79 |
+
cw = _module()
|
| 80 |
+
if isinstance(value, list):
|
| 81 |
+
return [_term_labels(item) for item in value]
|
| 82 |
+
if isinstance(value, dict):
|
| 83 |
+
result = {key: _term_labels(item) for key, item in value.items()}
|
| 84 |
+
if isinstance(value.get("term"), str):
|
| 85 |
+
result["termLabel"] = cw.display_term(value["term"])
|
| 86 |
+
if isinstance(value.get("terms"), list) and all(isinstance(term, str) for term in value["terms"]):
|
| 87 |
+
result["termLabels"] = {term: cw.display_term(term) for term in value["terms"]}
|
| 88 |
+
return result
|
| 89 |
+
return value
|
| 90 |
+
|
| 91 |
+
|
| 92 |
TEMPLATE_KIND = "competitor_watch"
|
| 93 |
|
| 94 |
|
|
|
|
| 99 |
app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
|
| 100 |
except Exception as exc: # noqa: BLE001
|
| 101 |
raise err(503, "competitor_watch_unavailable", "The App could not be read. Try again shortly.") from exc
|
| 102 |
+
if (not isinstance(app, dict) or app.get("tenant") != _tenant(session)
|
| 103 |
+
or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
|
| 104 |
or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
|
| 105 |
raise err(404, "competitor_watch_app_not_found", "That Competitor watch App does not exist.")
|
| 106 |
+
from routes_app_dashboard import admit_reader
|
| 107 |
+
admit_reader(session, 'competitor_watch', app)
|
| 108 |
return app
|
| 109 |
|
| 110 |
|
|
|
|
| 129 |
|
| 130 |
@router.get("/terms")
|
| 131 |
def list_terms(q: str = "", offset: int = 0, limit: int = 50,
|
| 132 |
+
tracked: bool = False, termOrder: Literal["asc", "desc"] | None = None,
|
| 133 |
session: Session = Depends(_app_reader)):
|
| 134 |
cw = _module()
|
| 135 |
+
return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit,
|
| 136 |
+
tracked=tracked, termOrder=termOrder)
|
| 137 |
|
| 138 |
|
| 139 |
@router.get("/terms/detail")
|
|
|
|
| 145 |
@router.post("/terms")
|
| 146 |
def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 147 |
cw = _module()
|
| 148 |
+
return _term_labels({"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)})
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@router.patch("/terms/star")
|
| 152 |
+
def star_term(term: str = Body(...), starred: bool = Body(...), session: Session = Depends(_app_writer)):
|
| 153 |
+
cw = _module()
|
| 154 |
+
return _call(cw.star_term, _tenant(session), term, starred, st=session.runtime)
|
| 155 |
|
| 156 |
|
| 157 |
@router.post("/terms/remove")
|
| 158 |
def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
|
| 159 |
cw = _module()
|
| 160 |
+
return _term_labels({"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)})
|
| 161 |
|
| 162 |
|
| 163 |
@router.get("/settings")
|
|
|
|
| 217 |
return cw.NOT_READY
|
| 218 |
|
| 219 |
|
| 220 |
+
def _term_rows(cw, tenant, st, page):
|
| 221 |
+
"""Read dated-week counts for this server page without loading its ranking history."""
|
| 222 |
+
terms = page["terms"]
|
| 223 |
+
counts = {}
|
| 224 |
+
if terms:
|
| 225 |
+
try:
|
| 226 |
+
with cw._reader(tenant, st) as cur:
|
| 227 |
+
cols = cw._ba_columns(cur)
|
| 228 |
+
if {"search_term", "_period_start", "_period_end"} <= cols:
|
| 229 |
+
key = cw._term_sql("search_term")
|
| 230 |
+
rows = cur.execute(
|
| 231 |
+
f'SELECT {key}, COUNT(DISTINCT CAST("_period_start" AS DATE)) '
|
| 232 |
+
f'FROM {cw.BA_TABLE} WHERE {key} IN (' + ','.join('?' for _ in terms) + ') '
|
| 233 |
+
f'AND ({cw._weekly_period_sql()}) GROUP BY {key}', terms).fetchall()
|
| 234 |
+
counts = dict(rows)
|
| 235 |
+
except Exception as exc: # noqa: BLE001
|
| 236 |
+
cause = str(exc) if isinstance(exc, cw._ReadFailed) else type(exc).__name__
|
| 237 |
+
raise cw.SourceNotReady(cause, page=page) from exc
|
| 238 |
+
flags = page.get("termFlags") or {}
|
| 239 |
+
checked = page.get("trackingReady") is True
|
| 240 |
+
return [{"term": term, "termLabel": cw.display_term(term), "datedWeeks": counts.get(term, 0),
|
| 241 |
+
"starred": flags.get(term, {}).get("starred") is True,
|
| 242 |
+
"type": "Ours" if checked and flags.get(term, {}).get("ours") is True else
|
| 243 |
+
"Starred" if flags.get(term, {}).get("starred") is True else "Watched"}
|
| 244 |
+
for term in terms]
|
| 245 |
+
|
| 246 |
+
|
| 247 |
def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
|
| 248 |
"""Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
|
| 249 |
periods = list(detail.get("periods") or ())
|
|
|
|
| 287 |
comparison = detail.get("comparison") or {}
|
| 288 |
ready = comparison.get("ready") is True
|
| 289 |
return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
|
| 290 |
+
"termLabel": cw.display_term(term) if term else None,
|
| 291 |
+
"termLabels": {key: cw.display_term(key) for key in [*terms, *([term] if term else [])]},
|
| 292 |
+
"termFlags": (page or {}).get("termFlags", {}),
|
| 293 |
+
"termRows": (page or {}).get("termRows", []),
|
| 294 |
"termsReady": (page or {}).get("termsReady", True),
|
| 295 |
+
"trackingReady": (page or {}).get("trackingReady", False),
|
| 296 |
+
"trackingCause": (page or {}).get("trackingCause"),
|
| 297 |
"week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
|
| 298 |
"previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
|
| 299 |
"comparisonReady": ready,
|
|
|
|
| 306 |
@router.get("/screen")
|
| 307 |
def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
|
| 308 |
q: str = "", offset: int = 0, limit: int = 50,
|
| 309 |
+
tracked: bool = False, termOrder: Literal["asc", "desc"] | None = None,
|
| 310 |
session: Session = Depends(_app_reader)):
|
| 311 |
"""S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
|
| 312 |
cw = _module()
|
| 313 |
tenant = _tenant(session)
|
| 314 |
record = _app(session, app)
|
| 315 |
page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
|
| 316 |
+
limit=limit, selected=term, tracked=tracked, termOrder=termOrder)
|
| 317 |
terms, picked = page["terms"], page["term"]
|
| 318 |
+
page = {**page, "termRows": _call(_term_rows, cw, tenant, session.runtime, page)}
|
| 319 |
detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
|
| 320 |
return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
|
api/routes_competitors.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Competitors doors: list, add and remove an ASIN's rivals, with each rival's quote and the medians.
|
| 2 |
+
|
| 3 |
+
A SKU reads its ASIN's list. A Parent ASIN has no list (R74). Editors of Command Center write.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
from contextlib import contextmanager
|
| 9 |
+
import logging
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
from fastapi import APIRouter, Body, Depends
|
| 14 |
+
|
| 15 |
+
from core import amazon_rivals as rivals
|
| 16 |
+
from core import amazon_projects as projects
|
| 17 |
+
from deps import Session, err, require_session
|
| 18 |
+
import routes_amazon_profit
|
| 19 |
+
|
| 20 |
+
router = APIRouter(prefix="/api/v1")
|
| 21 |
+
_log = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
GRAINS = ("sku", "asin", "parent_asin")
|
| 24 |
+
FORBIDDEN = "Only an editor of Command Center can change competitors."
|
| 25 |
+
NO_MIRROR = "This workspace has no Amazon data yet."
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@contextmanager
|
| 29 |
+
def _open_mirror(runtime):
|
| 30 |
+
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
|
| 31 |
+
from harness import runtime as R
|
| 32 |
+
if not Path(runtime.datastore_path).is_file():
|
| 33 |
+
yield None
|
| 34 |
+
return
|
| 35 |
+
cur = R.mirror_cursor(runtime, require_ready=False)
|
| 36 |
+
try:
|
| 37 |
+
yield cur
|
| 38 |
+
finally:
|
| 39 |
+
cur.close()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _rt(session):
|
| 43 |
+
session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
|
| 44 |
+
try:
|
| 45 |
+
rivals._backend(session.runtime)
|
| 46 |
+
except rivals.BackendUnavailable:
|
| 47 |
+
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 48 |
+
return session.runtime
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _grain(grain, key):
|
| 52 |
+
if grain not in GRAINS:
|
| 53 |
+
raise err(400, "bad_grain", "Grain must be sku, asin or parent_asin.")
|
| 54 |
+
if not isinstance(key, str) or not key.strip():
|
| 55 |
+
raise err(400, "bad_key", "Name the SKU, ASIN or Parent ASIN.")
|
| 56 |
+
return key.strip()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _target(cur, grain, key):
|
| 60 |
+
"""(asin, parent answer or None) for a grain and key."""
|
| 61 |
+
if grain == "sku":
|
| 62 |
+
try:
|
| 63 |
+
chain = projects.lineage(cur, "sku", key) if cur is not None else []
|
| 64 |
+
except projects.MirrorUnreadable as exc:
|
| 65 |
+
raise rivals.MirrorUnreadable(str(exc)) from exc
|
| 66 |
+
found = next((value for grain, value in chain if grain == "asin"), None)
|
| 67 |
+
if found is None:
|
| 68 |
+
raise err(404, "sku_not_listed", "This SKU has no ASIN in your Amazon listings.")
|
| 69 |
+
return found, None
|
| 70 |
+
try:
|
| 71 |
+
target = rivals.asin(key)
|
| 72 |
+
except ValueError as exc:
|
| 73 |
+
raise err(400, "bad_asin", str(exc)) from None
|
| 74 |
+
try:
|
| 75 |
+
kids = routes_amazon_profit.amazon_profit.M.parent_children(cur, target) if cur is not None else []
|
| 76 |
+
except projects.MirrorUnreadable as exc:
|
| 77 |
+
raise rivals.MirrorUnreadable(str(exc)) from exc
|
| 78 |
+
if grain == "parent_asin" or (kids and not rivals.our_asins(cur, [target])):
|
| 79 |
+
return target, {"children": kids}
|
| 80 |
+
return target, None
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _payload(runtime, grain, key, target, parent, can_edit, discovery_error=None):
|
| 84 |
+
if parent is not None:
|
| 85 |
+
# R74: the rollup and each child's standing, never a parent rival list or a parent status.
|
| 86 |
+
kids = parent["children"] or [target]
|
| 87 |
+
found = rivals.standing(runtime, kids)
|
| 88 |
+
return {"grain": grain, "key": key, "asin": target, "parent": True, "rivals": [],
|
| 89 |
+
"children": parent["children"], "cause": rivals.PARENT_ROLLS_UP, "canEdit": False,
|
| 90 |
+
"rollup": rivals.rollup([found.get(a) for a in kids]),
|
| 91 |
+
"childStanding": [found.get(a) or {"asin": a, "competitors_cause": rivals.NO_QUOTE} for a in kids]}
|
| 92 |
+
listed = rivals.rivals(runtime, target)
|
| 93 |
+
quotes = rivals.newest_quotes(runtime, [r["rival_asin"] for r in listed["rivals"]])
|
| 94 |
+
mine = rivals.standing(runtime, [target]).get(target)
|
| 95 |
+
group = (mine or {}).get("rank_group")
|
| 96 |
+
return {"grain": grain, "key": key, "asin": target, "parent": False,
|
| 97 |
+
"rivals": [{"asin": r["rival_asin"], "title": r["title"], "source": r["source"],
|
| 98 |
+
"clickShare": r["click_share"], "term": r["term"], "week": r["week"],
|
| 99 |
+
"addedBy": r["added_by"], "quote": quotes.get(r["rival_asin"]),
|
| 100 |
+
"sameGroup": bool(group) and (quotes.get(r["rival_asin"]) or {}).get("display_group") == group}
|
| 101 |
+
for r in listed["rivals"]],
|
| 102 |
+
"standing": mine,
|
| 103 |
+
"autoCount": sum(1 for r in listed["rivals"] if r["source"] == "auto"),
|
| 104 |
+
"removed": listed["removed"], "week": listed["week"], "discoveredAt": listed["discoveredAt"],
|
| 105 |
+
"cause": listed["cause"], "discoveryError": discovery_error, "canEdit": can_edit}
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _rival_set(runtime, target):
|
| 109 |
+
return {r["rival_asin"] for r in rivals.rivals(runtime, target)["rivals"]}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _notify_rival_change(runtime, target, before):
|
| 113 |
+
"""Repaint affected cells from stored quotes after a committed rival membership change."""
|
| 114 |
+
try:
|
| 115 |
+
if _rival_set(runtime, target) != before:
|
| 116 |
+
from amazon_quotes_loop import _after_quotes
|
| 117 |
+
_after_quotes(runtime.key, (target,))
|
| 118 |
+
except Exception as exc: # noqa: BLE001
|
| 119 |
+
_log.warning("tenant=%s rival-change notification failed (%s); the rival change remains saved.",
|
| 120 |
+
runtime.key, type(exc).__name__)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@router.get("/competitors")
|
| 124 |
+
def competitors_list(grain: str, key: str, session: Session = Depends(require_session)):
|
| 125 |
+
"""One SKU's, ASIN's or Parent ASIN's rivals, rediscovered when the newest week moved on."""
|
| 126 |
+
runtime = _rt(session)
|
| 127 |
+
key = _grain(grain, key)
|
| 128 |
+
problem = None
|
| 129 |
+
before = None
|
| 130 |
+
try:
|
| 131 |
+
with _open_mirror(runtime) as cur:
|
| 132 |
+
target, parent = _target(cur, grain, key)
|
| 133 |
+
if cur is None:
|
| 134 |
+
problem = NO_MIRROR
|
| 135 |
+
elif parent is None and rivals.stale(runtime, cur, target):
|
| 136 |
+
previous = _rival_set(runtime, target)
|
| 137 |
+
rivals.discover(runtime, cur, [target])
|
| 138 |
+
before = previous
|
| 139 |
+
except rivals.MirrorUnreadable as exc:
|
| 140 |
+
if grain == "sku":
|
| 141 |
+
raise err(503, "amazon_mirror_unreadable",
|
| 142 |
+
f"The Amazon data could not be read ({exc}). Try again shortly.") from None
|
| 143 |
+
target, parent = rivals.asin(key), (None if grain == "asin" else {"children": []})
|
| 144 |
+
problem = f"The Amazon data could not be read ({exc}), so the stored rivals show."
|
| 145 |
+
except rivals.BackendUnavailable:
|
| 146 |
+
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 147 |
+
except RuntimeError:
|
| 148 |
+
raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
|
| 149 |
+
if before is not None:
|
| 150 |
+
_notify_rival_change(runtime, target, before)
|
| 151 |
+
return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _writable(session, body):
|
| 155 |
+
runtime = _rt(session)
|
| 156 |
+
if not routes_amazon_profit._may_edit(session):
|
| 157 |
+
raise err(403, "forbidden", FORBIDDEN)
|
| 158 |
+
body = body if isinstance(body, dict) else {}
|
| 159 |
+
grain = body.get("grain")
|
| 160 |
+
key = _grain(grain, body.get("key"))
|
| 161 |
+
try:
|
| 162 |
+
with _open_mirror(runtime) as cur:
|
| 163 |
+
target, parent = _target(cur, grain, key)
|
| 164 |
+
except rivals.MirrorUnreadable as exc:
|
| 165 |
+
raise err(503, "amazon_mirror_unreadable",
|
| 166 |
+
f"The Amazon data could not be read ({exc}). Try again shortly.") from None
|
| 167 |
+
except rivals.BackendUnavailable:
|
| 168 |
+
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 169 |
+
except RuntimeError:
|
| 170 |
+
raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
|
| 171 |
+
if parent is not None:
|
| 172 |
+
raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
|
| 173 |
+
try:
|
| 174 |
+
rival = rivals.asin(body.get("rival"), "rival")
|
| 175 |
+
except ValueError as exc:
|
| 176 |
+
raise err(400, "bad_asin", str(exc)) from None
|
| 177 |
+
return runtime, grain, key, target, rival
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
@router.post("/competitors/rivals")
|
| 181 |
+
def competitors_add(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 182 |
+
"""Add a rival ASIN by hand. It sits above the auto-found rivals."""
|
| 183 |
+
runtime, grain, key, target, rival = _writable(session, body)
|
| 184 |
+
before = _rival_set(runtime, target)
|
| 185 |
+
try:
|
| 186 |
+
rivals.add(runtime, target, rival, user=session.uname)
|
| 187 |
+
except ValueError as exc:
|
| 188 |
+
raise err(400, "bad_rival", str(exc)) from None
|
| 189 |
+
_notify_rival_change(runtime, target, before)
|
| 190 |
+
return _payload(runtime, grain, key, target, None, True)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
@router.post("/competitors/rivals/remove")
|
| 194 |
+
def competitors_remove(body: Any = Body(default=None), session: Session = Depends(require_session)):
|
| 195 |
+
"""Remove a wrong rival. It stays removed after the next discovery."""
|
| 196 |
+
runtime, grain, key, target, rival = _writable(session, body)
|
| 197 |
+
before = _rival_set(runtime, target)
|
| 198 |
+
try:
|
| 199 |
+
rivals.remove(runtime, target, rival, user=session.uname)
|
| 200 |
+
except KeyError:
|
| 201 |
+
raise err(404, "rival_not_found", "This ASIN is not a rival of that product.") from None
|
| 202 |
+
_notify_rival_change(runtime, target, before)
|
| 203 |
+
return _payload(runtime, grain, key, target, None, True)
|
api/routes_connectors.py
CHANGED
|
@@ -65,7 +65,7 @@ from core import schema_overlay
|
|
| 65 |
#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
|
| 66 |
#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
|
| 67 |
#: second correct one here would leave the same two-copies shape that produced the defect.
|
| 68 |
-
from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available
|
| 69 |
#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a
|
| 70 |
#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any
|
| 71 |
#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router`
|
|
@@ -1095,15 +1095,25 @@ def _contact_rows(rt, uname, is_admin, incomplete):
|
|
| 1095 |
def read(source, keys, fn):
|
| 1096 |
try:
|
| 1097 |
fn()
|
| 1098 |
-
except Exception:
|
|
|
|
|
|
|
| 1099 |
for key in keys:
|
| 1100 |
by_key[key]["state"] = "unavailable"
|
| 1101 |
-
|
|
|
|
|
|
|
| 1102 |
"recommendation": "Try again."})
|
| 1103 |
|
| 1104 |
def whatsapp():
|
| 1105 |
-
import
|
| 1106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1107 |
by_key["whatsapp"]["state"] = "connected"
|
| 1108 |
|
| 1109 |
def slack():
|
|
@@ -2069,8 +2079,13 @@ def odoo_health_receipt(rt, connection_id=None, *, now=None, jobs=None):
|
|
| 2069 |
# that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
|
| 2070 |
cadence = stored.get("cadenceSeconds")
|
| 2071 |
try:
|
| 2072 |
-
|
| 2073 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2074 |
resident_cid = str(resident.get("connectionId") or "")
|
| 2075 |
if resident_cid == cid:
|
| 2076 |
cadence = resident.get("resolvedSeconds") or cadence
|
|
@@ -2150,21 +2165,66 @@ def _connection_jobs(rt, provider, connection_id):
|
|
| 2150 |
return connector_sync.jobs(rt, provider, cid)
|
| 2151 |
|
| 2152 |
|
| 2153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2154 |
"""One wire shape for every registered connector, including one not yet run."""
|
| 2155 |
meta = connector_sync.providers()[connector_key]
|
| 2156 |
slot = str(connection_id or "").strip() or "default"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2157 |
if row is None:
|
| 2158 |
-
|
| 2159 |
"connectionId": slot, "recorded": False, "state": None,
|
| 2160 |
"at": None, "stateAt": None, "lastSyncAt": None, "from": None,
|
| 2161 |
"detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
|
| 2162 |
-
"databases": [], "retryable": False, "retryHint": meta["retryHint"]}
|
| 2163 |
answer = dict(row)
|
|
|
|
| 2164 |
answer.update({"provider": connector_key, "connectionId": slot,
|
| 2165 |
-
"label": meta["label"], "recorded":
|
| 2166 |
"stateAt": row.get("at"), "lastSyncAt": None,
|
| 2167 |
"retryHint": meta["retryHint"]})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2168 |
return _safe_connector_status(connector_key, answer)
|
| 2169 |
|
| 2170 |
|
|
@@ -2710,7 +2770,8 @@ def _start_database_jobs():
|
|
| 2710 |
_materialization_wake.clear()
|
| 2711 |
try:
|
| 2712 |
for tenant, rt in tenant_scan.live_tenants(
|
| 2713 |
-
runtime,
|
|
|
|
| 2714 |
"connector database runtime failed: tenant=%s type=%s",
|
| 2715 |
tenant, type(exc).__name__)):
|
| 2716 |
if _materialization_stop.is_set():
|
|
@@ -2878,7 +2939,7 @@ def _odoo_status_reads(rt):
|
|
| 2878 |
import odoo_relational
|
| 2879 |
return _ConnectorStatusReads(rt, (
|
| 2880 |
keychain.KEY, keychain.SCOPE_KEY, odoo_relational.CONFIG_KEY,
|
| 2881 |
-
odoo_relational.MODEL_CATALOG_KEY, connector_sync.JOBS_KEY, connector_sync.STATE_KEY,
|
| 2882 |
_ODOO_HEALTH_STATUS, _DATABASE_DISPATCH_STATUS, _ODOO_SYNC_STATUS))
|
| 2883 |
|
| 2884 |
|
|
@@ -2958,7 +3019,22 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
|
|
| 2958 |
"receipt": receipt, "lastSyncAt": last_sync_at}
|
| 2959 |
|
| 2960 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2961 |
def _amazon_completed_sync(rt):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2962 |
"""Project completion markers without counting business rows or building database detail."""
|
| 2963 |
import duckdb
|
| 2964 |
from harness import amazon_store as az, datastore
|
|
@@ -2990,10 +3066,102 @@ def _amazon_completed_sync(rt):
|
|
| 2990 |
return None, False
|
| 2991 |
|
| 2992 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2993 |
def _daily_completed_sync(rt, connector_key):
|
| 2994 |
-
"""Read
|
| 2995 |
if connector_key == "amazon_sp":
|
| 2996 |
return _amazon_completed_sync(rt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2997 |
if connector_key == "meta_ads":
|
| 2998 |
from pathlib import Path
|
| 2999 |
from harness import datastore
|
|
@@ -3038,7 +3206,10 @@ def _amazon_restore_pending(rt):
|
|
| 3038 |
|
| 3039 |
|
| 3040 |
def _amazon_saved_completion(rt):
|
| 3041 |
-
"""The saved mirror manifest's completed_at
|
|
|
|
|
|
|
|
|
|
| 3042 |
key = str(getattr(rt, "key", "") or "")
|
| 3043 |
now = time.monotonic()
|
| 3044 |
cached = _SAVED_COMPLETION_CACHE.get(key)
|
|
@@ -3047,7 +3218,7 @@ def _amazon_saved_completion(rt):
|
|
| 3047 |
stamp = None
|
| 3048 |
try:
|
| 3049 |
from harness import amazon_store as az
|
| 3050 |
-
# Read the manifest object only: never
|
| 3051 |
download = getattr(rt, "download_bytes", None)
|
| 3052 |
raw = download(az.MIRROR_PREFIX + "manifest.json") if callable(download) else None
|
| 3053 |
manifest = json.loads(raw) if raw else None
|
|
@@ -3075,20 +3246,27 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 3075 |
if connector_key not in registered:
|
| 3076 |
raise err(404, "sync_provider_unknown",
|
| 3077 |
f"{connector_key} is not registered for connector sync")
|
|
|
|
|
|
|
| 3078 |
rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
|
| 3079 |
-
else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY
|
|
|
|
| 3080 |
if connector_key == "amazon_sp" else ()))
|
| 3081 |
resolved_connection = (_odoo_connection_slot(rt, connection_id)
|
| 3082 |
if connector_key == "odoo" else connection_id)
|
| 3083 |
jobs = _connection_jobs(rt, connector_key, resolved_connection)
|
| 3084 |
row = (connector_sync.job_summary(jobs)
|
| 3085 |
or connector_sync.read(rt, connector_key, resolved_connection))
|
| 3086 |
-
|
|
|
|
| 3087 |
if connector_key == "odoo":
|
| 3088 |
receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
|
| 3089 |
-
|
|
|
|
| 3090 |
if connector_key in ("amazon_sp", "meta_ads"):
|
| 3091 |
-
if
|
|
|
|
|
|
|
| 3092 |
completed, known = None, False
|
| 3093 |
else:
|
| 3094 |
completed, known = _daily_completed_sync(rt, connector_key)
|
|
@@ -3096,9 +3274,15 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 3096 |
if connector_key == "amazon_sp" and completed is None and known and _amazon_restore_pending(rt):
|
| 3097 |
completed = _amazon_saved_completion(rt)
|
| 3098 |
answer["restoring"] = True
|
| 3099 |
-
answer.update(
|
| 3100 |
-
lastCompletedKnown=known, canSync=bool(session.admin)
|
| 3101 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3102 |
return _safe_connector_status(connector_key, answer)
|
| 3103 |
|
| 3104 |
|
|
@@ -4019,7 +4203,9 @@ def _amazon_database_sync_health(raw, enabled=True, *, now=None, database_key=""
|
|
| 4019 |
refused = (raw.get("refused") is True or is_refusal_reason(raw.get("unknownCause")) or any(item.get("table") == table for item in
|
| 4020 |
((raw.get("firstLoad") or {}).get("refused") or ()) if isinstance(item, dict))
|
| 4021 |
)
|
| 4022 |
-
if
|
|
|
|
|
|
|
| 4023 |
state = "partial" if success else "failed"
|
| 4024 |
error = {"code": "amazon_report_refused"}
|
| 4025 |
elif raw.get("stuck") is True:
|
|
@@ -4034,9 +4220,11 @@ def _amazon_database_sync_health(raw, enabled=True, *, now=None, database_key=""
|
|
| 4034 |
state, error = "queued", None
|
| 4035 |
else:
|
| 4036 |
state, error = "never_run", None
|
| 4037 |
-
|
| 4038 |
state=state, last_success=success, materialized_timestamp=success,
|
| 4039 |
row_count=raw.get("rowsLoaded"), error=error, now=now)
|
|
|
|
|
|
|
| 4040 |
|
| 4041 |
|
| 4042 |
def _stored_database_sync_health(raw, enabled=True, *, now=None):
|
|
@@ -4834,6 +5022,12 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 4834 |
raise_if_resource_error(exc)
|
| 4835 |
states = {}
|
| 4836 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4837 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 4838 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
| 4839 |
by_report = {}
|
|
@@ -4872,6 +5066,13 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 4872 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 4873 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 4874 |
raw_sync = (states or {}).get(name) if is_admin else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4875 |
if isinstance(raw_sync, dict):
|
| 4876 |
from harness.amazon_store import plain_refusal_reason, is_refusal_reason
|
| 4877 |
raw_sync = dict(raw_sync)
|
|
@@ -5085,9 +5286,11 @@ def _databases_sync_health(databases):
|
|
| 5085 |
"succeeded", "disabled") if state in states), "never_run")
|
| 5086 |
successes = [_sync_stamp(row.get("lastSuccess")) for row in health]
|
| 5087 |
failure = next((row.get("error") for row in health if row.get("error")), None)
|
| 5088 |
-
|
| 5089 |
last_success=max((stamp for stamp in successes if stamp), default=None),
|
| 5090 |
error=failure)
|
|
|
|
|
|
|
| 5091 |
|
| 5092 |
|
| 5093 |
def _detail_overview(row, databases, graph, *, sync_health=None):
|
|
@@ -5650,16 +5853,12 @@ class _ReplacementConflict(Exception):
|
|
| 5650 |
"""The addressed keychain row changed between replacement preparation and its store update."""
|
| 5651 |
|
| 5652 |
|
| 5653 |
-
def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
|
| 5654 |
-
"""Validate and
|
| 5655 |
-
|
| 5656 |
-
``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny
|
| 5657 |
-
memory-only runtime lets this route reuse it while discarding the temporary identity it mints;
|
| 5658 |
-
the tenant store sees only the later, single-row payload swap.
|
| 5659 |
-
"""
|
| 5660 |
class _Stage:
|
| 5661 |
def __init__(self):
|
| 5662 |
self.bucket = {}
|
|
|
|
| 5663 |
|
| 5664 |
def get(self, name):
|
| 5665 |
return self.bucket if name == kc.KEY else {}
|
|
@@ -5676,8 +5875,9 @@ def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
|
|
| 5676 |
row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
|
| 5677 |
if not row.get("enc"):
|
| 5678 |
raise RuntimeError("the keychain encryptor returned no encrypted payload")
|
| 5679 |
-
|
| 5680 |
-
|
|
|
|
| 5681 |
if entry_type == "amazon_sp":
|
| 5682 |
payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
|
| 5683 |
return payload
|
|
@@ -5768,7 +5968,7 @@ def _replace_connector_key(connector_key, entry_id, body, session):
|
|
| 5768 |
raise err(400, "bad_replacement", refusal)
|
| 5769 |
try:
|
| 5770 |
encrypted = _encrypted_replacement_payload(
|
| 5771 |
-
kc, entry.get("label"), entry_type, merged_fields, session.uname)
|
| 5772 |
except kc.KeychainLocked as exc:
|
| 5773 |
raise err(503, "keychain_locked",
|
| 5774 |
f"the keychain is locked: {exc}. Nothing was changed.")
|
|
|
|
| 65 |
#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
|
| 66 |
#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
|
| 67 |
#: second correct one here would leave the same two-copies shape that produced the defect.
|
| 68 |
+
from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available, stored_business_tz
|
| 69 |
#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a
|
| 70 |
#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any
|
| 71 |
#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router`
|
|
|
|
| 1095 |
def read(source, keys, fn):
|
| 1096 |
try:
|
| 1097 |
fn()
|
| 1098 |
+
except Exception as exc: # noqa: BLE001
|
| 1099 |
+
logging.getLogger(__name__).warning("contact_check=%s exception=%s",
|
| 1100 |
+
source, type(exc).__name__)
|
| 1101 |
for key in keys:
|
| 1102 |
by_key[key]["state"] = "unavailable"
|
| 1103 |
+
label = {"whatsapp": "WhatsApp", "slack": "Slack", "mcp": "MCP",
|
| 1104 |
+
"loopable-api": "Loopable API"}[source]
|
| 1105 |
+
incomplete.append({"source": source, "cause": f"{label} connection status unavailable.",
|
| 1106 |
"recommendation": "Try again."})
|
| 1107 |
|
| 1108 |
def whatsapp():
|
| 1109 |
+
from core import account_rows, users
|
| 1110 |
+
from whatsapp_identity import WhatsAppRefusal
|
| 1111 |
+
# Directory status reads the reader's canonical account, never registration challenges.
|
| 1112 |
+
record = account_rows.get(uname, st=users.store, fresh=True)
|
| 1113 |
+
if not isinstance(record, dict) or str(record.get("tenant") or "royal-imports") != rt.key:
|
| 1114 |
+
raise WhatsAppRefusal("This account is not available in this workspace.")
|
| 1115 |
+
connection = record.get("whatsapp")
|
| 1116 |
+
if isinstance(connection, dict) and connection.get("phone"):
|
| 1117 |
by_key["whatsapp"]["state"] = "connected"
|
| 1118 |
|
| 1119 |
def slack():
|
|
|
|
| 2079 |
# that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
|
| 2080 |
cadence = stored.get("cadenceSeconds")
|
| 2081 |
try:
|
| 2082 |
+
canonical = connector_sync.status(rt, "odoo", cid, fresh=True)
|
| 2083 |
+
if canonical and isinstance(canonical.get("resync"), dict):
|
| 2084 |
+
resident = dict(canonical["resync"])
|
| 2085 |
+
resident["connectionId"] = cid
|
| 2086 |
+
else:
|
| 2087 |
+
_main = running_main() # D-781: resolve, never import
|
| 2088 |
+
resident = dict(_main.RESYNC_STATE)
|
| 2089 |
resident_cid = str(resident.get("connectionId") or "")
|
| 2090 |
if resident_cid == cid:
|
| 2091 |
cadence = resident.get("resolvedSeconds") or cadence
|
|
|
|
| 2165 |
return connector_sync.jobs(rt, provider, cid)
|
| 2166 |
|
| 2167 |
|
| 2168 |
+
_PASS_STATES = {"running": "syncing", "continuing": "syncing", "done": "ready", "waiting": "ready",
|
| 2169 |
+
"held": "partial", "failed": "error"}
|
| 2170 |
+
|
| 2171 |
+
|
| 2172 |
+
def _stored_sync_row(rt, connector_key, slot):
|
| 2173 |
+
"""C-status: a recorded lifecycle row from the store's last pass and last good sync, or None."""
|
| 2174 |
+
entry = connector_sync.status(rt, connector_key, slot, fresh=True) if rt is not None else None
|
| 2175 |
+
if not entry:
|
| 2176 |
+
return None
|
| 2177 |
+
last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
|
| 2178 |
+
good = _sync_stamp(entry.get("lastGoodSyncAt"))
|
| 2179 |
+
state = _PASS_STATES.get(last_pass.get("outcome")) or ("ready" if good else None)
|
| 2180 |
+
at = _sync_stamp(last_pass.get("endedAt") or last_pass.get("startedAt")) or good
|
| 2181 |
+
detail = str(last_pass.get("detail") or "") or ("Sync finished." if state == "ready" else "")
|
| 2182 |
+
return {"state": state, "at": at, "from": None, "detail": detail,
|
| 2183 |
+
"errors": [detail] if state == "error" and detail else [],
|
| 2184 |
+
"errorsTotal": 1 if state == "error" and detail else 0, "databases": [],
|
| 2185 |
+
"retryable": state in ("error", "partial"), "storedStatus": entry}
|
| 2186 |
+
|
| 2187 |
+
|
| 2188 |
+
def _sync_payload(connector_key, connection_id, row=None, *, rt=None):
|
| 2189 |
"""One wire shape for every registered connector, including one not yet run."""
|
| 2190 |
meta = connector_sync.providers()[connector_key]
|
| 2191 |
slot = str(connection_id or "").strip() or "default"
|
| 2192 |
+
stored_row = _stored_sync_row(rt, connector_key, slot)
|
| 2193 |
+
entry = (stored_row or {}).get("storedStatus") or {}
|
| 2194 |
+
if row is None:
|
| 2195 |
+
row = stored_row
|
| 2196 |
+
recorded = row is not None
|
| 2197 |
if row is None:
|
| 2198 |
+
row = {"provider": connector_key, "label": meta["label"],
|
| 2199 |
"connectionId": slot, "recorded": False, "state": None,
|
| 2200 |
"at": None, "stateAt": None, "lastSyncAt": None, "from": None,
|
| 2201 |
"detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
|
| 2202 |
+
"databases": [], "retryable": False, "retryHint": meta["retryHint"]}
|
| 2203 |
answer = dict(row)
|
| 2204 |
+
answer.pop("storedStatus", None)
|
| 2205 |
answer.update({"provider": connector_key, "connectionId": slot,
|
| 2206 |
+
"label": meta["label"], "recorded": recorded,
|
| 2207 |
"stateAt": row.get("at"), "lastSyncAt": None,
|
| 2208 |
"retryHint": meta["retryHint"]})
|
| 2209 |
+
if entry:
|
| 2210 |
+
last_pass = entry.get("pass") or {}
|
| 2211 |
+
duration, budget = last_pass.get("durationSeconds"), last_pass.get("budgetSeconds")
|
| 2212 |
+
import math
|
| 2213 |
+
def minutes(value, *, positive=False):
|
| 2214 |
+
return (round(value / 60, 3) if type(value) in (int, float) and math.isfinite(value)
|
| 2215 |
+
and (value > 0 if positive else value >= 0) else None)
|
| 2216 |
+
days = entry.get("dailyPasses") or {}
|
| 2217 |
+
answer.update(lastGoodSyncAt=entry.get("lastGoodSyncAt"), lastSyncAt=entry.get("lastGoodSyncAt"),
|
| 2218 |
+
lastCompletedKnown=True,
|
| 2219 |
+
**{"pass": last_pass}, passMinutes=minutes(duration), budgetMinutes=minutes(budget, positive=True),
|
| 2220 |
+
dailyPasses=days, dailyPass=days[max(days)] if days else None,
|
| 2221 |
+
resync=entry.get("resync"), held=last_pass.get("held") or {},
|
| 2222 |
+
notOffered=last_pass.get("notOffered") or 0)
|
| 2223 |
+
if connector_key == "amazon_sp" and rt is not None:
|
| 2224 |
+
from harness import amazon_store
|
| 2225 |
+
job = amazon_store.first_load_job_summary(rt)
|
| 2226 |
+
if job:
|
| 2227 |
+
answer["amazonFirstLoad"] = job
|
| 2228 |
return _safe_connector_status(connector_key, answer)
|
| 2229 |
|
| 2230 |
|
|
|
|
| 2770 |
_materialization_wake.clear()
|
| 2771 |
try:
|
| 2772 |
for tenant, rt in tenant_scan.live_tenants(
|
| 2773 |
+
runtime, name="connector-materialization",
|
| 2774 |
+
on_error=lambda tenant, exc: _DATABASE_JOBS_LOG.error(
|
| 2775 |
"connector database runtime failed: tenant=%s type=%s",
|
| 2776 |
tenant, type(exc).__name__)):
|
| 2777 |
if _materialization_stop.is_set():
|
|
|
|
| 2939 |
import odoo_relational
|
| 2940 |
return _ConnectorStatusReads(rt, (
|
| 2941 |
keychain.KEY, keychain.SCOPE_KEY, odoo_relational.CONFIG_KEY,
|
| 2942 |
+
odoo_relational.MODEL_CATALOG_KEY, connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY,
|
| 2943 |
_ODOO_HEALTH_STATUS, _DATABASE_DISPATCH_STATUS, _ODOO_SYNC_STATUS))
|
| 2944 |
|
| 2945 |
|
|
|
|
| 3019 |
"receipt": receipt, "lastSyncAt": last_sync_at}
|
| 3020 |
|
| 3021 |
|
| 3022 |
+
def _stored_completed_sync(rt, connector_key):
|
| 3023 |
+
"""C-status: the store's lastGoodSyncAt as (stamp, True), or None when the store holds no entry."""
|
| 3024 |
+
entry = connector_sync.status(rt, connector_key, fresh=True)
|
| 3025 |
+
if entry is None:
|
| 3026 |
+
return None
|
| 3027 |
+
stamp = _sync_stamp(entry.get("lastGoodSyncAt"))
|
| 3028 |
+
return stamp, True
|
| 3029 |
+
|
| 3030 |
+
|
| 3031 |
def _amazon_completed_sync(rt):
|
| 3032 |
+
"""The store's last good Amazon sync; the mirror markers answer only before the store holds one."""
|
| 3033 |
+
stored = _stored_completed_sync(rt, "amazon_sp")
|
| 3034 |
+
return stored if stored is not None else _amazon_mirror_completion(rt)
|
| 3035 |
+
|
| 3036 |
+
|
| 3037 |
+
def _amazon_mirror_completion(rt):
|
| 3038 |
"""Project completion markers without counting business rows or building database detail."""
|
| 3039 |
import duckdb
|
| 3040 |
from harness import amazon_store as az, datastore
|
|
|
|
| 3066 |
return None, False
|
| 3067 |
|
| 3068 |
|
| 3069 |
+
def _amazon_table_schedule(rt, connection_id=None, now=None):
|
| 3070 |
+
"""Read account cadence metadata without restoring a mirror or reading source rows."""
|
| 3071 |
+
import duckdb
|
| 3072 |
+
from core import keychain
|
| 3073 |
+
from harness import amazon_store as az, datastore
|
| 3074 |
+
|
| 3075 |
+
now = now or datetime.now(timezone.utc)
|
| 3076 |
+
result = {"tableSchedule": [], "scheduleState": "unavailable",
|
| 3077 |
+
"displayTimezone": stored_business_tz(rt) or "UTC"}
|
| 3078 |
+
|
| 3079 |
+
def read(cursor, entries):
|
| 3080 |
+
schedule = []
|
| 3081 |
+
for entry in entries:
|
| 3082 |
+
cid = entry["connection_id"]
|
| 3083 |
+
prefix = az._account_prefix(cid)
|
| 3084 |
+
keys = [prefix + az.entity_key(family + table)
|
| 3085 |
+
for table in az.TABLES for family in ("", "sync_tick_", az.WATERMARK_PREFIX)]
|
| 3086 |
+
rows = cursor.execute(
|
| 3087 |
+
"SELECT entity, phase, rows, updated_at, "
|
| 3088 |
+
"CASE WHEN phase = 'throttled' THEN cursor_wd ELSE NULL END FROM _sync_state "
|
| 3089 |
+
"WHERE entity IN (SELECT unnest(?))", [keys]).fetchall()
|
| 3090 |
+
markers = {entity: {"state": phase, "rows": count, "updated_at": _sync_stamp(at),
|
| 3091 |
+
"detail": _sync_stamp(detail)}
|
| 3092 |
+
for entity, phase, count, at, detail in rows}
|
| 3093 |
+
states = {table: state for table in az.TABLES
|
| 3094 |
+
if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))}
|
| 3095 |
+
if any(not state["updated_at"] for state in states.values()):
|
| 3096 |
+
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
| 3097 |
+
# C1 owns every due time, including throttle retries and calendar periods.
|
| 3098 |
+
for row in az.sync_schedule(states, now):
|
| 3099 |
+
table = row["table"]
|
| 3100 |
+
tick = states.get(table) or {}
|
| 3101 |
+
watermark = markers.get(prefix + az.entity_key(az.WATERMARK_PREFIX + table)) or {}
|
| 3102 |
+
landed = markers.get(prefix + az.entity_key(table)) or {}
|
| 3103 |
+
completed = (tick if tick.get("state") == "done"
|
| 3104 |
+
and landed.get("state") not in ("refused", "failed", "throttled") else watermark)
|
| 3105 |
+
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3106 |
+
"lastSyncAt": (completed.get("updated_at")
|
| 3107 |
+
if completed.get("state") == "done" else None),
|
| 3108 |
+
"state": (landed.get("state")
|
| 3109 |
+
if landed.get("state") in ("refused", "failed", "throttled")
|
| 3110 |
+
else tick.get("state") or "never_run")})
|
| 3111 |
+
return schedule
|
| 3112 |
+
|
| 3113 |
+
try:
|
| 3114 |
+
path = az._tenant_mirror_path(rt.key, rt=rt)
|
| 3115 |
+
if not keychain.list_entries(rt, "amazon_sp"):
|
| 3116 |
+
return {**result, "scheduleState": "empty"}
|
| 3117 |
+
entries = az.account_entries(rt.key, rt)
|
| 3118 |
+
cid = str(connection_id or "").strip()
|
| 3119 |
+
if cid and cid != "default":
|
| 3120 |
+
entries = [entry for entry in entries if entry["connection_id"] == cid]
|
| 3121 |
+
if not entries:
|
| 3122 |
+
return result
|
| 3123 |
+
if not path.is_file():
|
| 3124 |
+
return result
|
| 3125 |
+
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 3126 |
+
schedule = read(datastore.ro_cursor(), entries)
|
| 3127 |
+
else:
|
| 3128 |
+
with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
|
| 3129 |
+
schedule = read(cursor, entries)
|
| 3130 |
+
return {**result, "tableSchedule": schedule, "scheduleState": "ready"}
|
| 3131 |
+
except Exception as exc:
|
| 3132 |
+
raise_if_resource_error(exc)
|
| 3133 |
+
return result
|
| 3134 |
+
|
| 3135 |
+
|
| 3136 |
+
def _amazon_sync_staleness(completed, schedule, now=None):
|
| 3137 |
+
"""Use the fastest Amazon table's cadence and Odoo's existing grace period."""
|
| 3138 |
+
import odoo_relational as rel
|
| 3139 |
+
fastest = max(amazon_client.CADENCE[table]["perDay"]
|
| 3140 |
+
for table in amazon_client.GROUP_A)
|
| 3141 |
+
allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS
|
| 3142 |
+
stamps = [row.get("lastSyncAt") for row in schedule if row["perDay"] == fastest]
|
| 3143 |
+
if not stamps:
|
| 3144 |
+
stamps = [completed]
|
| 3145 |
+
now = now or datetime.now(timezone.utc)
|
| 3146 |
+
stale = any((now - datetime.fromisoformat(stamp)).total_seconds() > allowed
|
| 3147 |
+
for value in stamps if (stamp := _sync_stamp(value)))
|
| 3148 |
+
return {"stale": stale, "staleAfterSeconds": allowed}
|
| 3149 |
+
|
| 3150 |
+
|
| 3151 |
def _daily_completed_sync(rt, connector_key):
|
| 3152 |
+
"""Read the stored last good sync; never create a mirror or contact a provider."""
|
| 3153 |
if connector_key == "amazon_sp":
|
| 3154 |
return _amazon_completed_sync(rt)
|
| 3155 |
+
if connector_key == "meta_ads":
|
| 3156 |
+
stored = _stored_completed_sync(rt, "meta_ads")
|
| 3157 |
+
return stored if stored is not None else _mirror_completed_sync(rt, "meta_ads")
|
| 3158 |
+
return None, False
|
| 3159 |
+
|
| 3160 |
+
|
| 3161 |
+
def _mirror_completed_sync(rt, connector_key):
|
| 3162 |
+
"""The container mirror's completed markers. Writers seed C-status from it; cards read the store."""
|
| 3163 |
+
if connector_key == "amazon_sp":
|
| 3164 |
+
return _amazon_mirror_completion(rt)
|
| 3165 |
if connector_key == "meta_ads":
|
| 3166 |
from pathlib import Path
|
| 3167 |
from harness import datastore
|
|
|
|
| 3206 |
|
| 3207 |
|
| 3208 |
def _amazon_saved_completion(rt):
|
| 3209 |
+
"""The stored last good sync, else the saved mirror manifest's completed_at, or None."""
|
| 3210 |
+
stored = _stored_completed_sync(rt, "amazon_sp")
|
| 3211 |
+
if stored is not None:
|
| 3212 |
+
return stored[0]
|
| 3213 |
key = str(getattr(rt, "key", "") or "")
|
| 3214 |
now = time.monotonic()
|
| 3215 |
cached = _SAVED_COMPLETION_CACHE.get(key)
|
|
|
|
| 3218 |
stamp = None
|
| 3219 |
try:
|
| 3220 |
from harness import amazon_store as az
|
| 3221 |
+
# Read the manifest object only: never a duckdb connection.
|
| 3222 |
download = getattr(rt, "download_bytes", None)
|
| 3223 |
raw = download(az.MIRROR_PREFIX + "manifest.json") if callable(download) else None
|
| 3224 |
manifest = json.loads(raw) if raw else None
|
|
|
|
| 3246 |
if connector_key not in registered:
|
| 3247 |
raise err(404, "sync_provider_unknown",
|
| 3248 |
f"{connector_key} is not registered for connector sync")
|
| 3249 |
+
from core import keychain
|
| 3250 |
+
import odoo_relational as rel
|
| 3251 |
rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
|
| 3252 |
+
else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY,
|
| 3253 |
+
connector_sync.STATUS_KEY, keychain.KEY, rel.CONFIG_KEY)
|
| 3254 |
if connector_key == "amazon_sp" else ()))
|
| 3255 |
resolved_connection = (_odoo_connection_slot(rt, connection_id)
|
| 3256 |
if connector_key == "odoo" else connection_id)
|
| 3257 |
jobs = _connection_jobs(rt, connector_key, resolved_connection)
|
| 3258 |
row = (connector_sync.job_summary(jobs)
|
| 3259 |
or connector_sync.read(rt, connector_key, resolved_connection))
|
| 3260 |
+
# Odoo's lifecycle comes from its job receipts; the other connectors read C-status beside no row.
|
| 3261 |
+
answer = _sync_payload(connector_key, resolved_connection, row, rt=rt)
|
| 3262 |
if connector_key == "odoo":
|
| 3263 |
receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
|
| 3264 |
+
if not answer.get("lastGoodSyncAt"):
|
| 3265 |
+
answer["lastSyncAt"] = _odoo_last_sync_at(rt, receipt, connection_id)
|
| 3266 |
if connector_key in ("amazon_sp", "meta_ads"):
|
| 3267 |
+
if answer.get("lastCompletedKnown") is True:
|
| 3268 |
+
completed, known = answer.get("lastGoodSyncAt"), True
|
| 3269 |
+
elif connector_key == "amazon_sp" and (str(resolved_connection or "").strip() or "default") != "default":
|
| 3270 |
completed, known = None, False
|
| 3271 |
else:
|
| 3272 |
completed, known = _daily_completed_sync(rt, connector_key)
|
|
|
|
| 3274 |
if connector_key == "amazon_sp" and completed is None and known and _amazon_restore_pending(rt):
|
| 3275 |
completed = _amazon_saved_completion(rt)
|
| 3276 |
answer["restoring"] = True
|
| 3277 |
+
answer.update(lastCompletedAt=completed, lastSyncAt=completed,
|
| 3278 |
+
lastCompletedKnown=known, canSync=bool(session.admin))
|
| 3279 |
+
if connector_key == "amazon_sp":
|
| 3280 |
+
now = datetime.now(timezone.utc)
|
| 3281 |
+
schedule = _amazon_table_schedule(rt, resolved_connection, now)
|
| 3282 |
+
answer.update(schedule="per_table", **schedule,
|
| 3283 |
+
**_amazon_sync_staleness(completed, schedule["tableSchedule"], now))
|
| 3284 |
+
else:
|
| 3285 |
+
answer.update(schedule="daily", **_daily_sync_staleness(completed))
|
| 3286 |
return _safe_connector_status(connector_key, answer)
|
| 3287 |
|
| 3288 |
|
|
|
|
| 4203 |
refused = (raw.get("refused") is True or is_refusal_reason(raw.get("unknownCause")) or any(item.get("table") == table for item in
|
| 4204 |
((raw.get("firstLoad") or {}).get("refused") or ()) if isinstance(item, dict))
|
| 4205 |
)
|
| 4206 |
+
if raw.get("notOffered") is True:
|
| 4207 |
+
state, error = "succeeded", None
|
| 4208 |
+
elif refused:
|
| 4209 |
state = "partial" if success else "failed"
|
| 4210 |
error = {"code": "amazon_report_refused"}
|
| 4211 |
elif raw.get("stuck") is True:
|
|
|
|
| 4220 |
state, error = "queued", None
|
| 4221 |
else:
|
| 4222 |
state, error = "never_run", None
|
| 4223 |
+
answer = _database_sync_health(
|
| 4224 |
state=state, last_success=success, materialized_timestamp=success,
|
| 4225 |
row_count=raw.get("rowsLoaded"), error=error, now=now)
|
| 4226 |
+
answer["notOffered"] = int(raw.get("notOffered") is True)
|
| 4227 |
+
return answer
|
| 4228 |
|
| 4229 |
|
| 4230 |
def _stored_database_sync_health(raw, enabled=True, *, now=None):
|
|
|
|
| 5022 |
raise_if_resource_error(exc)
|
| 5023 |
states = {}
|
| 5024 |
|
| 5025 |
+
canonical = connector_sync.status(rt, "amazon_sp", fresh=True) or {}
|
| 5026 |
+
canonical_pass = canonical.get("pass") or {}
|
| 5027 |
+
held_tables = canonical_pass.get("held") or {}
|
| 5028 |
+
from harness import amazon_store as _load_store
|
| 5029 |
+
first_load_job = _load_store.first_load_job_summary(rt) if is_admin else None
|
| 5030 |
+
|
| 5031 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
| 5032 |
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
|
| 5033 |
by_report = {}
|
|
|
|
| 5066 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 5067 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 5068 |
raw_sync = (states or {}).get(name) if is_admin else None
|
| 5069 |
+
if is_admin and (name in held_tables or first_load_job):
|
| 5070 |
+
raw_sync = dict(raw_sync or {})
|
| 5071 |
+
if name in held_tables:
|
| 5072 |
+
raw_sync["held"] = held_tables[name]
|
| 5073 |
+
raw_sync["notOffered"] = name in (canonical_pass.get("notOfferedTables") or ())
|
| 5074 |
+
if first_load_job:
|
| 5075 |
+
raw_sync["firstLoad"] = {**(raw_sync.get("firstLoad") or {}), **first_load_job}
|
| 5076 |
if isinstance(raw_sync, dict):
|
| 5077 |
from harness.amazon_store import plain_refusal_reason, is_refusal_reason
|
| 5078 |
raw_sync = dict(raw_sync)
|
|
|
|
| 5286 |
"succeeded", "disabled") if state in states), "never_run")
|
| 5287 |
successes = [_sync_stamp(row.get("lastSuccess")) for row in health]
|
| 5288 |
failure = next((row.get("error") for row in health if row.get("error")), None)
|
| 5289 |
+
answer = _database_sync_health(state=state,
|
| 5290 |
last_success=max((stamp for stamp in successes if stamp), default=None),
|
| 5291 |
error=failure)
|
| 5292 |
+
answer["notOffered"] = sum(row.get("notOffered") or 0 for row in health)
|
| 5293 |
+
return answer
|
| 5294 |
|
| 5295 |
|
| 5296 |
def _detail_overview(row, databases, graph, *, sync_health=None):
|
|
|
|
| 5853 |
"""The addressed keychain row changed between replacement preparation and its store update."""
|
| 5854 |
|
| 5855 |
|
| 5856 |
+
def _encrypted_replacement_payload(kc, label, entry_type, fields, username, rt, entry_id):
|
| 5857 |
+
"""Validate a temporary keychain row and reseal its fields for the retained tenant and entry."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5858 |
class _Stage:
|
| 5859 |
def __init__(self):
|
| 5860 |
self.bucket = {}
|
| 5861 |
+
self.key = rt.key
|
| 5862 |
|
| 5863 |
def get(self, name):
|
| 5864 |
return self.bucket if name == kc.KEY else {}
|
|
|
|
| 5875 |
row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
|
| 5876 |
if not row.get("enc"):
|
| 5877 |
raise RuntimeError("the keychain encryptor returned no encrypted payload")
|
| 5878 |
+
validated_fields = kc.read_fields(stage, temporary["id"])
|
| 5879 |
+
envelope = kc.seal_fields(rt.key, str(entry_id), validated_fields)
|
| 5880 |
+
payload = {**envelope, "preview": str(row.get("preview") or "")}
|
| 5881 |
if entry_type == "amazon_sp":
|
| 5882 |
payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
|
| 5883 |
return payload
|
|
|
|
| 5968 |
raise err(400, "bad_replacement", refusal)
|
| 5969 |
try:
|
| 5970 |
encrypted = _encrypted_replacement_payload(
|
| 5971 |
+
kc, entry.get("label"), entry_type, merged_fields, session.uname, session.runtime, entry_id)
|
| 5972 |
except kc.KeychainLocked as exc:
|
| 5973 |
raise err(503, "keychain_locked",
|
| 5974 |
f"the keychain is locked: {exc}. Nothing was changed.")
|
api/routes_customers.py
CHANGED
|
@@ -29,7 +29,7 @@ import os
|
|
| 29 |
import threading
|
| 30 |
import time
|
| 31 |
|
| 32 |
-
from fastapi import APIRouter, Body, Depends, Response
|
| 33 |
|
| 34 |
import scope_cache
|
| 35 |
from deps import Session, err, module_gate, require_session
|
|
@@ -254,10 +254,13 @@ def _pool_for(rt, team_id, agent):
|
|
| 254 |
rt.pool_cache.pop(key, None)
|
| 255 |
|
| 256 |
def _build():
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
|
|
|
|
|
|
|
|
|
| 261 |
|
| 262 |
def _evict():
|
| 263 |
# Bounded: a scope cache that only ever grows is a memory leak in a shared process.
|
|
@@ -278,10 +281,11 @@ def warm_default(rt):
|
|
| 278 |
def warm_bu_pools(rt):
|
| 279 |
"""Warm each permitted business-unit pool outside the request path."""
|
| 280 |
from core import context, perms
|
|
|
|
| 281 |
for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
|
| 282 |
-
if label == "All" or label not in
|
| 283 |
continue
|
| 284 |
-
team = perms.scope_team_id({"bus": [
|
| 285 |
if team is None:
|
| 286 |
continue
|
| 287 |
try:
|
|
@@ -523,6 +527,8 @@ class _AssemblyStore:
|
|
| 523 |
self.cacheable = False
|
| 524 |
if os.environ.get("AIOS_HEAVY_PROFILE") == "1":
|
| 525 |
print(f"[assembly-miss] revision unreadable for {name}: {exc}", flush=True)
|
|
|
|
|
|
|
| 526 |
except Exception as exc:
|
| 527 |
if self.schema_only:
|
| 528 |
from core.request_timing import raise_if_resource_error
|
|
@@ -534,6 +540,8 @@ class _AssemblyStore:
|
|
| 534 |
if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs:
|
| 535 |
try:
|
| 536 |
self.docs[key] = read()
|
|
|
|
|
|
|
| 537 |
except Exception as exc:
|
| 538 |
if self.schema_only:
|
| 539 |
from core.request_timing import raise_if_resource_error
|
|
@@ -1303,7 +1311,16 @@ def allowed_pids(session: Session):
|
|
| 1303 |
|
| 1304 |
@router.get("/customers")
|
| 1305 |
def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
|
| 1306 |
-
return _payload(session, related=related)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1307 |
|
| 1308 |
|
| 1309 |
@router.post("/customers/aggregates")
|
|
@@ -1339,10 +1356,10 @@ def assigned_view_rows(view_id: str, session: Session = Depends(module_gate(MODU
|
|
| 1339 |
session, MODULE, rows, g['fields'], g['measures'], None)
|
| 1340 |
rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
|
| 1341 |
rows = _exclude_sourceless(rows, g['fields'])
|
| 1342 |
-
return {"fields": g["fields"], "rows": rows, "viewId": view_id,
|
| 1343 |
"locked": True, "today": g["today"],
|
| 1344 |
"refusals": metric_refusals,
|
| 1345 |
-
"limits": field_limits + linked_metric_limits(metric_refusals)}
|
| 1346 |
|
| 1347 |
|
| 1348 |
# ── Wave 44: normalized, Loopable-owned Route/Zone APIs ─────────────────────
|
|
@@ -1508,6 +1525,8 @@ def _route_application(session, application, *, write=False):
|
|
| 1508 |
raise err(404, "route_application_not_found", "that Routing App does not exist")
|
| 1509 |
if not registry.may_read(app, session.uname, is_admin=session.admin):
|
| 1510 |
raise err(403, "route_application_forbidden", "your account cannot open this Routing App")
|
|
|
|
|
|
|
| 1511 |
source = app.get("sourceDatabase")
|
| 1512 |
if source not in (MODULE, "ut_odoo_customers") and MODULE not in (app.get("compatibleDatabases") or ()):
|
| 1513 |
raise err(409, "route_application_source", "this Routing App does not use the customers database")
|
|
@@ -1649,6 +1668,26 @@ def _visible_routes(session, records=None, application=None):
|
|
| 1649 |
return [route for route, _mark in _visible_normalized_routes(session, records, application)]
|
| 1650 |
|
| 1651 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1652 |
def _legacy_route_navigation(session):
|
| 1653 |
"""Wave 42 per-reader Route folders, used only as a no-write migration fallback."""
|
| 1654 |
try:
|
|
@@ -1668,7 +1707,7 @@ def _route_navigation(session, routes, application=None):
|
|
| 1668 |
legacy_folders=folders, legacy_placements=placements, application=application)
|
| 1669 |
|
| 1670 |
|
| 1671 |
-
def _route_dto(route, allowed, active, session, share_mark=None):
|
| 1672 |
from core import route_system
|
| 1673 |
allowed = {int(pid) for pid in allowed}
|
| 1674 |
stops = [stop for stop in route_system.ordered_stops(route)
|
|
@@ -1696,6 +1735,9 @@ def _route_dto(route, allowed, active, session, share_mark=None):
|
|
| 1696 |
# ⭐ `not compatibility and ...` led this expression until the legacy projection was retired.
|
| 1697 |
# Only a projected Route ever carried that marker, and nothing mints one now.
|
| 1698 |
can_edit = bool(owns_route or mark.get("sharedRole") == "edit" or source_edit)
|
|
|
|
|
|
|
|
|
|
| 1699 |
dto = {"id": route["id"], "name": route["name"], "color": route["color"],
|
| 1700 |
"application": route.get("application"),
|
| 1701 |
"number": route["number"], "visibleStopIds": visible,
|
|
@@ -1754,6 +1796,32 @@ def _route_or_404(session, route_id, application=None):
|
|
| 1754 |
return route
|
| 1755 |
|
| 1756 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1757 |
def _route_mutation_allowed(session, route, requested_source=None):
|
| 1758 |
"""W47-T08 / RULING R6 — OWNERSHIP SHORT-CIRCUITS, and that is the whole of instruction 22.
|
| 1759 |
|
|
@@ -1896,15 +1964,19 @@ def routes(session: Session = Depends(module_gate(MODULE)), application: str = N
|
|
| 1896 |
50,000 visible records sends exactly the same request as an empty grid; a future annotation
|
| 1897 |
door may accept a compact saved-View/filter reference without reopening that row-sized wire.
|
| 1898 |
"""
|
| 1899 |
-
application = _entry_route_application(session, application)
|
| 1900 |
allowed = allowed_pids(session)
|
| 1901 |
records = _route_share_records(session)
|
| 1902 |
navigation = _route_navigation(session, _visible_routes(session, records, application), application)
|
| 1903 |
-
|
| 1904 |
-
|
| 1905 |
-
|
| 1906 |
-
|
| 1907 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1908 |
|
| 1909 |
|
| 1910 |
@router.get("/customers/routes/delivery-recipients")
|
|
@@ -2079,10 +2151,13 @@ def remove_route(route_id: str, session: Session = Depends(module_gate(MODULE)),
|
|
| 2079 |
def move_route_to_folder(route_id: str, body: dict = Body(default=None),
|
| 2080 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2081 |
from core import route_system
|
| 2082 |
-
application = _entry_route_application(session, application, write=True)
|
| 2083 |
_route_application_body(body or {}, application)
|
| 2084 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2085 |
if str(route_id) not in {str(route.get("id")) for route in routes}:
|
|
|
|
|
|
|
|
|
|
| 2086 |
raise err(404, "route_not_found", "that Route does not exist")
|
| 2087 |
folder_id = (body or {}).get("folderId")
|
| 2088 |
if folder_id is not None and not isinstance(folder_id, str):
|
|
@@ -2102,12 +2177,16 @@ def move_route_to_folder(route_id: str, body: dict = Body(default=None),
|
|
| 2102 |
def reorder_route_navigation(body: dict = Body(default=None),
|
| 2103 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2104 |
from core import route_system
|
| 2105 |
-
application = _entry_route_application(session, application, write=True)
|
| 2106 |
_route_application_body(body or {}, application)
|
| 2107 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2108 |
order = (body or {}).get("order")
|
| 2109 |
if not isinstance(order, list):
|
| 2110 |
raise err(400, "invalid_route_order", "order must be a list of visible Route ids")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2111 |
result = route_system.reorder_routes(
|
| 2112 |
session.runtime, session.uname, order, routes,
|
| 2113 |
legacy_folders=legacy_folders, legacy_placements=legacy_placements, application=application)
|
|
@@ -2715,8 +2794,8 @@ def route_delivery_list(session: Session = Depends(module_gate(MODULE)), applica
|
|
| 2715 |
def normalized_route_delivery_list(route_id: str,
|
| 2716 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2717 |
"""Deliveries for the one normalized Route whose detail is open."""
|
| 2718 |
-
application = _entry_route_application(session, application)
|
| 2719 |
-
|
| 2720 |
return {"deliveries": _route_delivery_definitions(session, route_id, application)}
|
| 2721 |
|
| 2722 |
|
|
|
|
| 29 |
import threading
|
| 30 |
import time
|
| 31 |
|
| 32 |
+
from fastapi import APIRouter, Body, Depends, HTTPException, Response
|
| 33 |
|
| 34 |
import scope_cache
|
| 35 |
from deps import Session, err, module_gate, require_session
|
|
|
|
| 254 |
rt.pool_cache.pop(key, None)
|
| 255 |
|
| 256 |
def _build():
|
| 257 |
+
from harness import canonical_source, runtime as mirror_runtime
|
| 258 |
+
from registry_source import BatchStore
|
| 259 |
+
source_store = BatchStore(rt, MODULE, {'fields': []}, [])
|
| 260 |
+
# Every cold scope reads its tenant's complete mirror through the canonical builder.
|
| 261 |
+
with canonical_source.mirrored(
|
| 262 |
+
lambda: mirror_runtime.mirror_cursor(rt).cursor(), source_store, kind='customer'):
|
| 263 |
+
return cl.pool(agent, team_id)
|
| 264 |
|
| 265 |
def _evict():
|
| 266 |
# Bounded: a scope cache that only ever grows is a memory leak in a shared process.
|
|
|
|
| 281 |
def warm_bu_pools(rt):
|
| 282 |
"""Warm each permitted business-unit pool outside the request path."""
|
| 283 |
from core import context, perms
|
| 284 |
+
brands = context.brands()
|
| 285 |
for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
|
| 286 |
+
if label == "All" or label not in brands:
|
| 287 |
continue
|
| 288 |
+
team = perms.scope_team_id({"bus": [brands[label]], "role": "user"})
|
| 289 |
if team is None:
|
| 290 |
continue
|
| 291 |
try:
|
|
|
|
| 527 |
self.cacheable = False
|
| 528 |
if os.environ.get("AIOS_HEAVY_PROFILE") == "1":
|
| 529 |
print(f"[assembly-miss] revision unreadable for {name}: {exc}", flush=True)
|
| 530 |
+
except HTTPException:
|
| 531 |
+
raise
|
| 532 |
except Exception as exc:
|
| 533 |
if self.schema_only:
|
| 534 |
from core.request_timing import raise_if_resource_error
|
|
|
|
| 540 |
if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs:
|
| 541 |
try:
|
| 542 |
self.docs[key] = read()
|
| 543 |
+
except HTTPException:
|
| 544 |
+
raise
|
| 545 |
except Exception as exc:
|
| 546 |
if self.schema_only:
|
| 547 |
from core.request_timing import raise_if_resource_error
|
|
|
|
| 1311 |
|
| 1312 |
@router.get("/customers")
|
| 1313 |
def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
|
| 1314 |
+
return _audit_customer_response(_payload(session, related=related), session)
|
| 1315 |
+
|
| 1316 |
+
|
| 1317 |
+
def _audit_customer_response(payload, session):
|
| 1318 |
+
if not isinstance(payload, dict) or payload.get('ok') is False:
|
| 1319 |
+
return payload
|
| 1320 |
+
from core import personal_reads
|
| 1321 |
+
personal_reads.record_rows(MODULE, payload.get('rows') or [], st=session.runtime,
|
| 1322 |
+
user=session.user, definition={'fields': payload.get('fields') or []})
|
| 1323 |
+
return payload
|
| 1324 |
|
| 1325 |
|
| 1326 |
@router.post("/customers/aggregates")
|
|
|
|
| 1356 |
session, MODULE, rows, g['fields'], g['measures'], None)
|
| 1357 |
rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
|
| 1358 |
rows = _exclude_sourceless(rows, g['fields'])
|
| 1359 |
+
return _audit_customer_response({"fields": g["fields"], "rows": rows, "viewId": view_id,
|
| 1360 |
"locked": True, "today": g["today"],
|
| 1361 |
"refusals": metric_refusals,
|
| 1362 |
+
"limits": field_limits + linked_metric_limits(metric_refusals)}, session)
|
| 1363 |
|
| 1364 |
|
| 1365 |
# ── Wave 44: normalized, Loopable-owned Route/Zone APIs ─────────────────────
|
|
|
|
| 1525 |
raise err(404, "route_application_not_found", "that Routing App does not exist")
|
| 1526 |
if not registry.may_read(app, session.uname, is_admin=session.admin):
|
| 1527 |
raise err(403, "route_application_forbidden", "your account cannot open this Routing App")
|
| 1528 |
+
if write and not registry.may_edit(app, session.uname, is_admin=session.admin):
|
| 1529 |
+
raise err(403, "route_application_read_only", "your account cannot edit this Routing App")
|
| 1530 |
source = app.get("sourceDatabase")
|
| 1531 |
if source not in (MODULE, "ut_odoo_customers") and MODULE not in (app.get("compatibleDatabases") or ()):
|
| 1532 |
raise err(409, "route_application_source", "this Routing App does not use the customers database")
|
|
|
|
| 1668 |
return [route for route, _mark in _visible_normalized_routes(session, records, application)]
|
| 1669 |
|
| 1670 |
|
| 1671 |
+
def _cross_partition_routes(session, records, application):
|
| 1672 |
+
"""R11: the no-App list also reads Routes in other partitions shared to the caller, plus the
|
| 1673 |
+
no-App partition for an admin whose own Routing App took the request. Never writable here."""
|
| 1674 |
+
from core import route_system
|
| 1675 |
+
records = records if isinstance(records, dict) else _route_share_records(session)
|
| 1676 |
+
home = str(application or "").strip() or None
|
| 1677 |
+
out = []
|
| 1678 |
+
for route in route_system.read(session.runtime).get("routes") or []:
|
| 1679 |
+
if not isinstance(route, dict):
|
| 1680 |
+
continue
|
| 1681 |
+
partition = str(route.get("application") or "").strip() or None
|
| 1682 |
+
if partition == home:
|
| 1683 |
+
continue
|
| 1684 |
+
mark = _route_share_mark(session, route, records)
|
| 1685 |
+
if mark["shared"] or (session.admin and home is not None and partition is None):
|
| 1686 |
+
out.append((deepcopy(route), mark))
|
| 1687 |
+
out.sort(key=lambda item: (int(item[0].get("number") or 0), str(item[0].get("id") or "")))
|
| 1688 |
+
return out
|
| 1689 |
+
|
| 1690 |
+
|
| 1691 |
def _legacy_route_navigation(session):
|
| 1692 |
"""Wave 42 per-reader Route folders, used only as a no-write migration fallback."""
|
| 1693 |
try:
|
|
|
|
| 1707 |
legacy_folders=folders, legacy_placements=placements, application=application)
|
| 1708 |
|
| 1709 |
|
| 1710 |
+
def _route_dto(route, allowed, active, session, share_mark=None, *, writable=True):
|
| 1711 |
from core import route_system
|
| 1712 |
allowed = {int(pid) for pid in allowed}
|
| 1713 |
stops = [stop for stop in route_system.ordered_stops(route)
|
|
|
|
| 1735 |
# ⭐ `not compatibility and ...` led this expression until the legacy projection was retired.
|
| 1736 |
# Only a projected Route ever carried that marker, and nothing mints one now.
|
| 1737 |
can_edit = bool(owns_route or mark.get("sharedRole") == "edit" or source_edit)
|
| 1738 |
+
if not writable:
|
| 1739 |
+
# R11: a Route listed from another partition offers no write; the client hides each one.
|
| 1740 |
+
can_edit = source_edit = False
|
| 1741 |
dto = {"id": route["id"], "name": route["name"], "color": route["color"],
|
| 1742 |
"application": route.get("application"),
|
| 1743 |
"number": route["number"], "visibleStopIds": visible,
|
|
|
|
| 1796 |
return route
|
| 1797 |
|
| 1798 |
|
| 1799 |
+
def _route_for_read_or_404(session, route_id, application=None, requested=None):
|
| 1800 |
+
"""A read door's Route lookup. On the no-App door it also finds a Route the no-App list shows
|
| 1801 |
+
from another partition (R11). Every write door keeps `_route_or_404`."""
|
| 1802 |
+
try:
|
| 1803 |
+
return _route_or_404(session, route_id, application)
|
| 1804 |
+
except Exception as exc:
|
| 1805 |
+
if requested is not None or getattr(exc, "status_code", None) != 404:
|
| 1806 |
+
raise
|
| 1807 |
+
found = next((route for route, _mark in _cross_partition_routes(session, None, application)
|
| 1808 |
+
if route.get("id") == str(route_id)), None)
|
| 1809 |
+
if found is None:
|
| 1810 |
+
raise
|
| 1811 |
+
return found
|
| 1812 |
+
|
| 1813 |
+
|
| 1814 |
+
def _cross_partition_refusal(session, route_id, application):
|
| 1815 |
+
"""R11: the 409 a no-App write door answers for a Route listed from another partition."""
|
| 1816 |
+
found = next((mark for route, mark in _cross_partition_routes(session, None, application)
|
| 1817 |
+
if route.get("id") == str(route_id)), None)
|
| 1818 |
+
if found is None:
|
| 1819 |
+
return None
|
| 1820 |
+
message = ("This route is shared with you. Only its owner can move it." if found["shared"]
|
| 1821 |
+
else "This route belongs to another Routing App. Only its owner can move it.")
|
| 1822 |
+
return err(409, "route_read_only_here", message)
|
| 1823 |
+
|
| 1824 |
+
|
| 1825 |
def _route_mutation_allowed(session, route, requested_source=None):
|
| 1826 |
"""W47-T08 / RULING R6 — OWNERSHIP SHORT-CIRCUITS, and that is the whole of instruction 22.
|
| 1827 |
|
|
|
|
| 1964 |
50,000 visible records sends exactly the same request as an empty grid; a future annotation
|
| 1965 |
door may accept a compact saved-View/filter reference without reopening that row-sized wire.
|
| 1966 |
"""
|
| 1967 |
+
requested, application = application, _entry_route_application(session, application)
|
| 1968 |
allowed = allowed_pids(session)
|
| 1969 |
records = _route_share_records(session)
|
| 1970 |
navigation = _route_navigation(session, _visible_routes(session, records, application), application)
|
| 1971 |
+
listed = [_route_dto(route, allowed, (), session, _route_share_mark(session, route, records))
|
| 1972 |
+
for route in navigation["routes"]]
|
| 1973 |
+
if requested is None:
|
| 1974 |
+
# R11: a no-App reader also sees what was shared to them from another partition.
|
| 1975 |
+
rank = len(listed)
|
| 1976 |
+
for index, (route, mark) in enumerate(_cross_partition_routes(session, records, application)):
|
| 1977 |
+
route.update(folderId=None, navOrder=rank + index)
|
| 1978 |
+
listed.append(_route_dto(route, allowed, (), session, mark, writable=False))
|
| 1979 |
+
return {"routes": listed, "folders": navigation["folders"]}
|
| 1980 |
|
| 1981 |
|
| 1982 |
@router.get("/customers/routes/delivery-recipients")
|
|
|
|
| 2151 |
def move_route_to_folder(route_id: str, body: dict = Body(default=None),
|
| 2152 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2153 |
from core import route_system
|
| 2154 |
+
requested, application = application, _entry_route_application(session, application, write=True)
|
| 2155 |
_route_application_body(body or {}, application)
|
| 2156 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2157 |
if str(route_id) not in {str(route.get("id")) for route in routes}:
|
| 2158 |
+
refusal = _cross_partition_refusal(session, route_id, application) if requested is None else None
|
| 2159 |
+
if refusal is not None:
|
| 2160 |
+
raise refusal
|
| 2161 |
raise err(404, "route_not_found", "that Route does not exist")
|
| 2162 |
folder_id = (body or {}).get("folderId")
|
| 2163 |
if folder_id is not None and not isinstance(folder_id, str):
|
|
|
|
| 2177 |
def reorder_route_navigation(body: dict = Body(default=None),
|
| 2178 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2179 |
from core import route_system
|
| 2180 |
+
requested, application = application, _entry_route_application(session, application, write=True)
|
| 2181 |
_route_application_body(body or {}, application)
|
| 2182 |
routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
|
| 2183 |
order = (body or {}).get("order")
|
| 2184 |
if not isinstance(order, list):
|
| 2185 |
raise err(400, "invalid_route_order", "order must be a list of visible Route ids")
|
| 2186 |
+
if requested is None:
|
| 2187 |
+
# R11: the no-App rail lists other partitions' Routes read-only. They are not ranked here.
|
| 2188 |
+
foreign = {str(route.get("id")) for route, _mark in _cross_partition_routes(session, None, application)}
|
| 2189 |
+
order = [value for value in order if str(value) not in foreign]
|
| 2190 |
result = route_system.reorder_routes(
|
| 2191 |
session.runtime, session.uname, order, routes,
|
| 2192 |
legacy_folders=legacy_folders, legacy_placements=legacy_placements, application=application)
|
|
|
|
| 2794 |
def normalized_route_delivery_list(route_id: str,
|
| 2795 |
session: Session = Depends(module_gate(MODULE)), application: str = None):
|
| 2796 |
"""Deliveries for the one normalized Route whose detail is open."""
|
| 2797 |
+
requested, application = application, _entry_route_application(session, application)
|
| 2798 |
+
_route_for_read_or_404(session, route_id, application, requested)
|
| 2799 |
return {"deliveries": _route_delivery_definitions(session, route_id, application)}
|
| 2800 |
|
| 2801 |
|
api/routes_external.py
CHANGED
|
@@ -153,6 +153,20 @@ def _activity_store_view(real, insert):
|
|
| 153 |
return view
|
| 154 |
|
| 155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
class _PreviewRuntime:
|
| 157 |
"""Lazy copy-on-write view of one tenant runtime; no method may reach a real writer.
|
| 158 |
|
|
@@ -180,11 +194,20 @@ class _PreviewRuntime:
|
|
| 180 |
if key not in self._docs:
|
| 181 |
import application_registry
|
| 182 |
value = application_registry._read_strict(self._real, key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
value = copy.deepcopy(value)
|
| 184 |
self._initial[key] = copy.deepcopy(value)
|
| 185 |
self._docs[key] = value
|
| 186 |
try:
|
| 187 |
-
revision = self._real.revision(key)
|
| 188 |
except Exception:
|
| 189 |
revision = None
|
| 190 |
self._base_revisions[key] = copy.deepcopy(revision)
|
|
@@ -198,6 +221,10 @@ class _PreviewRuntime:
|
|
| 198 |
return self.get(name)
|
| 199 |
|
| 200 |
def get_projection(self, name, drop=()):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
value = self.get(name)
|
| 202 |
if isinstance(value, dict):
|
| 203 |
for key in drop or ():
|
|
@@ -220,6 +247,12 @@ class _PreviewRuntime:
|
|
| 220 |
return sorted(key for key in keys if key.startswith(prefix))
|
| 221 |
|
| 222 |
def get_entry(self, name, entry):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
return copy.deepcopy((self.get(name) or {}).get(entry))
|
| 224 |
|
| 225 |
def get_projections(self, names, drop=(), timeout_s=None):
|
|
@@ -318,6 +351,70 @@ class _ConfirmationRuntime:
|
|
| 318 |
def tenant(self):
|
| 319 |
return getattr(self._real, "tenant", None)
|
| 320 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None):
|
| 322 |
from core import view_audit
|
| 323 |
key = str(name)
|
|
@@ -487,6 +584,8 @@ def _database_preview_documents(session, before, after):
|
|
| 487 |
pair[side] = None
|
| 488 |
continue
|
| 489 |
rt = _PreviewRuntime(session.runtime)
|
|
|
|
|
|
|
| 490 |
rt.put(user_tables.STORE_KEY, bucket)
|
| 491 |
reader = Session(session.tenant, session.user, session.claims, rt)
|
| 492 |
try:
|
|
@@ -738,14 +837,33 @@ def _query_publication(session: Session, database: str):
|
|
| 738 |
source = {"meta ads": "meta", "amazon sp-api": "amazon"}.get(source, source)
|
| 739 |
source = connector_spec.CONNECTOR_SPEC_KEYS.get(source, source)
|
| 740 |
if not source or source in {"blank", "loopable"}:
|
| 741 |
-
raise
|
| 742 |
identity = provider(session.tenant, source)
|
| 743 |
if not isinstance(identity, str) or not identity.strip():
|
| 744 |
-
raise
|
| 745 |
return {"source": source, "snapshot_id": identity}
|
|
|
|
|
|
|
| 746 |
except Exception as exc:
|
| 747 |
-
raise
|
| 748 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 749 |
|
| 750 |
|
| 751 |
def _assert_query_publication(previous, current):
|
|
@@ -972,6 +1090,85 @@ def _assert_database_scope(session, database):
|
|
| 972 |
{"database": named})
|
| 973 |
|
| 974 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 975 |
def _external_read_scope(session: Session, database, *args, **kwargs):
|
| 976 |
"""Attach external refusal details at the shared Assistant read boundary."""
|
| 977 |
try:
|
|
@@ -981,7 +1178,7 @@ def _external_read_scope(session: Session, database, *args, **kwargs):
|
|
| 981 |
if database == "ut_activity":
|
| 982 |
scoped = _native_activity_scope(session, *args, **kwargs)
|
| 983 |
else:
|
| 984 |
-
scoped =
|
| 985 |
if pin is not None and pin.source == database:
|
| 986 |
mode = kwargs.get("mode")
|
| 987 |
if scoped.get("source_kind") == "mirror_user_table":
|
|
@@ -1146,7 +1343,7 @@ def _native_activity_scope(session, fields=None, filters=None, *, mode=None):
|
|
| 1146 |
try:
|
| 1147 |
activity_log._backend(session.runtime)
|
| 1148 |
except (TypeError, ValueError) as exc:
|
| 1149 |
-
raise
|
| 1150 |
grid = routes_tables._compute_ut_assembly(session, "ut_activity", with_rows=False, fields_only=True)
|
| 1151 |
visible = list(grid["fields"])
|
| 1152 |
keys = {field["key"] for field in visible}
|
|
@@ -1793,6 +1990,22 @@ def list_keys(session: Session = Depends(require_session)):
|
|
| 1793 |
return {"keys": keys, "can_mint": _can_use_analyst(session)}
|
| 1794 |
|
| 1795 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1796 |
@router.patch("/keys/{key_id}")
|
| 1797 |
def edit_key(key_id: str, body=Body(default=None), session: Session = Depends(require_session)):
|
| 1798 |
"""Edit metadata and narrowing scope without ever reading or replacing the secret."""
|
|
@@ -2569,9 +2782,10 @@ def query_as(rt, tenant, username, database, payload, scope=None) -> dict:
|
|
| 2569 |
if not message.endswith((".", "!", "?")):
|
| 2570 |
message += "."
|
| 2571 |
return {"ok": False, "error": {"code": code, "message": message}}
|
| 2572 |
-
except Exception:
|
| 2573 |
-
|
| 2574 |
-
|
|
|
|
| 2575 |
|
| 2576 |
|
| 2577 |
def _query_database(session: Session, database: str, payload: dict):
|
|
@@ -2590,10 +2804,11 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 2590 |
return result
|
| 2591 |
except HTTPException as exc:
|
| 2592 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 2593 |
-
|
| 2594 |
-
|
| 2595 |
-
|
| 2596 |
-
|
|
|
|
| 2597 |
if info.get("code") == "assistant_bad_filter":
|
| 2598 |
# S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
|
| 2599 |
info["message"] = bad_filter_message(info.get("message"))
|
|
@@ -2607,6 +2822,8 @@ def _query_database(session: Session, database: str, payload: dict):
|
|
| 2607 |
info["retry_query"] = {"database": database, **copy.deepcopy(payload)}
|
| 2608 |
info["retry_query"].pop("cursor", None)
|
| 2609 |
raise
|
|
|
|
|
|
|
| 2610 |
|
| 2611 |
|
| 2612 |
def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggregates, groups, limit, cursor):
|
|
@@ -2741,8 +2958,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
|
|
| 2741 |
if identity is not None and (
|
| 2742 |
scoped.get("source_kind") != "runtime_cache"
|
| 2743 |
or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
|
| 2744 |
-
raise err(409, "assistant_snapshot_unavailable",
|
| 2745 |
-
"Selected record reads are unavailable in this deployment. Retry after an update.")
|
| 2746 |
mirror = scoped.get("source_kind") == "mirror_user_table"
|
| 2747 |
visible_fields = list(scoped.get("fields") or [])
|
| 2748 |
visible_keys = {
|
|
@@ -3003,7 +3219,9 @@ def _view_context(session, database, event):
|
|
| 3003 |
if scope.startswith("ut_"):
|
| 3004 |
from routes_tables import ut_write_ctx
|
| 3005 |
|
| 3006 |
-
|
|
|
|
|
|
|
| 3007 |
if g.get("limits") and (event.get("pid") is not None or event.get("pids")):
|
| 3008 |
limit = g["limits"][0]
|
| 3009 |
raise err(409, "pid_scope_unresolved",
|
|
@@ -3109,11 +3327,8 @@ def _validate_personal_view_draft(session: Session, database: str, raw: dict) ->
|
|
| 3109 |
if any(key not in visible for key in _view_referenced_fields(config)):
|
| 3110 |
raise err(400, "assistant_unknown_field",
|
| 3111 |
"Every requested field must be readable in this database.")
|
| 3112 |
-
#
|
| 3113 |
-
|
| 3114 |
-
filters = config.get("filters")
|
| 3115 |
-
_external_read_scope(
|
| 3116 |
-
session, database, fields=[], filters=None if filters == [] else filters)
|
| 3117 |
|
| 3118 |
|
| 3119 |
def _saved_view_filters(config):
|
|
@@ -3122,9 +3337,9 @@ def _saved_view_filters(config):
|
|
| 3122 |
_assert_query_filter_bounds([value] if isinstance(value, dict) else value)
|
| 3123 |
import aios_grid
|
| 3124 |
|
| 3125 |
-
def refuse():
|
| 3126 |
raise err(400, "view_filters_unsupported",
|
| 3127 |
-
"These filters cannot be saved without changing their meaning. "
|
| 3128 |
"Use colId, op and value leaves, with conj and children for nested groups.")
|
| 3129 |
|
| 3130 |
def node(raw):
|
|
@@ -3136,8 +3351,17 @@ def _saved_view_filters(config):
|
|
| 3136 |
or not isinstance(raw[key], list) or not raw[key]):
|
| 3137 |
refuse()
|
| 3138 |
return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]}
|
| 3139 |
-
if (
|
| 3140 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3141 |
refuse()
|
| 3142 |
for key in ("value", "value2"):
|
| 3143 |
value = raw.get(key)
|
|
@@ -3575,10 +3799,236 @@ def _database_record_write_guard(session, database, *, field_keys=None):
|
|
| 3575 |
from core import user_tables
|
| 3576 |
|
| 3577 |
guard = routes_tables._records_or_refuse
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3578 |
if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime):
|
| 3579 |
# The shared wall admits a locked database's update only when every key is an authored field.
|
| 3580 |
-
return guard(session, database, cell_updates=dict.fromkeys(field_keys))
|
| 3581 |
-
return guard(session, database)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3582 |
|
| 3583 |
|
| 3584 |
def _plan_database_write(session, args):
|
|
@@ -3595,19 +4045,20 @@ def _plan_database_write(session, args):
|
|
| 3595 |
if action == "rename":
|
| 3596 |
return _plan_database_rename(session, args)
|
| 3597 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 3598 |
-
|
| 3599 |
-
|
| 3600 |
-
|
| 3601 |
-
|
|
|
|
| 3602 |
if not table or not user_tables.is_user_table(database, st=session.runtime):
|
| 3603 |
raise err(403, "connected_source_read_only",
|
| 3604 |
"Records from connected sources are read only.")
|
| 3605 |
-
revision = _bucket_revision(session.runtime, user_tables.STORE_KEY,
|
|
|
|
| 3606 |
if type(revision.get("rev")) is not int:
|
| 3607 |
raise err(503, "confirmation_revision_unavailable",
|
| 3608 |
"The database version could not be checked. Nothing was saved.")
|
| 3609 |
warnings = []
|
| 3610 |
-
bucket_after = copy.deepcopy(bucket_before)
|
| 3611 |
|
| 3612 |
if action.startswith("record_"):
|
| 3613 |
field_keys = None
|
|
@@ -3617,7 +4068,9 @@ def _plan_database_write(session, args):
|
|
| 3617 |
raise err(400, "tool_arguments_invalid", "Record values must be a non-empty object.")
|
| 3618 |
field_keys = tuple(str(key) for key in raw_values)
|
| 3619 |
_database_record_write_guard(session, database, field_keys=field_keys)
|
| 3620 |
-
|
|
|
|
|
|
|
| 3621 |
rows = table.get("rows") or {}
|
| 3622 |
if action == "record_create":
|
| 3623 |
raw_values = args.get("values")
|
|
@@ -3629,6 +4082,11 @@ def _plan_database_write(session, args):
|
|
| 3629 |
record_id = str(args.get("record_id") or "").strip()
|
| 3630 |
if not record_id.isdigit() or int(record_id) not in g["pids"]:
|
| 3631 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3632 |
before = copy.deepcopy(rows.get(record_id))
|
| 3633 |
if before is None:
|
| 3634 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
|
@@ -3645,7 +4103,8 @@ def _plan_database_write(session, args):
|
|
| 3645 |
after[field["key"]] = ""
|
| 3646 |
else:
|
| 3647 |
after = None
|
| 3648 |
-
|
|
|
|
| 3649 |
if action == "record_delete":
|
| 3650 |
(target.get("rows") or {}).pop(record_id, None)
|
| 3651 |
else:
|
|
@@ -3659,13 +4118,12 @@ def _plan_database_write(session, args):
|
|
| 3659 |
# This is part of the UI row writer's durable result, not an optional callback.
|
| 3660 |
# Include newly encountered select values in the preview and the one guarded commit.
|
| 3661 |
user_tables._hydrate_choice_options(target.get("fields") or [], [after])
|
| 3662 |
-
changed, truncated = _diff_documents(_database_preview_documents(session,
|
| 3663 |
if truncated:
|
| 3664 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 3665 |
return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
|
| 3666 |
"warnings": warnings, "revision": revision,
|
| 3667 |
-
|
| 3668 |
-
"bucket_after": bucket_after,
|
| 3669 |
"planned": {"database": database, "action": action, "record_id": record_id}}
|
| 3670 |
|
| 3671 |
field_key = str(args.get("field_key") or "").strip()
|
|
@@ -3719,8 +4177,11 @@ def _plan_database_write(session, args):
|
|
| 3719 |
# Build the exact schema document through the core schema mechanics on an isolated runtime.
|
| 3720 |
# This includes reciprocal-link cleanup/creation while keeping preview free of relation
|
| 3721 |
# refreshes, audit calls, automation events and every other out-of-store side effect.
|
|
|
|
|
|
|
|
|
|
| 3722 |
preview = _PreviewRuntime(session.runtime)
|
| 3723 |
-
|
| 3724 |
if action == "field_create":
|
| 3725 |
stored = user_tables.add_field(
|
| 3726 |
database, after, st=preview, recurrence_validator=recurrence_validator)
|
|
@@ -3739,8 +4200,8 @@ def _plan_database_write(session, args):
|
|
| 3739 |
warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
|
| 3740 |
elif not user_tables.delete_field(database, field_key, st=preview):
|
| 3741 |
raise err(403, "field_refused", "That field cannot be removed.")
|
| 3742 |
-
|
| 3743 |
-
changed, truncated = _diff_documents(_database_preview_documents(session,
|
| 3744 |
if truncated:
|
| 3745 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 3746 |
affected_ids = [field_key]
|
|
@@ -3752,8 +4213,7 @@ def _plan_database_write(session, args):
|
|
| 3752 |
return {"diff": changed,
|
| 3753 |
"affected": {"ids": affected_ids, "count": len(affected_ids)},
|
| 3754 |
"warnings": warnings, "revision": revision,
|
| 3755 |
-
|
| 3756 |
-
"bucket_after": bucket_after,
|
| 3757 |
"planned": {"database": database, "action": action, "field_key": field_key}}
|
| 3758 |
|
| 3759 |
|
|
@@ -3782,28 +4242,47 @@ def _execute_database_object(session, args, *, plan=None):
|
|
| 3782 |
"The database metadata changed after the preview, " + _STALE_FIX) from exc
|
| 3783 |
return {"database": database, "action": action, "meta": result["meta"],
|
| 3784 |
"affected": copy.deepcopy(plan.get("affected") or {})}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3785 |
expected_rev = plan.get("revision", {}).get("rev")
|
| 3786 |
expected_digest = plan.get("bucket_before_digest")
|
| 3787 |
replacement = copy.deepcopy(plan.get("bucket_after"))
|
| 3788 |
-
if type(expected_rev) is not int or not isinstance(replacement, dict):
|
| 3789 |
raise err(503, "confirmation_revision_unavailable",
|
| 3790 |
"The database version could not be checked. Nothing was saved.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3791 |
|
| 3792 |
def apply_exact(current):
|
| 3793 |
-
|
|
|
|
| 3794 |
raise view_audit.ViewRevisionStale(
|
| 3795 |
rev_sent=expected_rev, rev_current=expected_rev + 1,
|
| 3796 |
message="The database changed after preview.")
|
| 3797 |
-
|
|
|
|
|
|
|
| 3798 |
|
| 3799 |
try:
|
| 3800 |
-
# One
|
| 3801 |
# inside the PG lock where available; the closure's digest check also protects a backend
|
| 3802 |
# that can only provide the runtime's documented guarded fallback.
|
| 3803 |
user_tables.update_with_activity(
|
| 3804 |
database, apply_exact, st=session.runtime, by=session.uname,
|
| 3805 |
-
via=_trusted_activity_via(session), row_ids=
|
| 3806 |
-
flush="sync", expect_rev=expected_rev)
|
| 3807 |
except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
|
| 3808 |
raise err(409, "confirmation_stale",
|
| 3809 |
"The database changed after the preview, " + _STALE_FIX) from exc
|
|
@@ -4204,7 +4683,15 @@ def _tool_context(session: Session, *, preview=False) -> external_tools.ToolCont
|
|
| 4204 |
)
|
| 4205 |
|
| 4206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4207 |
def _strong_confirmation_required(name, args):
|
|
|
|
|
|
|
|
|
|
| 4208 |
if name not in external_tools.TOOLS:
|
| 4209 |
import tool_groups
|
| 4210 |
resolved = tool_groups.resolve(name, args)
|
|
@@ -4429,7 +4916,7 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4429 |
"Part of this change could not be accepted. Nothing was previewed.")
|
| 4430 |
|
| 4431 |
remote_note = _is_website_note(name, normalised)
|
| 4432 |
-
plan = result if name == "database_object" or remote_note else None
|
| 4433 |
if plan is not None:
|
| 4434 |
diff = copy.deepcopy(plan.get("diff") or [])
|
| 4435 |
warnings = list(plan.get("warnings") or [])
|
|
@@ -4462,6 +4949,9 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4462 |
normalised["arguments"]["expect"] = [
|
| 4463 |
{"user": row["user"], "role": row["role"]}
|
| 4464 |
for row in result["current_entries"]]
|
|
|
|
|
|
|
|
|
|
| 4465 |
|
| 4466 |
_canonical_json(diff, capped=False)
|
| 4467 |
diff_digest = _json_digest(diff)
|
|
@@ -4520,6 +5010,8 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
|
|
| 4520 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 4521 |
if name == "apps" and normalised.get("action") == "catalog_build":
|
| 4522 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
|
|
|
|
|
|
| 4523 |
if payload["strong"]:
|
| 4524 |
response["confirmation_phrase"] = _CONFIRMATION_PHRASE
|
| 4525 |
if issue_token:
|
|
@@ -4698,8 +5190,10 @@ def _operation_access(session, item, *, confirming=False):
|
|
| 4698 |
routes_nav._nav_meta_write_patch(_database_nav_body(args), session)
|
| 4699 |
elif name == "database_object":
|
| 4700 |
import routes_tables
|
| 4701 |
-
g = routes_tables.ut_write_ctx(session, args["database"])
|
| 4702 |
action = args["action"]
|
|
|
|
|
|
|
|
|
|
| 4703 |
if action in {"record_create", "record_update"} and item["status"] == "committed":
|
| 4704 |
if any(int(rid) not in g["pids"] for rid in item["affected"]["ids"]):
|
| 4705 |
raise err(403, "operation_unavailable",
|
|
@@ -5055,6 +5549,9 @@ def _confirm_change(session, args):
|
|
| 5055 |
if tool == "database_object":
|
| 5056 |
result = _execute_database_object(
|
| 5057 |
fresh_session, arguments, plan=recomputed.get("_plan"))
|
|
|
|
|
|
|
|
|
|
| 5058 |
else:
|
| 5059 |
from datetime import datetime, timezone
|
| 5060 |
commit_started = datetime.now(timezone.utc)
|
|
@@ -5085,11 +5582,13 @@ def _confirm_change(session, args):
|
|
| 5085 |
from core import user_tables
|
| 5086 |
import routes_nav
|
| 5087 |
bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
|
| 5088 |
-
|
| 5089 |
-
|
| 5090 |
produced_rev = before_revision["rev"] + 1
|
| 5091 |
if after_revision.get("rev") != produced_rev:
|
| 5092 |
after_revision = {"rev": produced_rev}
|
|
|
|
|
|
|
| 5093 |
else:
|
| 5094 |
after_revision = copy.deepcopy(guarded_session.runtime._expected)
|
| 5095 |
after_revision_digest = _json_digest(after_revision)
|
|
@@ -5119,10 +5618,14 @@ def _confirm_change(session, args):
|
|
| 5119 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 5120 |
if tool == "apps" and arguments.get("action") == "catalog_build":
|
| 5121 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
|
|
|
|
|
|
|
|
|
| 5122 |
_confirmation_status(fresh_session, operation_id, "committed", response)
|
| 5123 |
if isinstance(stored.get("arguments_ref"), dict):
|
| 5124 |
_drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
|
| 5125 |
-
if tool != "database_object" and
|
|
|
|
| 5126 |
try:
|
| 5127 |
targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
|
| 5128 |
build_kind=payload["buildKind"])
|
|
@@ -5138,6 +5641,13 @@ def _confirm_change(session, args):
|
|
| 5138 |
|
| 5139 |
# Codes are public identifiers. Reasons are product copy, never exception diagnostics.
|
| 5140 |
_MCP_CODE_GROUPS = (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5141 |
("The arguments do not match this action. Read its guide and correct them.", """
|
| 5142 |
tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
|
| 5143 |
assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
|
|
@@ -5254,7 +5764,7 @@ MCP_RESULT_CODES.update({
|
|
| 5254 |
"query_filter_limit": "Filters allow 3 levels, 100 nodes and 50 conditions per group to bound query validation work. To reduce that work, " + _MCP_FIXES["query_filter_limit"] + ".",
|
| 5255 |
"query_cursor_stale": "The query, data or permissions changed. To restart, " + _MCP_FIXES["query_cursor_stale"] + ".",
|
| 5256 |
"query_cursor_invalid": "The query cursor is invalid. Repeat the original query without cursor for a fresh first page.",
|
| 5257 |
-
"query_source_unavailable": "This database
|
| 5258 |
"test_connection_production": "Test connections cannot write to production. Use a staging environment.",
|
| 5259 |
"test_connection_environment_unknown": "This environment is unknown. Test connections need a named non-production environment.",
|
| 5260 |
"external_test_connection_invalid": "Set test_connection to true or false.",
|
|
@@ -5288,7 +5798,8 @@ _MCP_INTERNAL_PROSE = re.compile(
|
|
| 5288 |
r"Traceback|stack trace|\bengine\b|\bmanifest\b|\brevision-guard\b|\bgate\b|"
|
| 5289 |
r"\b(?:tenant store|dispatcher|document cleaner)\b|"
|
| 5290 |
r"(?:[A-Za-z]:[\\/]|/(?:tmp|root|work|home|srv|app)/)|"
|
| 5291 |
-
|
|
|
|
| 5292 |
r"\.(?:claude|agents|codex)/|~[/\\]", re.I)
|
| 5293 |
_MCP_PROSE_REPLACEMENTS = (
|
| 5294 |
(r"strict Assistant scope wall|scope wall|live UI walls|UI walls", "permissions"),
|
|
@@ -5674,6 +6185,7 @@ def _guide_skills(session: Session) -> list[dict]:
|
|
| 5674 |
scoped = SkillSession(session.tenant, session.user, session.claims, session.runtime)
|
| 5675 |
rows = [{"key": row["key"], "title": _markdown_inline(row["title"])}
|
| 5676 |
for row in loopy_skills.list_skills(scoped)]
|
|
|
|
| 5677 |
try:
|
| 5678 |
files = importlib.import_module("intelligence_store")
|
| 5679 |
except ModuleNotFoundError as exc:
|
|
@@ -5686,10 +6198,15 @@ def _guide_skills(session: Session) -> list[dict]:
|
|
| 5686 |
if exc.status_code != 403:
|
| 5687 |
raise
|
| 5688 |
return rows
|
|
|
|
| 5689 |
offset = 0
|
| 5690 |
while True:
|
| 5691 |
page = files.list_files(session, section="playbooks", offset=offset, limit=100)
|
| 5692 |
for row in page["files"]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5693 |
if row.get("fieldCheck", {}).get("state", "ready") == "ready":
|
| 5694 |
rows.append({"path": row["path"], "title": _markdown_inline(row["title"])})
|
| 5695 |
offset += len(page["files"])
|
|
@@ -5893,6 +6410,8 @@ def _mcp_capabilities(session, groups, rows):
|
|
| 5893 |
required.append("a named non-production environment for this test connection")
|
| 5894 |
if group == "websites":
|
| 5895 |
required.extend(_website_requirements(session, note=action == "add_internal_note"))
|
|
|
|
|
|
|
| 5896 |
if legacy in {"describe_database", "query_database", "view_upsert", "view_delete", "view_share_preview", "view_object", "app_upsert", "app_delete"} or (
|
| 5897 |
legacy == "database_object" and entry.get("legacy_action") != "list") or legacy == "sheet_set_binding":
|
| 5898 |
if not rows:
|
|
@@ -5993,10 +6512,37 @@ def _guide_unread_count(session: Session) -> int:
|
|
| 5993 |
return count if type(count) is int and count >= 0 else 0
|
| 5994 |
|
| 5995 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5996 |
def _mcp_guide(session: Session, group=None) -> str:
|
| 5997 |
"""Return the group index or one group's action schemas without reading records."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5998 |
import tool_groups
|
| 5999 |
groups = tool_groups.groups()
|
|
|
|
| 6000 |
if isinstance(group, str) and group not in groups and group != "data":
|
| 6001 |
# A cased, plural or dotted name (Sheets, sheet.write_cells) reads the one group it names.
|
| 6002 |
key = group.strip().lower().partition(".")[0]
|
|
@@ -6004,9 +6550,10 @@ def _mcp_guide(session: Session, group=None) -> str:
|
|
| 6004 |
if name in groups or name == "data"), group)
|
| 6005 |
alias = group == "data"
|
| 6006 |
group = tool_groups.canonical_name(group)
|
|
|
|
| 6007 |
if group is not None and group not in groups:
|
| 6008 |
-
|
| 6009 |
-
|
| 6010 |
# One rows-free definitions read per guide: a live runtime pays one pool checkout per database.
|
| 6011 |
# The guide reads schemas only, so no reader here needs `rows`. On a failed read, keep the old path.
|
| 6012 |
try:
|
|
@@ -6024,7 +6571,7 @@ def _mcp_guide(session: Session, group=None) -> str:
|
|
| 6024 |
rows = [row for row in rows if row["permitted"]]
|
| 6025 |
capabilities = _mcp_capabilities(session, groups, [row for row in rows if row.get("schema")])
|
| 6026 |
unread = _guide_unread_count(session)
|
| 6027 |
-
lines = ["# Loopable guide", "", f"Signed in as {session.uname}.", "", MCP_INSTRUCTIONS, "",
|
| 6028 |
f"Scheduled results you have not read: {unread}.",
|
| 6029 |
*(["Read schedules.inbox before answering."] if unread > 0 else []), "",
|
| 6030 |
"Call a group with action and arguments. Each action keeps its own read or write rules.",
|
|
@@ -6141,8 +6688,17 @@ def _mcp_tool_specs() -> list[dict]:
|
|
| 6141 |
"for supported actions, current permissions, schemas and confirmation rules.")
|
| 6142 |
if spec["name"] == "apps":
|
| 6143 |
spec["description"] += " Read apps.guide before creating or editing an App."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6144 |
spec["inputSchema"]["properties"]["purpose"] = copy.deepcopy(wiki_log.PURPOSE_SCHEMA)
|
| 6145 |
spec["inputSchema"]["properties"]["about"] = copy.deepcopy(external_tools.ABOUT_SCHEMA)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6146 |
spec["inputSchema"]["properties"]["action"]["enum"] = list(groups[spec["name"]]["actions"])
|
| 6147 |
if spec["name"] == "sheet":
|
| 6148 |
from tool_groups.sheet import WORKBOOK_RESOURCE_URI, write_guide
|
|
@@ -6176,7 +6732,7 @@ def _dispatch_external_tool(ctx, name, arguments, *, about=None):
|
|
| 6176 |
import tool_groups
|
| 6177 |
resolved = tool_groups.resolve(name, arguments)
|
| 6178 |
if resolved is None:
|
| 6179 |
-
raise sheets.SheetRefusal("tool_not_found",
|
| 6180 |
entry, args = resolved
|
| 6181 |
if call is not None and call["tool"] == name:
|
| 6182 |
external_activity.note_action(arguments["action"])
|
|
@@ -6191,6 +6747,30 @@ def _dispatch_external_tool(ctx, name, arguments, *, about=None):
|
|
| 6191 |
return result
|
| 6192 |
|
| 6193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6194 |
def _record_mcp_method(session, req, reply):
|
| 6195 |
"""Retain bounded method names for admitted protocol requests without request values."""
|
| 6196 |
try:
|
|
@@ -6249,8 +6829,9 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
|
|
| 6249 |
code = error.get("code", "tool_refused") if error else ("tool_refused" if reply.get("error") else "")
|
| 6250 |
if code and not call["limitation_recorded"]:
|
| 6251 |
from routes_feedback import record_limitation
|
| 6252 |
-
|
| 6253 |
-
|
|
|
|
| 6254 |
except Exception:
|
| 6255 |
pass
|
| 6256 |
finally:
|
|
@@ -6463,6 +7044,9 @@ def _mcp_next_call(code, message):
|
|
| 6463 |
if code == "assistant_unknown_database":
|
| 6464 |
# The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
|
| 6465 |
return fix
|
|
|
|
|
|
|
|
|
|
| 6466 |
return text if not fix or fix in text else (text + " " + fix).strip()
|
| 6467 |
|
| 6468 |
|
|
@@ -6534,6 +7118,9 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6534 |
if method != "tools/call":
|
| 6535 |
return protocol_error(rid, -32601, "The requested MCP method was not found.")
|
| 6536 |
arguments = _mcp_hoist_call_metadata(name, params.get("arguments", {}))
|
|
|
|
|
|
|
|
|
|
| 6537 |
intent = _mcp_call_intent(arguments)
|
| 6538 |
if intent is None and name == "confirm_change" and isinstance(arguments, dict):
|
| 6539 |
intent = _mcp_saved_intent(session, arguments)
|
|
@@ -6628,6 +7215,16 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6628 |
out["error"]["data"] = data # Accepted edits remain visible even on partial refusal.
|
| 6629 |
else:
|
| 6630 |
out = {"ok": True, "data": data}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6631 |
if name == "sheet" and arguments.get("action") == "workbook.view" and out.get("ok"):
|
| 6632 |
book = data["workbook"]
|
| 6633 |
reply = _mcp_content(rid, {"ok": True, "data": {"workbook": {
|
|
@@ -6680,9 +7277,11 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
|
|
| 6680 |
out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
|
| 6681 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 6682 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
| 6683 |
-
except Exception:
|
|
|
|
| 6684 |
out = _tool_refusal(name, "tool_unavailable",
|
| 6685 |
"The tool could not complete this operation, so try the same call again shortly.", 503)
|
|
|
|
| 6686 |
return _mcp_content(rid, out, session)
|
| 6687 |
|
| 6688 |
|
|
@@ -6736,22 +7335,44 @@ _WRITE_TOOLS = frozenset({
|
|
| 6736 |
"database_object", "view_object", "workbook_object", "worksheet_object",
|
| 6737 |
"range_object", "calculation_object", "validated_batch",
|
| 6738 |
"confirm_change",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6739 |
})
|
| 6740 |
|
| 6741 |
|
| 6742 |
def _assert_write_allow_list(*, check_groups=False) -> None:
|
| 6743 |
declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
|
| 6744 |
-
|
| 6745 |
-
|
|
|
|
| 6746 |
if undeclared or stale:
|
| 6747 |
raise RuntimeError("External write allow-list mismatch: undeclared="
|
| 6748 |
+ repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
|
| 6749 |
if check_groups:
|
| 6750 |
import tool_groups
|
| 6751 |
-
|
|
|
|
| 6752 |
for entry in group["actions"].values()}
|
| 6753 |
if set(external_tools.TOOLS) - mapped - {"confirm_change"}:
|
| 6754 |
raise RuntimeError("Every legacy tool must have a grouped action")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6755 |
|
| 6756 |
|
| 6757 |
_assert_write_allow_list()
|
|
|
|
| 153 |
return view
|
| 154 |
|
| 155 |
|
| 156 |
+
def _sheet_row_revision(tx, key):
|
| 157 |
+
"""Read the header's store revision on the same transaction as its sheet rows."""
|
| 158 |
+
from core import store_pg, view_audit
|
| 159 |
+
if isinstance(tx, sheets._PgSheetRows):
|
| 160 |
+
row = tx.con.execute(store_pg._q(
|
| 161 |
+
"SELECT rev, updated_at FROM {tbl} WHERE key = %s", "store_kv", tx.slug),
|
| 162 |
+
(tx.namespace + key,)).fetchone()
|
| 163 |
+
return store_pg._revision_value(row[0], row[1]) if row else store_pg._revision_value(None, None)
|
| 164 |
+
reader = getattr(type(tx), "revision", None)
|
| 165 |
+
if callable(reader):
|
| 166 |
+
return reader(tx, key)
|
| 167 |
+
raise view_audit.RevisionUnverifiable("The Sheet transaction cannot verify its header revision.")
|
| 168 |
+
|
| 169 |
+
|
| 170 |
class _PreviewRuntime:
|
| 171 |
"""Lazy copy-on-write view of one tenant runtime; no method may reach a real writer.
|
| 172 |
|
|
|
|
| 194 |
if key not in self._docs:
|
| 195 |
import application_registry
|
| 196 |
value = application_registry._read_strict(self._real, key)
|
| 197 |
+
revision = None
|
| 198 |
+
if key.startswith(sheets.WORKBOOK_KEY_PREFIX) and sheets.is_row_workbook(value):
|
| 199 |
+
# Hydrate through the reader, never delegate a native writer to this preview.
|
| 200 |
+
with sheets._sheet_rows_transaction(self._real) as tx:
|
| 201 |
+
if tx is None:
|
| 202 |
+
raise err(503, "workbook_unavailable", "The workbook snapshot is unavailable. Try again.")
|
| 203 |
+
head = tx.get(key)
|
| 204 |
+
value = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head
|
| 205 |
+
revision = _sheet_row_revision(tx, key)
|
| 206 |
value = copy.deepcopy(value)
|
| 207 |
self._initial[key] = copy.deepcopy(value)
|
| 208 |
self._docs[key] = value
|
| 209 |
try:
|
| 210 |
+
revision = revision if revision is not None else self._real.revision(key)
|
| 211 |
except Exception:
|
| 212 |
revision = None
|
| 213 |
self._base_revisions[key] = copy.deepcopy(revision)
|
|
|
|
| 221 |
return self.get(name)
|
| 222 |
|
| 223 |
def get_projection(self, name, drop=()):
|
| 224 |
+
project = getattr(self._real, "get_projection", None)
|
| 225 |
+
if str(name) not in self._docs and callable(project):
|
| 226 |
+
# D-11174: an unstaged document is projected by its source, never copied whole here.
|
| 227 |
+
return project(name, drop=drop)
|
| 228 |
value = self.get(name)
|
| 229 |
if isinstance(value, dict):
|
| 230 |
for key in drop or ():
|
|
|
|
| 247 |
return sorted(key for key in keys if key.startswith(prefix))
|
| 248 |
|
| 249 |
def get_entry(self, name, entry):
|
| 250 |
+
if str(name) not in self._docs:
|
| 251 |
+
# D-11174: an unstaged document answers one entry from its source's entry door.
|
| 252 |
+
from core import user_tables
|
| 253 |
+
reader = user_tables._entry_reader(self._real)
|
| 254 |
+
if reader is not None:
|
| 255 |
+
return copy.deepcopy(reader(name, entry))
|
| 256 |
return copy.deepcopy((self.get(name) or {}).get(entry))
|
| 257 |
|
| 258 |
def get_projections(self, names, drop=(), timeout_s=None):
|
|
|
|
| 351 |
def tenant(self):
|
| 352 |
return getattr(self._real, "tenant", None)
|
| 353 |
|
| 354 |
+
@contextmanager
|
| 355 |
+
def sheet_rows_transaction(self, *, write=False):
|
| 356 |
+
"""Guard native Sheet writes under the header lock with the preview's store revision."""
|
| 357 |
+
from core import view_audit
|
| 358 |
+
with sheets._sheet_rows_transaction(self._real, write=write) as tx:
|
| 359 |
+
if tx is None:
|
| 360 |
+
yield tx
|
| 361 |
+
return
|
| 362 |
+
owner = self
|
| 363 |
+
held = {}
|
| 364 |
+
|
| 365 |
+
class GuardedRows:
|
| 366 |
+
def get(self, key, *, lock=False, **selection):
|
| 367 |
+
if lock and not write:
|
| 368 |
+
raise view_audit.RevisionUnverifiable("A Sheet read cannot acquire write authority.")
|
| 369 |
+
value = tx.get(key, lock=lock, **selection)
|
| 370 |
+
if lock:
|
| 371 |
+
expected = owner._expected.get(key)
|
| 372 |
+
rev = expected.get("rev") if isinstance(expected, dict) else None
|
| 373 |
+
if (not key.startswith(sheets.WORKBOOK_KEY_PREFIX) or type(rev) is not int
|
| 374 |
+
or held):
|
| 375 |
+
raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed header.")
|
| 376 |
+
current = _sheet_row_revision(tx, key)
|
| 377 |
+
if current.get("rev") != rev or ("token" in expected and current.get("token") != expected["token"]):
|
| 378 |
+
raise view_audit.ViewRevisionStale(rev_current=current.get("rev"), rev_sent=rev)
|
| 379 |
+
before = (sheets._verify_row_layout(tx, value) if sheets.is_row_workbook(value)
|
| 380 |
+
else tx.get(key))
|
| 381 |
+
held.update(key=key, before=copy.deepcopy(before), head=value)
|
| 382 |
+
return value
|
| 383 |
+
|
| 384 |
+
def revision(self, key):
|
| 385 |
+
return _sheet_row_revision(tx, key)
|
| 386 |
+
|
| 387 |
+
def put(self, key, value):
|
| 388 |
+
header = held.get("key")
|
| 389 |
+
if not header or (key != header and key != sheets.sheet_row_key(
|
| 390 |
+
header[len(sheets.WORKBOOK_KEY_PREFIX):], value.get("id", ""))):
|
| 391 |
+
raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed workbook.")
|
| 392 |
+
if key == header and value and value.get("id") != header[len(sheets.WORKBOOK_KEY_PREFIX):]:
|
| 393 |
+
raise view_audit.RevisionUnverifiable("The Sheet header identity changed.")
|
| 394 |
+
tx.put(key, value)
|
| 395 |
+
|
| 396 |
+
def delete(self, key):
|
| 397 |
+
head = held.get("head") or {}
|
| 398 |
+
if key not in {sheets.sheet_row_key(head["id"], ref["id"])
|
| 399 |
+
for ref in head.get("rowSheets", [])}:
|
| 400 |
+
raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed sheet.")
|
| 401 |
+
tx.delete(key)
|
| 402 |
+
|
| 403 |
+
yield GuardedRows()
|
| 404 |
+
if held:
|
| 405 |
+
key = held["key"]
|
| 406 |
+
head = tx.get(key)
|
| 407 |
+
after = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head
|
| 408 |
+
produced = _sheet_row_revision(tx, key)
|
| 409 |
+
if held:
|
| 410 |
+
self._expected[key] = copy.deepcopy(produced)
|
| 411 |
+
if held["before"] != after:
|
| 412 |
+
previous = self._committed_documents.get(key)
|
| 413 |
+
self._committed_documents[key] = {
|
| 414 |
+
"before": previous["before"] if previous else held["before"],
|
| 415 |
+
"after": copy.deepcopy(after),
|
| 416 |
+
}
|
| 417 |
+
|
| 418 |
def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None):
|
| 419 |
from core import view_audit
|
| 420 |
key = str(name)
|
|
|
|
| 584 |
pair[side] = None
|
| 585 |
continue
|
| 586 |
rt = _PreviewRuntime(session.runtime)
|
| 587 |
+
# Stage the planned bucket without reading the source's whole document first.
|
| 588 |
+
_seed_preview(rt, user_tables.STORE_KEY, {})
|
| 589 |
rt.put(user_tables.STORE_KEY, bucket)
|
| 590 |
reader = Session(session.tenant, session.user, session.claims, rt)
|
| 591 |
try:
|
|
|
|
| 837 |
source = {"meta ads": "meta", "amazon sp-api": "amazon"}.get(source, source)
|
| 838 |
source = connector_spec.CONNECTOR_SPEC_KEYS.get(source, source)
|
| 839 |
if not source or source in {"blank", "loopable"}:
|
| 840 |
+
raise _query_source_unavailable("no_connector")
|
| 841 |
identity = provider(session.tenant, source)
|
| 842 |
if not isinstance(identity, str) or not identity.strip():
|
| 843 |
+
raise _query_source_unavailable("invalid_publication")
|
| 844 |
return {"source": source, "snapshot_id": identity}
|
| 845 |
+
except HTTPException:
|
| 846 |
+
raise
|
| 847 |
except Exception as exc:
|
| 848 |
+
raise _query_failure(exc, "query publication") from exc
|
| 849 |
+
|
| 850 |
+
|
| 851 |
+
def _query_source_unavailable(cause, *, message=None):
|
| 852 |
+
"""S82-T125: a 503 `query_source_unavailable` naming one cause from `deps`' closed set."""
|
| 853 |
+
return err(503, "query_source_unavailable", deps._assistant_source_message(cause, "query", message=message))
|
| 854 |
+
|
| 855 |
+
|
| 856 |
+
def _query_failure(exc, where):
|
| 857 |
+
"""Name transport failures and log unexpected exceptions as unavailable tools."""
|
| 858 |
+
import logging
|
| 859 |
+
logging.getLogger(__name__).warning("%s failed (%s)", where, type(exc).__name__)
|
| 860 |
+
transport = isinstance(exc, (ConnectionError, TimeoutError, OSError)) or any(
|
| 861 |
+
cls.__module__.split(".")[0] in {"duckdb", "_duckdb", "psycopg", "psycopg2", "psycopg_pool"}
|
| 862 |
+
and cls.__name__ in {"IOException", "ConnectionException", "OperationalError", "InterfaceError", "PoolTimeout"}
|
| 863 |
+
for cls in type(exc).__mro__)
|
| 864 |
+
if transport:
|
| 865 |
+
return _query_source_unavailable("store_silent")
|
| 866 |
+
return err(503, "tool_unavailable", "This query could not be completed because of an error on the server.")
|
| 867 |
|
| 868 |
|
| 869 |
def _assert_query_publication(previous, current):
|
|
|
|
| 1090 |
{"database": named})
|
| 1091 |
|
| 1092 |
|
| 1093 |
+
def _external_shared_cells(session, database, rows):
|
| 1094 |
+
"""Read overlay values for exactly the records returned by the permissioned source reader."""
|
| 1095 |
+
from core import shared_overlay
|
| 1096 |
+
return shared_overlay.cells(database, [row["pid"] for row in rows], st=session.runtime)
|
| 1097 |
+
|
| 1098 |
+
|
| 1099 |
+
def _external_shared_read_scope(session, database, fields=None, filters=None, *, mode=None, **kwargs):
|
| 1100 |
+
"""Project shared user cells over permitted connector pages with the existing field walls."""
|
| 1101 |
+
from core import shared_overlay
|
| 1102 |
+
|
| 1103 |
+
shared = {}
|
| 1104 |
+
if str(database).startswith("ut_") and database != "ut_activity":
|
| 1105 |
+
definition, _ = deps._assistant_table_defn(session, database)
|
| 1106 |
+
declared = {field.get("key") for field in definition.get("fields") or ()}
|
| 1107 |
+
shared = {key: dict(field, key=key, source="overlay", shared=True)
|
| 1108 |
+
for key, field in shared_overlay.fields(database, st=session.runtime).items()
|
| 1109 |
+
if key not in declared and isinstance(field, dict)}
|
| 1110 |
+
if not shared:
|
| 1111 |
+
return deps.assistant_read_scope(session, database, fields=fields, filters=filters,
|
| 1112 |
+
mode=mode, **kwargs)
|
| 1113 |
+
# The source reader still resolves every row permission before any overlay cell is fetched.
|
| 1114 |
+
schema = deps.assistant_read_scope(session, database, fields=None, mode="schema", **kwargs)
|
| 1115 |
+
contract = list(definition.get("fields") or ()) + list(shared.values())
|
| 1116 |
+
available = list(schema["fields"]) + list(shared.values())
|
| 1117 |
+
withheld = set()
|
| 1118 |
+
for user in [session.user, *perm_scope._read_users(kwargs.get("also_as", ()))]:
|
| 1119 |
+
hidden = perm_scope.hidden_keys(user, database, contract, st=session.runtime)
|
| 1120 |
+
visible = (perm_scope.assistant_visible_fields(available, user, database)
|
| 1121 |
+
if perm_scope.assistant_entry(user, database) is not None else available)
|
| 1122 |
+
allowed = {field["key"] for field in visible if field.get("key") not in hidden}
|
| 1123 |
+
withheld.update(field["key"] for field in contract if field.get("key") not in allowed)
|
| 1124 |
+
available = [field for field in available if field.get("key") in allowed]
|
| 1125 |
+
selected = deps._assistant_requested_fields(fields, available)
|
| 1126 |
+
try:
|
| 1127 |
+
perm_scope.validate_assistant_filter(filters, available)
|
| 1128 |
+
except ValueError as exc:
|
| 1129 |
+
deps._assistant_refuse_filter(exc, filters, available)
|
| 1130 |
+
|
| 1131 |
+
def refuse(key, code, verb):
|
| 1132 |
+
raise err(400, code, f'Field "{deps._assistant_display_key(key)}" is stored beside the connector rows, '
|
| 1133 |
+
f"so it can be read but not {verb}.")
|
| 1134 |
+
|
| 1135 |
+
for leaf in deps._assistant_filter_leaves(filters):
|
| 1136 |
+
rhs = leaf.get("rhs") if isinstance(leaf, dict) else None
|
| 1137 |
+
for key in (leaf.get("colId") if isinstance(leaf, dict) else None,
|
| 1138 |
+
rhs.get("colId") if isinstance(rhs, dict) else None):
|
| 1139 |
+
if key in shared:
|
| 1140 |
+
refuse(key, "assistant_bad_filter", "filtered")
|
| 1141 |
+
if isinstance(mode, tuple) and mode:
|
| 1142 |
+
if mode[0] == "page" and len(mode) > 3:
|
| 1143 |
+
for rule in mode[3] or ():
|
| 1144 |
+
if rule.get("field") in shared:
|
| 1145 |
+
refuse(rule["field"], "query_sorts_invalid", "sorted")
|
| 1146 |
+
if mode[0] == "aggregate":
|
| 1147 |
+
for spec in mode[1] or ():
|
| 1148 |
+
if spec.get("field") in shared:
|
| 1149 |
+
refuse(spec["field"], "query_aggregates_invalid", "aggregated")
|
| 1150 |
+
for group in (mode[2].get("groupBy") or ()) if len(mode) > 2 else ():
|
| 1151 |
+
if group.get("field") in shared:
|
| 1152 |
+
refuse(group["field"], "query_groups_invalid", "grouped")
|
| 1153 |
+
source_fields = None if fields is None else [key for key in selected if key not in shared]
|
| 1154 |
+
scoped = deps.assistant_read_scope(session, database, fields=source_fields, filters=filters,
|
| 1155 |
+
mode=mode, **kwargs)
|
| 1156 |
+
# Query schemas may omit declared fields that have no mirror expression.
|
| 1157 |
+
source_keys = {field["key"] for field in scoped["fields"]}
|
| 1158 |
+
available = [field for field in available if field["key"] in shared or field["key"] in source_keys]
|
| 1159 |
+
selected = deps._assistant_requested_fields(fields, available)
|
| 1160 |
+
overlay_keys = set(selected) & shared.keys()
|
| 1161 |
+
rows = scoped.get("records") or []
|
| 1162 |
+
cells = _external_shared_cells(session, database, rows) if overlay_keys else {}
|
| 1163 |
+
scoped["records"] = [{"pid": row["pid"], **{
|
| 1164 |
+
key: copy.deepcopy((cells.get(str(row["pid"])) or {}).get(key) if key in overlay_keys else row.get(key))
|
| 1165 |
+
for key in selected}} for row in rows]
|
| 1166 |
+
from core import if_field
|
| 1167 |
+
scoped["fields"] = if_field.project_fields(
|
| 1168 |
+
[copy.deepcopy(field) for field in available if field["key"] in selected], withheld)
|
| 1169 |
+
return scoped
|
| 1170 |
+
|
| 1171 |
+
|
| 1172 |
def _external_read_scope(session: Session, database, *args, **kwargs):
|
| 1173 |
"""Attach external refusal details at the shared Assistant read boundary."""
|
| 1174 |
try:
|
|
|
|
| 1178 |
if database == "ut_activity":
|
| 1179 |
scoped = _native_activity_scope(session, *args, **kwargs)
|
| 1180 |
else:
|
| 1181 |
+
scoped = _external_shared_read_scope(session, database, *args, **kwargs)
|
| 1182 |
if pin is not None and pin.source == database:
|
| 1183 |
mode = kwargs.get("mode")
|
| 1184 |
if scoped.get("source_kind") == "mirror_user_table":
|
|
|
|
| 1343 |
try:
|
| 1344 |
activity_log._backend(session.runtime)
|
| 1345 |
except (TypeError, ValueError) as exc:
|
| 1346 |
+
raise _query_source_unavailable("no_activity_store") from exc
|
| 1347 |
grid = routes_tables._compute_ut_assembly(session, "ut_activity", with_rows=False, fields_only=True)
|
| 1348 |
visible = list(grid["fields"])
|
| 1349 |
keys = {field["key"] for field in visible}
|
|
|
|
| 1990 |
return {"keys": keys, "can_mint": _can_use_analyst(session)}
|
| 1991 |
|
| 1992 |
|
| 1993 |
+
@router.post("/keys/verify")
|
| 1994 |
+
def verify_owned_key(response: Response, body=Body(default=None), session: Session = Depends(require_session)):
|
| 1995 |
+
"""Return a private diagnosis for this session owner's key without accepting bearer sign-in."""
|
| 1996 |
+
require_external_analyst(session)
|
| 1997 |
+
token = body.get("token") if isinstance(body, dict) else None
|
| 1998 |
+
if not isinstance(token, str) or len(token) > 1024:
|
| 1999 |
+
token = ""
|
| 2000 |
+
try:
|
| 2001 |
+
result = external_access.verify_for_owner(
|
| 2002 |
+
session.runtime, session.uname, session.tenant, token)
|
| 2003 |
+
except external_access.ExternalAccessError as exc:
|
| 2004 |
+
_management_refusal(exc)
|
| 2005 |
+
response.headers["Cache-Control"] = "no-store"
|
| 2006 |
+
return result
|
| 2007 |
+
|
| 2008 |
+
|
| 2009 |
@router.patch("/keys/{key_id}")
|
| 2010 |
def edit_key(key_id: str, body=Body(default=None), session: Session = Depends(require_session)):
|
| 2011 |
"""Edit metadata and narrowing scope without ever reading or replacing the secret."""
|
|
|
|
| 2782 |
if not message.endswith((".", "!", "?")):
|
| 2783 |
message += "."
|
| 2784 |
return {"ok": False, "error": {"code": code, "message": message}}
|
| 2785 |
+
except Exception as exc:
|
| 2786 |
+
refused = _query_failure(exc, "saved query").detail["error"]
|
| 2787 |
+
message = re.split(r"(?<=[.!?])\s+", refused["message"], maxsplit=1)[0]
|
| 2788 |
+
return {"ok": False, "error": {"code": refused["code"], "message": message}}
|
| 2789 |
|
| 2790 |
|
| 2791 |
def _query_database(session: Session, database: str, payload: dict):
|
|
|
|
| 2804 |
return result
|
| 2805 |
except HTTPException as exc:
|
| 2806 |
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 2807 |
+
cause = {"assistant_store_not_ready": deps._assistant_source_cause_of(info.get("message")) or "syncing",
|
| 2808 |
+
"store_not_ready": "syncing",
|
| 2809 |
+
"amazon_mirror_not_loaded": deps._assistant_source_cause_of(info.get("message")) or "loading"}.get(info.get("code"))
|
| 2810 |
+
if exc.status_code == 503 and cause:
|
| 2811 |
+
raise _query_source_unavailable(cause, message=info.get("message")) from exc
|
| 2812 |
if info.get("code") == "assistant_bad_filter":
|
| 2813 |
# S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
|
| 2814 |
info["message"] = bad_filter_message(info.get("message"))
|
|
|
|
| 2822 |
info["retry_query"] = {"database": database, **copy.deepcopy(payload)}
|
| 2823 |
info["retry_query"].pop("cursor", None)
|
| 2824 |
raise
|
| 2825 |
+
except Exception as exc:
|
| 2826 |
+
raise _query_failure(exc, "database query") from exc
|
| 2827 |
|
| 2828 |
|
| 2829 |
def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggregates, groups, limit, cursor):
|
|
|
|
| 2958 |
if identity is not None and (
|
| 2959 |
scoped.get("source_kind") != "runtime_cache"
|
| 2960 |
or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
|
| 2961 |
+
raise err(409, "assistant_snapshot_unavailable", deps._assistant_source_message("not_supported"))
|
|
|
|
| 2962 |
mirror = scoped.get("source_kind") == "mirror_user_table"
|
| 2963 |
visible_fields = list(scoped.get("fields") or [])
|
| 2964 |
visible_keys = {
|
|
|
|
| 3219 |
if scope.startswith("ut_"):
|
| 3220 |
from routes_tables import ut_write_ctx
|
| 3221 |
|
| 3222 |
+
record_pid = str(event.get("pid") or "").strip()
|
| 3223 |
+
g = ut_write_ctx(session, scope, pid=(
|
| 3224 |
+
int(record_pid) if record_pid.isdigit() and not event.get("pids") else None))
|
| 3225 |
if g.get("limits") and (event.get("pid") is not None or event.get("pids")):
|
| 3226 |
limit = g["limits"][0]
|
| 3227 |
raise err(409, "pid_scope_unresolved",
|
|
|
|
| 3327 |
if any(key not in visible for key in _view_referenced_fields(config)):
|
| 3328 |
raise err(400, "assistant_unknown_field",
|
| 3329 |
"Every requested field must be readable in this database.")
|
| 3330 |
+
# Saved predicates use Table's cleaner. The reader still validates permanent permission filters.
|
| 3331 |
+
_external_read_scope(session, database, fields=[], filters=None)
|
|
|
|
|
|
|
|
|
|
| 3332 |
|
| 3333 |
|
| 3334 |
def _saved_view_filters(config):
|
|
|
|
| 3337 |
_assert_query_filter_bounds([value] if isinstance(value, dict) else value)
|
| 3338 |
import aios_grid
|
| 3339 |
|
| 3340 |
+
def refuse(message=None):
|
| 3341 |
raise err(400, "view_filters_unsupported",
|
| 3342 |
+
message or "These filters cannot be saved without changing their meaning. "
|
| 3343 |
"Use colId, op and value leaves, with conj and children for nested groups.")
|
| 3344 |
|
| 3345 |
def node(raw):
|
|
|
|
| 3351 |
or not isinstance(raw[key], list) or not raw[key]):
|
| 3352 |
refuse()
|
| 3353 |
return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]}
|
| 3354 |
+
if any(key in raw for key in ("valueRef", "value2Ref")):
|
| 3355 |
+
refuse("Cell references cannot be saved through this tool because workbook and cell access "
|
| 3356 |
+
"cannot be verified here. Use literal values or a readable rhs field instead.")
|
| 3357 |
+
op = raw.get("op")
|
| 3358 |
+
if not isinstance(op, str) or op not in aios_grid.FILTER_OPS:
|
| 3359 |
+
name = json.dumps(op[:120]) if isinstance(op, str) else "(missing or invalid)"
|
| 3360 |
+
refuse(f"Unsupported filter operator {name}. Use a supported operator: "
|
| 3361 |
+
+ ", ".join(sorted(aios_grid.FILTER_OPS)) + ".")
|
| 3362 |
+
if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked",
|
| 3363 |
+
"dateWindow", "dateMode", "rhs"}
|
| 3364 |
+
or not isinstance(raw.get("colId"), str)):
|
| 3365 |
refuse()
|
| 3366 |
for key in ("value", "value2"):
|
| 3367 |
value = raw.get(key)
|
|
|
|
| 3799 |
from core import user_tables
|
| 3800 |
|
| 3801 |
guard = routes_tables._records_or_refuse
|
| 3802 |
+
# D-11174: the wall reads this one definition, never a lend of the whole bucket.
|
| 3803 |
+
definition = user_tables.get_definition(database, st=session.runtime)
|
| 3804 |
+
lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: (
|
| 3805 |
+
{database: definition} if isinstance(definition, dict) else {})})
|
| 3806 |
if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime):
|
| 3807 |
# The shared wall admits a locked database's update only when every key is an authored field.
|
| 3808 |
+
return guard(session, database, st=lent, cell_updates=dict.fromkeys(field_keys))
|
| 3809 |
+
return guard(session, database, st=lent)
|
| 3810 |
+
|
| 3811 |
+
|
| 3812 |
+
def _database_write_source(runtime):
|
| 3813 |
+
"""The runtime that owns the unstaged `user_tables` document beneath any preview layers."""
|
| 3814 |
+
from core import user_tables
|
| 3815 |
+
|
| 3816 |
+
source = runtime
|
| 3817 |
+
while isinstance(source, _PreviewRuntime) and user_tables.STORE_KEY not in source._docs:
|
| 3818 |
+
source = source._real
|
| 3819 |
+
return source
|
| 3820 |
+
|
| 3821 |
+
|
| 3822 |
+
def _database_write_tables(runtime, keys):
|
| 3823 |
+
"""D-11174: copy only the named tables of `user_tables`, never the whole bucket."""
|
| 3824 |
+
from core import user_tables
|
| 3825 |
+
|
| 3826 |
+
source = _database_write_source(runtime)
|
| 3827 |
+
reader = user_tables._entry_reader(source)
|
| 3828 |
+
if reader is None:
|
| 3829 |
+
# Only a store without an entry door reads the document; the plan still keeps the named tables.
|
| 3830 |
+
whole = source.get(user_tables.STORE_KEY)
|
| 3831 |
+
if not isinstance(whole, dict):
|
| 3832 |
+
raise err(503, "store_unavailable", "The database is unavailable. Nothing was saved.")
|
| 3833 |
+
reader = lambda _name, key: whole.get(key)
|
| 3834 |
+
tables = {}
|
| 3835 |
+
for key in dict.fromkeys(str(key) for key in keys if key):
|
| 3836 |
+
table = reader(user_tables.STORE_KEY, key)
|
| 3837 |
+
if isinstance(table, dict):
|
| 3838 |
+
tables[key] = copy.deepcopy(table)
|
| 3839 |
+
return tables
|
| 3840 |
+
|
| 3841 |
+
|
| 3842 |
+
def _seed_preview(preview, key, value, revision=None):
|
| 3843 |
+
"""Stage the known part of a document in a preview without reading the rest from its source."""
|
| 3844 |
+
preview._initial[key] = copy.deepcopy(value)
|
| 3845 |
+
preview._docs[key] = copy.deepcopy(value)
|
| 3846 |
+
preview._base_revisions[key] = copy.deepcopy(revision)
|
| 3847 |
+
preview._local_revisions[key] = 0
|
| 3848 |
+
|
| 3849 |
+
|
| 3850 |
+
def _database_row_maps():
|
| 3851 |
+
from core import user_table_rows
|
| 3852 |
+
return ("rows",) + tuple(user_table_rows.ROW_MAPS)
|
| 3853 |
+
|
| 3854 |
+
|
| 3855 |
+
def _database_table_delta(before, after):
|
| 3856 |
+
"""The definition keys and row-map entries that turn one table into the other, by name."""
|
| 3857 |
+
maps = _database_row_maps()
|
| 3858 |
+
before = before if isinstance(before, dict) else {}
|
| 3859 |
+
after = after if isinstance(after, dict) else {}
|
| 3860 |
+
definition, records = [], {}
|
| 3861 |
+
missing = object()
|
| 3862 |
+
for key in sorted(set(before) | set(after), key=str):
|
| 3863 |
+
old, new = before.get(key, missing), after.get(key, missing)
|
| 3864 |
+
if (key in maps and (old is missing or isinstance(old, dict))
|
| 3865 |
+
and (new is missing or isinstance(new, dict))):
|
| 3866 |
+
old = {} if old is missing else old
|
| 3867 |
+
new = {} if new is missing else new
|
| 3868 |
+
ids = sorted((rid for rid in set(old) | set(new)
|
| 3869 |
+
if old.get(rid, missing) != new.get(rid, missing)), key=str)
|
| 3870 |
+
if ids:
|
| 3871 |
+
records[str(key)] = [str(rid) for rid in ids]
|
| 3872 |
+
elif old != new:
|
| 3873 |
+
definition.append(str(key))
|
| 3874 |
+
return {"definition": definition, "records": records}
|
| 3875 |
+
|
| 3876 |
+
|
| 3877 |
+
def _database_table_guard(table, delta):
|
| 3878 |
+
"""One table's definition and the records a delta names: what a confirm must find unchanged."""
|
| 3879 |
+
if not isinstance(table, dict):
|
| 3880 |
+
return None
|
| 3881 |
+
maps = _database_row_maps()
|
| 3882 |
+
# A row map is read only at the ids the delta names, so a record-mode table stays per record.
|
| 3883 |
+
view = {"definition": {str(key): value for key, value in table.items()
|
| 3884 |
+
if not (key in maps and isinstance(value, dict))},
|
| 3885 |
+
"records": {}}
|
| 3886 |
+
for name, ids in delta["records"].items():
|
| 3887 |
+
rows = table.get(name)
|
| 3888 |
+
view["records"][name] = (None if not isinstance(rows, dict)
|
| 3889 |
+
else {rid: rows[rid] for rid in ids if rid in rows})
|
| 3890 |
+
return view
|
| 3891 |
+
|
| 3892 |
+
|
| 3893 |
+
def _database_tables_digest(tables, deltas):
|
| 3894 |
+
return _json_digest({key: _database_table_guard(tables.get(key), deltas[key])
|
| 3895 |
+
for key in sorted(deltas)})
|
| 3896 |
+
|
| 3897 |
+
|
| 3898 |
+
def _apply_database_table_delta(document, key, delta, after):
|
| 3899 |
+
"""Write one table's delta in place, so a row-mode store commits only those rows."""
|
| 3900 |
+
if not isinstance(after, dict):
|
| 3901 |
+
if key in document:
|
| 3902 |
+
del document[key]
|
| 3903 |
+
return
|
| 3904 |
+
table = document.get(key)
|
| 3905 |
+
if not isinstance(table, dict):
|
| 3906 |
+
document[key] = copy.deepcopy(after)
|
| 3907 |
+
return
|
| 3908 |
+
for name in delta["definition"]:
|
| 3909 |
+
if name in after:
|
| 3910 |
+
table[name] = copy.deepcopy(after[name])
|
| 3911 |
+
else:
|
| 3912 |
+
table.pop(name, None)
|
| 3913 |
+
for name, ids in delta["records"].items():
|
| 3914 |
+
new = after.get(name)
|
| 3915 |
+
if not isinstance(new, dict):
|
| 3916 |
+
table.pop(name, None)
|
| 3917 |
+
continue
|
| 3918 |
+
rows = table.get(name)
|
| 3919 |
+
if not isinstance(rows, dict):
|
| 3920 |
+
table[name] = {rid: copy.deepcopy(new[rid]) for rid in ids if rid in new}
|
| 3921 |
+
continue
|
| 3922 |
+
for rid in ids:
|
| 3923 |
+
if rid in new:
|
| 3924 |
+
rows[rid] = copy.deepcopy(new[rid])
|
| 3925 |
+
else:
|
| 3926 |
+
rows.pop(rid, None)
|
| 3927 |
+
|
| 3928 |
+
|
| 3929 |
+
def _database_write_plan(tables_before, tables_after, database):
|
| 3930 |
+
"""The per-table commit plan: deltas, the before digest, and the target's changed record ids."""
|
| 3931 |
+
deltas = {key: _database_table_delta(tables_before.get(key), tables_after.get(key))
|
| 3932 |
+
for key in sorted(set(tables_before) | set(tables_after))}
|
| 3933 |
+
row_ids = sorted({rid for ids in deltas.get(database, {"records": {}})["records"].values()
|
| 3934 |
+
for rid in ids}, key=str)
|
| 3935 |
+
# `bucket_after` keeps its name for existing readers; it now holds the planned tables only.
|
| 3936 |
+
return {"bucket_before_digest": _database_tables_digest(tables_before, deltas),
|
| 3937 |
+
"table_deltas": deltas, "bucket_after": tables_after, "row_ids": row_ids}
|
| 3938 |
+
|
| 3939 |
+
|
| 3940 |
+
def _plan_database_shared_update(session, database, record_id, raw_values, g, revision):
|
| 3941 |
+
"""D-11337: plan a record_update of shared fields on their overlay cell, or None for row fields."""
|
| 3942 |
+
from core import shared_overlay, user_tables
|
| 3943 |
+
import aios_grid
|
| 3944 |
+
|
| 3945 |
+
source = session.runtime
|
| 3946 |
+
while isinstance(source, _PreviewRuntime):
|
| 3947 |
+
source = source._real
|
| 3948 |
+
# Permission projection can normalize preview fields. The locked commit compares their stored definitions.
|
| 3949 |
+
definitions, stored_cells = shared_overlay.snapshot(database, [int(record_id)], st=source)
|
| 3950 |
+
keys = [str(key) for key in raw_values]
|
| 3951 |
+
shared = [key for key in keys if key in definitions]
|
| 3952 |
+
if not shared:
|
| 3953 |
+
return None
|
| 3954 |
+
if len(shared) != len(keys):
|
| 3955 |
+
raise err(400, "tool_arguments_invalid",
|
| 3956 |
+
"Change shared fields and other fields in separate calls.")
|
| 3957 |
+
# The grid's shared-cell door refuses an unresolved pid scope before any cell is read.
|
| 3958 |
+
if g.get("limits"):
|
| 3959 |
+
raise err(409, "pid_scope_unresolved",
|
| 3960 |
+
"The permitted record set could not be resolved. Nothing was saved.")
|
| 3961 |
+
by_key = {str(field.get("key")): field for field in g["fields"] if isinstance(field, dict)}
|
| 3962 |
+
from core import shares
|
| 3963 |
+
if not shares.may_edit("database", database, session.uname, is_admin=session.admin,
|
| 3964 |
+
st=session.runtime):
|
| 3965 |
+
raise err(403, "forbidden", "An edit share on this database is required to change a shared cell.")
|
| 3966 |
+
values = _database_value_plan(session, database, int(record_id), raw_values, g["fields"])
|
| 3967 |
+
for key in shared:
|
| 3968 |
+
# D-9735 item 3: `put_cells` stores any scalar, so a server-assigned kind refuses here.
|
| 3969 |
+
if any((field or {}).get("type") == "autonumber"
|
| 3970 |
+
or (field or {}).get("type") in user_tables.AUDIT_FIELD_TYPES
|
| 3971 |
+
for field in (definitions.get(key), by_key.get(key))):
|
| 3972 |
+
raise err(403, "computed_field", "Autonumber and record audit values are assigned by the server.")
|
| 3973 |
+
try:
|
| 3974 |
+
for value in values.values():
|
| 3975 |
+
shared_overlay._value(value)
|
| 3976 |
+
except ValueError as exc:
|
| 3977 |
+
raise err(400, "bad_value", str(exc)[:300]) from exc
|
| 3978 |
+
choices = {}
|
| 3979 |
+
for key, value in values.items():
|
| 3980 |
+
field = by_key.get(key) or definitions.get(key) or {}
|
| 3981 |
+
if field.get("type") in ("select", "multiselect") and value:
|
| 3982 |
+
known = {option.casefold() for option in aios_grid.merge_choice_options(
|
| 3983 |
+
(definitions.get(key) or {}).get("options") or field.get("options"))}
|
| 3984 |
+
parts = [value] if field["type"] == "select" else value.split(",")
|
| 3985 |
+
added = [part for part in parts if part.casefold() not in known]
|
| 3986 |
+
if added:
|
| 3987 |
+
choices[key] = added
|
| 3988 |
+
held = stored_cells.get(record_id) or {}
|
| 3989 |
+
before = {key: copy.deepcopy(held.get(key)) for key in values}
|
| 3990 |
+
changed, _ = _diff_documents({"databases/" + database: {
|
| 3991 |
+
"before": {"records": {record_id: before}}, "after": {"records": {record_id: values}}}})
|
| 3992 |
+
return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
|
| 3993 |
+
"warnings": [], "revision": {**revision, "shared_fields": _json_digest({
|
| 3994 |
+
key: definitions[key] for key in values})},
|
| 3995 |
+
"shared_cells": {"record_id": record_id, "before": before, "after": values,
|
| 3996 |
+
"fields": {key: copy.deepcopy(definitions[key]) for key in values},
|
| 3997 |
+
"choices": choices},
|
| 3998 |
+
"planned": {"database": database, "action": "record_update", "record_id": record_id}}
|
| 3999 |
+
|
| 4000 |
+
|
| 4001 |
+
def _commit_database_shared_update(session, database, shared):
|
| 4002 |
+
"""Write a planned shared-cell update after checking the cell still holds its previewed value."""
|
| 4003 |
+
from core import shared_overlay
|
| 4004 |
+
|
| 4005 |
+
record_id = str(shared.get("record_id") or "")
|
| 4006 |
+
before, after = shared.get("before"), shared.get("after")
|
| 4007 |
+
if not record_id.isdigit() or not isinstance(before, dict) or not isinstance(after, dict) or not after:
|
| 4008 |
+
raise err(503, "confirmation_revision_unavailable",
|
| 4009 |
+
"The database version could not be checked. Nothing was saved.")
|
| 4010 |
+
held = shared_overlay.cells(database, [int(record_id)], st=session.runtime).get(record_id) or {}
|
| 4011 |
+
if any(held.get(key) != value for key, value in before.items()):
|
| 4012 |
+
raise err(409, "confirmation_stale", "The database changed after the preview, " + _STALE_FIX)
|
| 4013 |
+
try:
|
| 4014 |
+
expected = {"cells": before, "fields": shared["fields"]}
|
| 4015 |
+
if shared.get("choices"):
|
| 4016 |
+
shared_overlay.put_cells_and_choice_options(
|
| 4017 |
+
database, int(record_id), after, shared["choices"], st=session.runtime, expected=expected)
|
| 4018 |
+
else:
|
| 4019 |
+
shared_overlay.put_cells(database, int(record_id), after, st=session.runtime, expected=expected)
|
| 4020 |
+
except ValueError as exc:
|
| 4021 |
+
raise err(400, getattr(exc, "code", None) or "bad_value", str(exc)[:300]) from exc
|
| 4022 |
+
|
| 4023 |
+
|
| 4024 |
+
def _field_link_tables(*fields):
|
| 4025 |
+
"""The tables a link field's reciprocal lives on, before and after a schema write."""
|
| 4026 |
+
tables = []
|
| 4027 |
+
for field in fields:
|
| 4028 |
+
bag = field.get("link") if isinstance(field, dict) else None
|
| 4029 |
+
if isinstance(bag, dict) and bag.get("table"):
|
| 4030 |
+
tables.append(str(bag["table"]))
|
| 4031 |
+
return tables
|
| 4032 |
|
| 4033 |
|
| 4034 |
def _plan_database_write(session, args):
|
|
|
|
| 4045 |
if action == "rename":
|
| 4046 |
return _plan_database_rename(session, args)
|
| 4047 |
routes_tables._managed_or_refuse(session, database, mutation=True)
|
| 4048 |
+
if action == "record_create":
|
| 4049 |
+
routes_tables._adds_or_refuse(database)
|
| 4050 |
+
# D-11174: a database write reads, copies and diffs only the tables it changes.
|
| 4051 |
+
tables_before = _database_write_tables(session.runtime, (database,))
|
| 4052 |
+
table = copy.deepcopy(tables_before.get(database) or {})
|
| 4053 |
if not table or not user_tables.is_user_table(database, st=session.runtime):
|
| 4054 |
raise err(403, "connected_source_read_only",
|
| 4055 |
"Records from connected sources are read only.")
|
| 4056 |
+
revision = _bucket_revision(_database_write_source(session.runtime), user_tables.STORE_KEY,
|
| 4057 |
+
tables_before)
|
| 4058 |
if type(revision.get("rev")) is not int:
|
| 4059 |
raise err(503, "confirmation_revision_unavailable",
|
| 4060 |
"The database version could not be checked. Nothing was saved.")
|
| 4061 |
warnings = []
|
|
|
|
| 4062 |
|
| 4063 |
if action.startswith("record_"):
|
| 4064 |
field_keys = None
|
|
|
|
| 4068 |
raise err(400, "tool_arguments_invalid", "Record values must be a non-empty object.")
|
| 4069 |
field_keys = tuple(str(key) for key in raw_values)
|
| 4070 |
_database_record_write_guard(session, database, field_keys=field_keys)
|
| 4071 |
+
record_pid = str(args.get("record_id") or "").strip()
|
| 4072 |
+
g = routes_tables.ut_write_ctx(session, database,
|
| 4073 |
+
pid=int(record_pid) if action in {"record_update", "record_delete"} and record_pid.isdigit() else None)
|
| 4074 |
rows = table.get("rows") or {}
|
| 4075 |
if action == "record_create":
|
| 4076 |
raw_values = args.get("values")
|
|
|
|
| 4082 |
record_id = str(args.get("record_id") or "").strip()
|
| 4083 |
if not record_id.isdigit() or int(record_id) not in g["pids"]:
|
| 4084 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
| 4085 |
+
if action == "record_update":
|
| 4086 |
+
shared_plan = _plan_database_shared_update(
|
| 4087 |
+
session, database, record_id, raw_values, g, revision)
|
| 4088 |
+
if shared_plan is not None:
|
| 4089 |
+
return shared_plan
|
| 4090 |
before = copy.deepcopy(rows.get(record_id))
|
| 4091 |
if before is None:
|
| 4092 |
raise err(403, "record_unavailable", "That record is not available to this account.")
|
|
|
|
| 4103 |
after[field["key"]] = ""
|
| 4104 |
else:
|
| 4105 |
after = None
|
| 4106 |
+
tables_after = copy.deepcopy(tables_before)
|
| 4107 |
+
target = tables_after[database]
|
| 4108 |
if action == "record_delete":
|
| 4109 |
(target.get("rows") or {}).pop(record_id, None)
|
| 4110 |
else:
|
|
|
|
| 4118 |
# This is part of the UI row writer's durable result, not an optional callback.
|
| 4119 |
# Include newly encountered select values in the preview and the one guarded commit.
|
| 4120 |
user_tables._hydrate_choice_options(target.get("fields") or [], [after])
|
| 4121 |
+
changed, truncated = _diff_documents(_database_preview_documents(session, tables_before, tables_after))
|
| 4122 |
if truncated:
|
| 4123 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 4124 |
return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
|
| 4125 |
"warnings": warnings, "revision": revision,
|
| 4126 |
+
**_database_write_plan(tables_before, tables_after, database),
|
|
|
|
| 4127 |
"planned": {"database": database, "action": action, "record_id": record_id}}
|
| 4128 |
|
| 4129 |
field_key = str(args.get("field_key") or "").strip()
|
|
|
|
| 4177 |
# Build the exact schema document through the core schema mechanics on an isolated runtime.
|
| 4178 |
# This includes reciprocal-link cleanup/creation while keeping preview free of relation
|
| 4179 |
# refreshes, audit calls, automation events and every other out-of-store side effect.
|
| 4180 |
+
# A link field's reciprocal lives on its target table, so that table joins the plan.
|
| 4181 |
+
tables_before.update(_database_write_tables(
|
| 4182 |
+
session.runtime, [key for key in _field_link_tables(before, after) if key not in tables_before]))
|
| 4183 |
preview = _PreviewRuntime(session.runtime)
|
| 4184 |
+
_seed_preview(preview, user_tables.STORE_KEY, tables_before, revision)
|
| 4185 |
if action == "field_create":
|
| 4186 |
stored = user_tables.add_field(
|
| 4187 |
database, after, st=preview, recurrence_validator=recurrence_validator)
|
|
|
|
| 4200 |
warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
|
| 4201 |
elif not user_tables.delete_field(database, field_key, st=preview):
|
| 4202 |
raise err(403, "field_refused", "That field cannot be removed.")
|
| 4203 |
+
tables_after = preview.get(user_tables.STORE_KEY)
|
| 4204 |
+
changed, truncated = _diff_documents(_database_preview_documents(session, tables_before, tables_after))
|
| 4205 |
if truncated:
|
| 4206 |
warnings.append("The preview diff reached its display cap; split this change.")
|
| 4207 |
affected_ids = [field_key]
|
|
|
|
| 4213 |
return {"diff": changed,
|
| 4214 |
"affected": {"ids": affected_ids, "count": len(affected_ids)},
|
| 4215 |
"warnings": warnings, "revision": revision,
|
| 4216 |
+
**_database_write_plan(tables_before, tables_after, database),
|
|
|
|
| 4217 |
"planned": {"database": database, "action": action, "field_key": field_key}}
|
| 4218 |
|
| 4219 |
|
|
|
|
| 4242 |
"The database metadata changed after the preview, " + _STALE_FIX) from exc
|
| 4243 |
return {"database": database, "action": action, "meta": result["meta"],
|
| 4244 |
"affected": copy.deepcopy(plan.get("affected") or {})}
|
| 4245 |
+
if plan.get("shared_cells") is not None:
|
| 4246 |
+
# D-11337: a shared field's value lives in the overlay, never in the table's rows.
|
| 4247 |
+
_commit_database_shared_update(session, database, plan["shared_cells"])
|
| 4248 |
+
try:
|
| 4249 |
+
import routes_tables
|
| 4250 |
+
routes_tables._refresh_relations(session)
|
| 4251 |
+
except Exception as exc: # noqa: BLE001
|
| 4252 |
+
print(f"[external] relation refresh deferred: {type(exc).__name__}: {exc}")
|
| 4253 |
+
return {"database": database, **copy.deepcopy(plan.get("planned") or {}),
|
| 4254 |
+
"affected": copy.deepcopy(plan.get("affected") or {})}
|
| 4255 |
expected_rev = plan.get("revision", {}).get("rev")
|
| 4256 |
expected_digest = plan.get("bucket_before_digest")
|
| 4257 |
replacement = copy.deepcopy(plan.get("bucket_after"))
|
| 4258 |
+
if type(expected_rev) is not int or not isinstance(replacement, dict) or not expected_digest:
|
| 4259 |
raise err(503, "confirmation_revision_unavailable",
|
| 4260 |
"The database version could not be checked. Nothing was saved.")
|
| 4261 |
+
deltas = copy.deepcopy(plan.get("table_deltas"))
|
| 4262 |
+
if not isinstance(deltas, dict):
|
| 4263 |
+
# A plan without deltas replaces its planned tables whole, under the same digest guard.
|
| 4264 |
+
deltas = {key: _database_table_delta(None, table) for key, table in replacement.items()}
|
| 4265 |
+
# The changed record ids keep a record-mode table per record; an older plan selects every record.
|
| 4266 |
+
row_ids = None if plan.get("row_ids") is None else list(plan["row_ids"])
|
| 4267 |
|
| 4268 |
def apply_exact(current):
|
| 4269 |
+
# D-11174: read and write only the planned tables, in place, so no whole document moves.
|
| 4270 |
+
if _database_tables_digest(current, deltas) != expected_digest:
|
| 4271 |
raise view_audit.ViewRevisionStale(
|
| 4272 |
rev_sent=expected_rev, rev_current=expected_rev + 1,
|
| 4273 |
message="The database changed after preview.")
|
| 4274 |
+
for key in sorted(deltas):
|
| 4275 |
+
_apply_database_table_delta(current, key, deltas[key], replacement.get(key))
|
| 4276 |
+
return current
|
| 4277 |
|
| 4278 |
try:
|
| 4279 |
+
# One guarded read-modify-write over the planned tables. The expected revision is checked
|
| 4280 |
# inside the PG lock where available; the closure's digest check also protects a backend
|
| 4281 |
# that can only provide the runtime's documented guarded fallback.
|
| 4282 |
user_tables.update_with_activity(
|
| 4283 |
database, apply_exact, st=session.runtime, by=session.uname,
|
| 4284 |
+
via=_trusted_activity_via(session), row_ids=row_ids,
|
| 4285 |
+
field_keys=None, flush="sync", expect_rev=expected_rev)
|
| 4286 |
except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
|
| 4287 |
raise err(409, "confirmation_stale",
|
| 4288 |
"The database changed after the preview, " + _STALE_FIX) from exc
|
|
|
|
| 4683 |
)
|
| 4684 |
|
| 4685 |
|
| 4686 |
+
def _is_members_write(name, arguments):
|
| 4687 |
+
"""A members permission write previews from the global account store, not the tenant runtime."""
|
| 4688 |
+
return name == "members" and isinstance(arguments, dict) and arguments.get("action") == "set_permissions"
|
| 4689 |
+
|
| 4690 |
+
|
| 4691 |
def _strong_confirmation_required(name, args):
|
| 4692 |
+
if _is_members_write(name, args):
|
| 4693 |
+
# A business unit widening needs the strong phrase in place of the editor's confirm_widen.
|
| 4694 |
+
return bool((args.get("arguments") or {}).get("confirm_widen"))
|
| 4695 |
if name not in external_tools.TOOLS:
|
| 4696 |
import tool_groups
|
| 4697 |
resolved = tool_groups.resolve(name, args)
|
|
|
|
| 4916 |
"Part of this change could not be accepted. Nothing was previewed.")
|
| 4917 |
|
| 4918 |
remote_note = _is_website_note(name, normalised)
|
| 4919 |
+
plan = result if name == "database_object" or remote_note or _is_members_write(name, normalised) else None
|
| 4920 |
if plan is not None:
|
| 4921 |
diff = copy.deepcopy(plan.get("diff") or [])
|
| 4922 |
warnings = list(plan.get("warnings") or [])
|
|
|
|
| 4949 |
normalised["arguments"]["expect"] = [
|
| 4950 |
{"user": row["user"], "role": row["role"]}
|
| 4951 |
for row in result["current_entries"]]
|
| 4952 |
+
if _is_members_write(name, normalised):
|
| 4953 |
+
# Pin the account state the preview read, and the widening it found, into the signed call.
|
| 4954 |
+
normalised["arguments"].update(expect=result["expect"], confirm_widen=bool(result["widening"]))
|
| 4955 |
|
| 4956 |
_canonical_json(diff, capped=False)
|
| 4957 |
diff_digest = _json_digest(diff)
|
|
|
|
| 5010 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 5011 |
if name == "apps" and normalised.get("action") == "catalog_build":
|
| 5012 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
| 5013 |
+
if _is_members_write(name, normalised):
|
| 5014 |
+
response.update({key: copy.deepcopy(result[key]) for key in ("username", "database", "before", "after")})
|
| 5015 |
if payload["strong"]:
|
| 5016 |
response["confirmation_phrase"] = _CONFIRMATION_PHRASE
|
| 5017 |
if issue_token:
|
|
|
|
| 5190 |
routes_nav._nav_meta_write_patch(_database_nav_body(args), session)
|
| 5191 |
elif name == "database_object":
|
| 5192 |
import routes_tables
|
|
|
|
| 5193 |
action = args["action"]
|
| 5194 |
+
record_pid = str(args.get("record_id") or "").strip()
|
| 5195 |
+
g = routes_tables.ut_write_ctx(session, args["database"],
|
| 5196 |
+
pid=int(record_pid) if action in {"record_update", "record_delete"} and record_pid.isdigit() else None)
|
| 5197 |
if action in {"record_create", "record_update"} and item["status"] == "committed":
|
| 5198 |
if any(int(rid) not in g["pids"] for rid in item["affected"]["ids"]):
|
| 5199 |
raise err(403, "operation_unavailable",
|
|
|
|
| 5549 |
if tool == "database_object":
|
| 5550 |
result = _execute_database_object(
|
| 5551 |
fresh_session, arguments, plan=recomputed.get("_plan"))
|
| 5552 |
+
elif _is_members_write(tool, arguments):
|
| 5553 |
+
# The account writer compares the pinned expect itself, so no runtime guard is needed.
|
| 5554 |
+
result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"])
|
| 5555 |
else:
|
| 5556 |
from datetime import datetime, timezone
|
| 5557 |
commit_started = datetime.now(timezone.utc)
|
|
|
|
| 5582 |
from core import user_tables
|
| 5583 |
import routes_nav
|
| 5584 |
bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
|
| 5585 |
+
# A failed revision read is replaced by the produced rev below, so no document is read here.
|
| 5586 |
+
after_revision = _bucket_revision(fresh_session.runtime, bucket, None)
|
| 5587 |
produced_rev = before_revision["rev"] + 1
|
| 5588 |
if after_revision.get("rev") != produced_rev:
|
| 5589 |
after_revision = {"rev": produced_rev}
|
| 5590 |
+
elif _is_members_write(tool, arguments):
|
| 5591 |
+
after_revision = copy.deepcopy(result["revision"])
|
| 5592 |
else:
|
| 5593 |
after_revision = copy.deepcopy(guarded_session.runtime._expected)
|
| 5594 |
after_revision_digest = _json_digest(after_revision)
|
|
|
|
| 5618 |
response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
|
| 5619 |
if tool == "apps" and arguments.get("action") == "catalog_build":
|
| 5620 |
response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
|
| 5621 |
+
if _is_members_write(tool, arguments):
|
| 5622 |
+
response.update(username=result["username"], database=result["database"],
|
| 5623 |
+
after=copy.deepcopy(result["entry"]))
|
| 5624 |
_confirmation_status(fresh_session, operation_id, "committed", response)
|
| 5625 |
if isinstance(stored.get("arguments_ref"), dict):
|
| 5626 |
_drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
|
| 5627 |
+
if (tool != "database_object" and not _is_members_write(tool, arguments)
|
| 5628 |
+
and receipt.get("persisted") is True):
|
| 5629 |
try:
|
| 5630 |
targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
|
| 5631 |
build_kind=payload["buildKind"])
|
|
|
|
| 5641 |
|
| 5642 |
# Codes are public identifiers. Reasons are product copy, never exception diagnostics.
|
| 5643 |
_MCP_CODE_GROUPS = (
|
| 5644 |
+
("Index the current permitted schema before querying its terms.", "semantic_unindexed"),
|
| 5645 |
+
("The schema or citation changed. Refresh its index and read the current citation.", "semantic_stale semantic_citation_stale"),
|
| 5646 |
+
("The schema is indexed but its execution connection is unavailable.", "semantic_execution_unavailable semantic_source_unavailable semantic_store_unavailable"),
|
| 5647 |
+
("Every requested field needs an approved current definition.", "semantic_not_approved"),
|
| 5648 |
+
("Use the existing Table query schema and its declared direct fields.", "semantic_query_operation"),
|
| 5649 |
+
("That personal memory is unavailable. Read your current personal facts.", "memory_not_found memory_stale personal_scope_unavailable"),
|
| 5650 |
+
("Send a bounded personal fact and its permitted source fields.", "memory_arguments_invalid"),
|
| 5651 |
("The arguments do not match this action. Read its guide and correct them.", """
|
| 5652 |
tool_arguments_invalid query_arguments_invalid query_aggregates_invalid query_aggregate_type
|
| 5653 |
assistant_bad_filter query_groups_invalid query_limit_invalid query_sorts_invalid query_value_invalid query_link_invalid
|
|
|
|
| 5764 |
"query_filter_limit": "Filters allow 3 levels, 100 nodes and 50 conditions per group to bound query validation work. To reduce that work, " + _MCP_FIXES["query_filter_limit"] + ".",
|
| 5765 |
"query_cursor_stale": "The query, data or permissions changed. To restart, " + _MCP_FIXES["query_cursor_stale"] + ".",
|
| 5766 |
"query_cursor_invalid": "The query cursor is invalid. Repeat the original query without cursor for a fresh first page.",
|
| 5767 |
+
"query_source_unavailable": "This database's source could not be read for this query.",
|
| 5768 |
"test_connection_production": "Test connections cannot write to production. Use a staging environment.",
|
| 5769 |
"test_connection_environment_unknown": "This environment is unknown. Test connections need a named non-production environment.",
|
| 5770 |
"external_test_connection_invalid": "Set test_connection to true or false.",
|
|
|
|
| 5798 |
r"Traceback|stack trace|\bengine\b|\bmanifest\b|\brevision-guard\b|\bgate\b|"
|
| 5799 |
r"\b(?:tenant store|dispatcher|document cleaner)\b|"
|
| 5800 |
r"(?:[A-Za-z]:[\\/]|/(?:tmp|root|work|home|srv|app)/)|"
|
| 5801 |
+
# A file suffix ends at a word boundary: the views action view.share is no shell script.
|
| 5802 |
+
r"(?:[\w.-]+[/\\])*[\w.-]+\.(?:py|tsx?|pyc|ps1|sh|json)\b(?::\d+)?|"
|
| 5803 |
r"\.(?:claude|agents|codex)/|~[/\\]", re.I)
|
| 5804 |
_MCP_PROSE_REPLACEMENTS = (
|
| 5805 |
(r"strict Assistant scope wall|scope wall|live UI walls|UI walls", "permissions"),
|
|
|
|
| 6185 |
scoped = SkillSession(session.tenant, session.user, session.claims, session.runtime)
|
| 6186 |
rows = [{"key": row["key"], "title": _markdown_inline(row["title"])}
|
| 6187 |
for row in loopy_skills.list_skills(scoped)]
|
| 6188 |
+
skill_keys = {row["key"] for row in rows}
|
| 6189 |
try:
|
| 6190 |
files = importlib.import_module("intelligence_store")
|
| 6191 |
except ModuleNotFoundError as exc:
|
|
|
|
| 6198 |
if exc.status_code != 403:
|
| 6199 |
raise
|
| 6200 |
return rows
|
| 6201 |
+
templates = getattr(files, "_template_catalog", lambda: {})()
|
| 6202 |
offset = 0
|
| 6203 |
while True:
|
| 6204 |
page = files.list_files(session, section="playbooks", offset=offset, limit=100)
|
| 6205 |
for row in page["files"]:
|
| 6206 |
+
# Publisher playbooks with empty reads still inherit their connector skill's scope.
|
| 6207 |
+
source = templates.get(row["path"], {}).get("templateSource")
|
| 6208 |
+
if source in loopy_skills.KNOWLEDGE_PACKS and source not in skill_keys:
|
| 6209 |
+
continue
|
| 6210 |
if row.get("fieldCheck", {}).get("state", "ready") == "ready":
|
| 6211 |
rows.append({"path": row["path"], "title": _markdown_inline(row["title"])})
|
| 6212 |
offset += len(page["files"])
|
|
|
|
| 6410 |
required.append("a named non-production environment for this test connection")
|
| 6411 |
if group == "websites":
|
| 6412 |
required.extend(_website_requirements(session, note=action == "add_internal_note"))
|
| 6413 |
+
if group == "members" and not deps.perms.is_admin(session.user):
|
| 6414 |
+
required.append("the Administrator role")
|
| 6415 |
if legacy in {"describe_database", "query_database", "view_upsert", "view_delete", "view_share_preview", "view_object", "app_upsert", "app_delete"} or (
|
| 6416 |
legacy == "database_object" and entry.get("legacy_action") != "list") or legacy == "sheet_set_binding":
|
| 6417 |
if not rows:
|
|
|
|
| 6512 |
return count if type(count) is int and count >= 0 else 0
|
| 6513 |
|
| 6514 |
|
| 6515 |
+
def _mcp_guide_group_note(requested, groups) -> str:
|
| 6516 |
+
"""Name an unknown guide group and the groups that exist at call time, in one line."""
|
| 6517 |
+
shown = requested.strip() if isinstance(requested, str) else ""
|
| 6518 |
+
# Only a short identifier is echoed: a credential-shaped name would withhold the whole guide.
|
| 6519 |
+
named = (f"No guide group is named `{shown}`."
|
| 6520 |
+
if re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]{0,59}", shown) and not _contains_credential(shown)
|
| 6521 |
+
else "No guide group has that name.")
|
| 6522 |
+
return (named + " The groups are " + ", ".join(f"`{name}`" for name in groups)
|
| 6523 |
+
+ ". Read one with group, or use the index below.")
|
| 6524 |
+
|
| 6525 |
+
|
| 6526 |
def _mcp_guide(session: Session, group=None) -> str:
|
| 6527 |
"""Return the group index or one group's action schemas without reading records."""
|
| 6528 |
+
try:
|
| 6529 |
+
return _mcp_guide_text(session, group)
|
| 6530 |
+
except HTTPException as exc:
|
| 6531 |
+
info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
|
| 6532 |
+
if exc.status_code in (401, 403, 404, 429) or exc.status_code >= 500 or info.get("code") in MCP_RESULT_CODES:
|
| 6533 |
+
raise
|
| 6534 |
+
except sheets.SheetRefusal as exc:
|
| 6535 |
+
if exc.code in MCP_RESULT_CODES:
|
| 6536 |
+
raise
|
| 6537 |
+
# An unpublished refusal from a guide read would collapse to tool_refused, which names a rejected change.
|
| 6538 |
+
raise err(503, "tool_unavailable", "The guide could not be read right now. Try again shortly.")
|
| 6539 |
+
|
| 6540 |
+
|
| 6541 |
+
def _mcp_guide_text(session: Session, group=None) -> str:
|
| 6542 |
+
"""Render the group index or one group's action schemas without reading records."""
|
| 6543 |
import tool_groups
|
| 6544 |
groups = tool_groups.groups()
|
| 6545 |
+
requested = group
|
| 6546 |
if isinstance(group, str) and group not in groups and group != "data":
|
| 6547 |
# A cased, plural or dotted name (Sheets, sheet.write_cells) reads the one group it names.
|
| 6548 |
key = group.strip().lower().partition(".")[0]
|
|
|
|
| 6550 |
if name in groups or name == "data"), group)
|
| 6551 |
alias = group == "data"
|
| 6552 |
group = tool_groups.canonical_name(group)
|
| 6553 |
+
note = None
|
| 6554 |
if group is not None and group not in groups:
|
| 6555 |
+
# An unknown group answers the index, led by one line naming it and the real groups.
|
| 6556 |
+
note, group = _mcp_guide_group_note(requested, groups), None
|
| 6557 |
# One rows-free definitions read per guide: a live runtime pays one pool checkout per database.
|
| 6558 |
# The guide reads schemas only, so no reader here needs `rows`. On a failed read, keep the old path.
|
| 6559 |
try:
|
|
|
|
| 6571 |
rows = [row for row in rows if row["permitted"]]
|
| 6572 |
capabilities = _mcp_capabilities(session, groups, [row for row in rows if row.get("schema")])
|
| 6573 |
unread = _guide_unread_count(session)
|
| 6574 |
+
lines = [*([note, ""] if note else []), "# Loopable guide", "", f"Signed in as {session.uname}.", "", MCP_INSTRUCTIONS, "",
|
| 6575 |
f"Scheduled results you have not read: {unread}.",
|
| 6576 |
*(["Read schedules.inbox before answering."] if unread > 0 else []), "",
|
| 6577 |
"Call a group with action and arguments. Each action keeps its own read or write rules.",
|
|
|
|
| 6688 |
"for supported actions, current permissions, schemas and confirmation rules.")
|
| 6689 |
if spec["name"] == "apps":
|
| 6690 |
spec["description"] += " Read apps.guide before creating or editing an App."
|
| 6691 |
+
if spec["name"] == "intelligence":
|
| 6692 |
+
spec["description"] += (" Recall private facts across clients with memory_recall. "
|
| 6693 |
+
"Send useful personal facts from this interaction in personal_context for automatic private capture. "
|
| 6694 |
+
"Use semantic_search for cited schema and Brain terms before semantic_query.")
|
| 6695 |
spec["inputSchema"]["properties"]["purpose"] = copy.deepcopy(wiki_log.PURPOSE_SCHEMA)
|
| 6696 |
spec["inputSchema"]["properties"]["about"] = copy.deepcopy(external_tools.ABOUT_SCHEMA)
|
| 6697 |
+
spec["inputSchema"]["properties"]["personal_context"] = {
|
| 6698 |
+
"type": "object", "description": "The person's own useful personal statement from this interaction. Optional automatic private memory.",
|
| 6699 |
+
"properties": {"text": {"type": "string", "minLength": 1, "maxLength": 500},
|
| 6700 |
+
"interaction_id": {"type": "string", "minLength": 1, "maxLength": 128}},
|
| 6701 |
+
"required": ["text", "interaction_id"], "additionalProperties": False}
|
| 6702 |
spec["inputSchema"]["properties"]["action"]["enum"] = list(groups[spec["name"]]["actions"])
|
| 6703 |
if spec["name"] == "sheet":
|
| 6704 |
from tool_groups.sheet import WORKBOOK_RESOURCE_URI, write_guide
|
|
|
|
| 6732 |
import tool_groups
|
| 6733 |
resolved = tool_groups.resolve(name, arguments)
|
| 6734 |
if resolved is None:
|
| 6735 |
+
raise sheets.SheetRefusal("tool_not_found", _unknown_tool_message(name))
|
| 6736 |
entry, args = resolved
|
| 6737 |
if call is not None and call["tool"] == name:
|
| 6738 |
external_activity.note_action(arguments["action"])
|
|
|
|
| 6747 |
return result
|
| 6748 |
|
| 6749 |
|
| 6750 |
+
def _unknown_tool_message(name):
|
| 6751 |
+
"""Name the group's real actions for a dotted name such as table.call, never echoing the caller's text."""
|
| 6752 |
+
import tool_groups
|
| 6753 |
+
prefix = name.partition(".")[0] if isinstance(name, str) else ""
|
| 6754 |
+
group = tool_groups.groups().get(tool_groups.canonical_name(prefix)) if prefix else None
|
| 6755 |
+
if group is None:
|
| 6756 |
+
return "The requested tool is not available."
|
| 6757 |
+
return ("That is not a tool name. Call the " + group["name"] + " tool and set action to one of the "
|
| 6758 |
+
+ group["name"] + " actions: " + ", ".join(group["actions"]) + ". Put that action's inputs "
|
| 6759 |
+
"inside arguments.")
|
| 6760 |
+
|
| 6761 |
+
|
| 6762 |
+
def _note_internal_code(out):
|
| 6763 |
+
"""Keep the refusal code before _mcp_public_code collapses it, for the limitation mailbox."""
|
| 6764 |
+
call = external_activity._CALL.get()
|
| 6765 |
+
if call is None or not isinstance(out, dict):
|
| 6766 |
+
return
|
| 6767 |
+
data = out.get("data") if isinstance(out.get("data"), dict) else {}
|
| 6768 |
+
error = out.get("error") if not out.get("ok") else data.get("error")
|
| 6769 |
+
error = error if isinstance(error, dict) else {}
|
| 6770 |
+
code = error.get("code") or (data.get("code") if out.get("ok") and data.get("ok") is False else "")
|
| 6771 |
+
call["internal_code"] = code if isinstance(code, str) else ""
|
| 6772 |
+
|
| 6773 |
+
|
| 6774 |
def _record_mcp_method(session, req, reply):
|
| 6775 |
"""Retain bounded method names for admitted protocol requests without request values."""
|
| 6776 |
try:
|
|
|
|
| 6829 |
code = error.get("code", "tool_refused") if error else ("tool_refused" if reply.get("error") else "")
|
| 6830 |
if code and not call["limitation_recorded"]:
|
| 6831 |
from routes_feedback import record_limitation
|
| 6832 |
+
# The reply's code is already public. The mailbox also keeps the one noted at dispatch.
|
| 6833 |
+
record_limitation(session, tool=limitation_tool, action=limitation_action, code=code,
|
| 6834 |
+
question=call["question"], internal_code=call.get("internal_code") or None)
|
| 6835 |
except Exception:
|
| 6836 |
pass
|
| 6837 |
finally:
|
|
|
|
| 7044 |
if code == "assistant_unknown_database":
|
| 7045 |
# The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
|
| 7046 |
return fix
|
| 7047 |
+
if code in {"assistant_snapshot_unavailable", "assistant_store_not_ready"} and deps._assistant_source_cause_of(text):
|
| 7048 |
+
# S82-T125: a named cause carries its own retry line, or none where no retry can succeed.
|
| 7049 |
+
return text
|
| 7050 |
return text if not fix or fix in text else (text + " " + fix).strip()
|
| 7051 |
|
| 7052 |
|
|
|
|
| 7118 |
if method != "tools/call":
|
| 7119 |
return protocol_error(rid, -32601, "The requested MCP method was not found.")
|
| 7120 |
arguments = _mcp_hoist_call_metadata(name, params.get("arguments", {}))
|
| 7121 |
+
personal_context = arguments.get("personal_context") if isinstance(arguments, dict) else None
|
| 7122 |
+
if isinstance(arguments, dict) and "personal_context" in arguments:
|
| 7123 |
+
arguments = {key: value for key, value in arguments.items() if key != "personal_context"}
|
| 7124 |
intent = _mcp_call_intent(arguments)
|
| 7125 |
if intent is None and name == "confirm_change" and isinstance(arguments, dict):
|
| 7126 |
intent = _mcp_saved_intent(session, arguments)
|
|
|
|
| 7215 |
out["error"]["data"] = data # Accepted edits remain visible even on partial refusal.
|
| 7216 |
else:
|
| 7217 |
out = {"ok": True, "data": data}
|
| 7218 |
+
context = personal_context
|
| 7219 |
+
if context is None and isinstance(params.get("_meta"), dict):
|
| 7220 |
+
context = params["_meta"].get("loopable/personal_context")
|
| 7221 |
+
if out.get("ok") and context is not None:
|
| 7222 |
+
import assistant_memory
|
| 7223 |
+
try:
|
| 7224 |
+
out["personal_memory"] = assistant_memory.capture_mcp_context(session, context)
|
| 7225 |
+
except Exception:
|
| 7226 |
+
# Optional capture must not turn a successful business read into a failed read.
|
| 7227 |
+
out["personal_memory"] = {"state": "unavailable"}
|
| 7228 |
if name == "sheet" and arguments.get("action") == "workbook.view" and out.get("ok"):
|
| 7229 |
book = data["workbook"]
|
| 7230 |
reply = _mcp_content(rid, {"ok": True, "data": {"workbook": {
|
|
|
|
| 7277 |
out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
|
| 7278 |
except (TypeError, ValueError, KeyError, AttributeError):
|
| 7279 |
out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
|
| 7280 |
+
except Exception as exc:
|
| 7281 |
+
deps._assistant_log_cause("external tool", exc)
|
| 7282 |
out = _tool_refusal(name, "tool_unavailable",
|
| 7283 |
"The tool could not complete this operation, so try the same call again shortly.", 503)
|
| 7284 |
+
_note_internal_code(out)
|
| 7285 |
return _mcp_content(rid, out, session)
|
| 7286 |
|
| 7287 |
|
|
|
|
| 7335 |
"database_object", "view_object", "workbook_object", "worksheet_object",
|
| 7336 |
"range_object", "calculation_object", "validated_batch",
|
| 7337 |
"confirm_change",
|
| 7338 |
+
# Dotted names declare every grouped write with no legacy tool.
|
| 7339 |
+
"apps.catalog_build", "apps.apply_template", "apps.save_code", "apps.set_status",
|
| 7340 |
+
"automations.create", "automations.pause", "automations.resume", "automations.delete",
|
| 7341 |
+
"intelligence.propose", "intelligence.create_wiki",
|
| 7342 |
+
"intelligence.memory_remember", "intelligence.memory_correct", "intelligence.memory_forget",
|
| 7343 |
+
"intelligence.semantic_ingest",
|
| 7344 |
+
"members.set_permissions",
|
| 7345 |
+
"requests.send_message", "requests.ask_user", "requests.send", "requests.answer", "requests.complete",
|
| 7346 |
+
"schedules.create", "schedules.pause", "schedules.resume", "schedules.delete",
|
| 7347 |
+
"sheet.worksheet.rename", "sheet.worksheet.reorder", "sheet.worksheet.duplicate", "sheet.refresh_scope",
|
| 7348 |
+
"views.share", "websites.add_internal_note",
|
| 7349 |
})
|
| 7350 |
|
| 7351 |
|
| 7352 |
def _assert_write_allow_list(*, check_groups=False) -> None:
|
| 7353 |
declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
|
| 7354 |
+
legacy = {name for name in _WRITE_TOOLS if "." not in name}
|
| 7355 |
+
undeclared = declared - legacy
|
| 7356 |
+
stale = legacy - declared
|
| 7357 |
if undeclared or stale:
|
| 7358 |
raise RuntimeError("External write allow-list mismatch: undeclared="
|
| 7359 |
+ repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
|
| 7360 |
if check_groups:
|
| 7361 |
import tool_groups
|
| 7362 |
+
found = tool_groups.groups()
|
| 7363 |
+
mapped = {entry.get("legacy_name") for group in found.values()
|
| 7364 |
for entry in group["actions"].values()}
|
| 7365 |
if set(external_tools.TOOLS) - mapped - {"confirm_change"}:
|
| 7366 |
raise RuntimeError("Every legacy tool must have a grouped action")
|
| 7367 |
+
standalone = _WRITE_TOOLS - legacy
|
| 7368 |
+
grouped = {group + "." + action for group, definition in found.items()
|
| 7369 |
+
for action, entry in definition["actions"].items()
|
| 7370 |
+
if entry.get("read") is False and not entry.get("legacy_name")}
|
| 7371 |
+
undeclared = grouped - standalone
|
| 7372 |
+
stale = standalone - grouped
|
| 7373 |
+
if undeclared or stale:
|
| 7374 |
+
raise RuntimeError("Grouped write allow-list mismatch: undeclared="
|
| 7375 |
+
+ repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
|
| 7376 |
|
| 7377 |
|
| 7378 |
_assert_write_allow_list()
|
api/routes_feedback.py
CHANGED
|
@@ -72,6 +72,9 @@ MAX_CHARS = 4000
|
|
| 72 |
#: the newest, which is the one direction this store must never move in.
|
| 73 |
MAX_ROWS = 5000
|
| 74 |
QUESTION_MAX = 500
|
|
|
|
|
|
|
|
|
|
| 75 |
|
| 76 |
|
| 77 |
def _limitation_address(tool, action):
|
|
@@ -97,19 +100,55 @@ def _limitation_address(tool, action):
|
|
| 97 |
return ("loopable_guide", "read") if tool == "loopable_guide" else (tool, action)
|
| 98 |
|
| 99 |
|
| 100 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
"""Coalesce value-free AI limitations in the existing platform feedback document."""
|
| 102 |
try:
|
| 103 |
# Protocol discovery belongs in method telemetry, outside the product limitation mailbox.
|
| 104 |
if code == "request_method_unknown":
|
| 105 |
return False
|
| 106 |
from core import audit, freeze_notice
|
| 107 |
-
from routes_external import _contains_credential, _mcp_public_code
|
| 108 |
if (freeze_notice.enabled() or getattr(session.runtime, "preview_only", False)
|
| 109 |
or not store.available()):
|
| 110 |
return False
|
| 111 |
tool, action = _limitation_address(tool, action)
|
|
|
|
| 112 |
code = _mcp_public_code(code)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
fallback = f"I could not complete {tool}.{action}."
|
| 114 |
# The caller supplies prose explicitly. Never derive it from arguments or results.
|
| 115 |
explicit = (isinstance(question, str) and 0 < len(question.strip()) <= QUESTION_MAX
|
|
@@ -128,7 +167,7 @@ def record_limitation(session, *, tool, action, code, question=None):
|
|
| 128 |
report = {"user": user, "ts": ts, "question": question}
|
| 129 |
identity = hashlib.sha256(json.dumps([tenant, tool, action, code]).encode()).hexdigest()
|
| 130 |
row = {"id": "lim_" + identity, "category": "limitation", "tenant": tenant,
|
| 131 |
-
"user": user, "tool": tool, "action": action, "code": code,
|
| 132 |
"question": question, "ts": ts, "firstTs": ts, "count": 1,
|
| 133 |
"reports": [report], "filers": {user: {"count": 1, "lastTs": ts}}}
|
| 134 |
|
|
@@ -145,7 +184,9 @@ def record_limitation(session, *, tool, action, code, question=None):
|
|
| 145 |
filers = {old["user"]: {"count": int(old.get("count", 1)), "lastTs": old["ts"]}}
|
| 146 |
filer = filers.get(user, {})
|
| 147 |
# Reports have no cap. The feedback document remains one JSON value under the 128 MB store ceiling.
|
| 148 |
-
|
|
|
|
|
|
|
| 149 |
"count": int(old.get("count", 1)) + 1, "reports": reports + [report],
|
| 150 |
"filers": {**filers, user: {"count": int(filer.get("count", 0)) + 1,
|
| 151 |
"lastTs": ts}}}
|
|
|
|
| 72 |
#: the newest, which is the one direction this store must never move in.
|
| 73 |
MAX_ROWS = 5000
|
| 74 |
QUESTION_MAX = 500
|
| 75 |
+
#: A report without a code borrows the filer's refusal at the same address from this recent window.
|
| 76 |
+
REFUSAL_WINDOW_SECONDS = 3600
|
| 77 |
+
_CODE = re.compile(r"[a-z][a-z0-9_]{0,99}")
|
| 78 |
|
| 79 |
|
| 80 |
def _limitation_address(tool, action):
|
|
|
|
| 100 |
return ("loopable_guide", "read") if tool == "loopable_guide" else (tool, action)
|
| 101 |
|
| 102 |
|
| 103 |
+
def refusal_code(session, tool, action, code=None):
|
| 104 |
+
"""Return (internal, public) of this filer's latest recent refusal at the address, or None."""
|
| 105 |
+
try:
|
| 106 |
+
import tool_groups
|
| 107 |
+
from routes_external import _mcp_public_code
|
| 108 |
+
tool, action = _limitation_address(tool, action)
|
| 109 |
+
grouped = tool in tool_groups.groups()
|
| 110 |
+
user, tenant = str(session.uname or "").lower(), str(session.tenant or "").lower()
|
| 111 |
+
public = _mcp_public_code(code) if code else None
|
| 112 |
+
since = time.time() - REFUSAL_WINDOW_SECONDS
|
| 113 |
+
best = {}
|
| 114 |
+
for row in _rows():
|
| 115 |
+
filers = row.get("filers") if isinstance(row.get("filers"), dict) else {}
|
| 116 |
+
filer = {str(name).lower(): value for name, value in filers.items()}.get(user)
|
| 117 |
+
stamp = filer.get("lastTs") if isinstance(filer, dict) else (
|
| 118 |
+
row.get("ts") if str(row.get("user", "")).lower() == user else None)
|
| 119 |
+
if (row.get("category") != "limitation" or str(row.get("tenant", "")).lower() != tenant
|
| 120 |
+
or row.get("tool") != tool or grouped and row.get("action") != action
|
| 121 |
+
or type(stamp) not in (int, float) or stamp < since
|
| 122 |
+
or row.get("code") in (None, "capability_unavailable")
|
| 123 |
+
or public is not None and row.get("code") != public):
|
| 124 |
+
continue
|
| 125 |
+
# A group matches its exact action. A single tool such as confirm_change prefers it.
|
| 126 |
+
rank = (row.get("action") == action, stamp)
|
| 127 |
+
if not best or rank > best["rank"]:
|
| 128 |
+
best = {"rank": rank, "found": (row.get("internal_code") or row["code"], row["code"])}
|
| 129 |
+
return best.get("found")
|
| 130 |
+
except Exception:
|
| 131 |
+
return None
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def record_limitation(session, *, tool, action, code, question=None, internal_code=None):
|
| 135 |
"""Coalesce value-free AI limitations in the existing platform feedback document."""
|
| 136 |
try:
|
| 137 |
# Protocol discovery belongs in method telemetry, outside the product limitation mailbox.
|
| 138 |
if code == "request_method_unknown":
|
| 139 |
return False
|
| 140 |
from core import audit, freeze_notice
|
| 141 |
+
from routes_external import MCP_RESULT_CODES, _contains_credential, _mcp_public_code
|
| 142 |
if (freeze_notice.enabled() or getattr(session.runtime, "preview_only", False)
|
| 143 |
or not store.available()):
|
| 144 |
return False
|
| 145 |
tool, action = _limitation_address(tool, action)
|
| 146 |
+
internal = internal_code if internal_code is not None else code
|
| 147 |
code = _mcp_public_code(code)
|
| 148 |
+
# The row keeps the refusal's own code beside the public one. The id stays over the public code.
|
| 149 |
+
if (not isinstance(internal, str) or not _CODE.fullmatch(internal)
|
| 150 |
+
or internal in MCP_RESULT_CODES and internal != code):
|
| 151 |
+
internal = code
|
| 152 |
fallback = f"I could not complete {tool}.{action}."
|
| 153 |
# The caller supplies prose explicitly. Never derive it from arguments or results.
|
| 154 |
explicit = (isinstance(question, str) and 0 < len(question.strip()) <= QUESTION_MAX
|
|
|
|
| 167 |
report = {"user": user, "ts": ts, "question": question}
|
| 168 |
identity = hashlib.sha256(json.dumps([tenant, tool, action, code]).encode()).hexdigest()
|
| 169 |
row = {"id": "lim_" + identity, "category": "limitation", "tenant": tenant,
|
| 170 |
+
"user": user, "tool": tool, "action": action, "code": code, "internal_code": internal,
|
| 171 |
"question": question, "ts": ts, "firstTs": ts, "count": 1,
|
| 172 |
"reports": [report], "filers": {user: {"count": 1, "lastTs": ts}}}
|
| 173 |
|
|
|
|
| 184 |
filers = {old["user"]: {"count": int(old.get("count", 1)), "lastTs": old["ts"]}}
|
| 185 |
filer = filers.get(user, {})
|
| 186 |
# Reports have no cap. The feedback document remains one JSON value under the 128 MB store ceiling.
|
| 187 |
+
# A bare public code never overwrites a refusal code the row already holds.
|
| 188 |
+
kept = internal if internal != code else old.get("internal_code") or internal
|
| 189 |
+
updated = {**old, "user": user, "ts": ts, "internal_code": kept,
|
| 190 |
"count": int(old.get("count", 1)) + 1, "reports": reports + [report],
|
| 191 |
"filers": {**filers, user: {"count": int(filer.get("count", 0)) + 1,
|
| 192 |
"lastTs": ts}}}
|
api/routes_forms.py
CHANGED
|
@@ -772,6 +772,11 @@ async def submit_form(token: str, request: Request):
|
|
| 772 |
if problem:
|
| 773 |
raise err(400, "invalid_submission", problem)
|
| 774 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 775 |
# ⚠ THE DAILY CAP IS SPENT ON A WRITE, NOT ON A REQUEST, and the order is the whole point.
|
| 776 |
# Counted before validation — where it was first — 500 malformed POSTs take a LIVE form
|
| 777 |
# offline for 24 hours, and one address can send them in about 17 minutes under the per-IP
|
|
|
|
| 772 |
if problem:
|
| 773 |
raise err(400, "invalid_submission", problem)
|
| 774 |
|
| 775 |
+
from core import module_tables
|
| 776 |
+
add_refusal = module_tables.records_add_refusal(table_key)
|
| 777 |
+
if add_refusal:
|
| 778 |
+
raise err(403, "records_add_refused", add_refusal)
|
| 779 |
+
|
| 780 |
# ⚠ THE DAILY CAP IS SPENT ON A WRITE, NOT ON A REQUEST, and the order is the whole point.
|
| 781 |
# Counted before validation — where it was first — 500 malformed POSTs take a LIVE form
|
| 782 |
# offline for 24 hours, and one address can send them in about 17 minutes under the per-IP
|
api/routes_grid.py
CHANGED
|
@@ -405,6 +405,38 @@ def _workspace_values(values, session, module, fields):
|
|
| 405 |
return {str(pid): cells(raw) for pid, raw in values.items()}
|
| 406 |
|
| 407 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 408 |
def _workspace_filter_removals(workspace, fields, measures, hidden=()):
|
| 409 |
"""Clean saved display filters on a detached wire copy, retaining removal reasons."""
|
| 410 |
from copy import deepcopy
|
|
@@ -534,6 +566,8 @@ def workspace(scope: str = "customer",
|
|
| 534 |
workspace["limits"] = g.get("limits") or []
|
| 535 |
if fields_only:
|
| 536 |
workspace["fieldsOnly"] = True
|
|
|
|
|
|
|
| 537 |
return {"workspace": _workspace_filter_removals(
|
| 538 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 539 |
|
|
@@ -588,12 +622,14 @@ def workspace(scope: str = "customer",
|
|
| 588 |
workspace["userAvatars"] = {}
|
| 589 |
workspace["scopeKey"] = scope
|
| 590 |
workspace["auditTimestamps"] = ({} if fields_only else
|
| 591 |
-
_audit_timestamps(scope, g["pids"],
|
| 592 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 593 |
if map_setup:
|
| 594 |
workspace["mapSetup"] = map_setup
|
| 595 |
if fields_only:
|
| 596 |
workspace["fieldsOnly"] = True
|
|
|
|
|
|
|
| 597 |
return {"workspace": _workspace_filter_removals(
|
| 598 |
workspace, g['fields'], g['measures'], g.get('_note_hidden') or ())}
|
| 599 |
|
|
@@ -698,6 +734,8 @@ def workspace(scope: str = "customer",
|
|
| 698 |
workspace["cohortMode"] = True
|
| 699 |
# The create pane offers "this cohort table only" (default) vs "all customer tables".
|
| 700 |
workspace["scopeChoice"] = True
|
|
|
|
|
|
|
| 701 |
return {"workspace": _workspace_filter_removals(
|
| 702 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 703 |
|
|
@@ -2076,7 +2114,7 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2076 |
Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
|
| 2077 |
same sessions the read door (`/workspace`) admits, or a tenant without the customer module
|
| 2078 |
can SEE its own user tables and not write to them."""
|
| 2079 |
-
from core import grid_events, shared_overlay
|
| 2080 |
from routes_customers import grid_assembly
|
| 2081 |
from routes_tables import _app_write_via, _managed_or_refuse
|
| 2082 |
|
|
@@ -2180,7 +2218,11 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2180 |
# answer "the batch was refused" and leave the caller to guess which member, which is
|
| 2181 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2182 |
_before = len(ctx.out.refusals)
|
| 2183 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2184 |
row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
|
| 2185 |
"rerender": bool(rerender)}
|
| 2186 |
mine = ctx.out.refusals[_before:]
|
|
|
|
| 405 |
return {str(pid): cells(raw) for pid, raw in values.items()}
|
| 406 |
|
| 407 |
|
| 408 |
+
def _published_rev(ws):
|
| 409 |
+
"""S82-T156: the store revision the workspace was read at, which a view write stamps."""
|
| 410 |
+
rev = ws.get("rev") if isinstance(ws, dict) else None
|
| 411 |
+
return rev if isinstance(rev, int) and not isinstance(rev, bool) else None
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def _settle_refused_view_write(ctx, event):
|
| 415 |
+
"""Store a no-op receipt for a refused stale view write, so its resend re-reads and stops.
|
| 416 |
+
|
| 417 |
+
The refusal rolls back the event's own receipt, and the outbox resends the same id. Without
|
| 418 |
+
this receipt every resend is refused again and the browser's save queue never moves (D-634).
|
| 419 |
+
"""
|
| 420 |
+
from core import edit_receipts, grid_events
|
| 421 |
+
|
| 422 |
+
if not getattr(ctx, "durable_receipts", False) or not isinstance(event, dict):
|
| 423 |
+
return
|
| 424 |
+
identity = event.get("opId", event.get("id"))
|
| 425 |
+
if not identity:
|
| 426 |
+
return
|
| 427 |
+
try:
|
| 428 |
+
identity = edit_receipts.op_id(identity)
|
| 429 |
+
dropped = {"rerender": True, "write": {"opId": identity}, "refusals": [],
|
| 430 |
+
"doc": None, "toast": None}
|
| 431 |
+
edit_receipts.run_once(
|
| 432 |
+
grid_events._store_of(ctx), ctx.uname,
|
| 433 |
+
edit_receipts.operation(ctx.scope_key, event.get("type"), event.get("pid")),
|
| 434 |
+
identity, lambda: dropped)
|
| 435 |
+
except Exception as exc: # noqa: BLE001
|
| 436 |
+
print(f"[grid/events] refused view write {identity!r}: resend receipt not saved "
|
| 437 |
+
f"({type(exc).__name__})")
|
| 438 |
+
|
| 439 |
+
|
| 440 |
def _workspace_filter_removals(workspace, fields, measures, hidden=()):
|
| 441 |
"""Clean saved display filters on a detached wire copy, retaining removal reasons."""
|
| 442 |
from copy import deepcopy
|
|
|
|
| 566 |
workspace["limits"] = g.get("limits") or []
|
| 567 |
if fields_only:
|
| 568 |
workspace["fieldsOnly"] = True
|
| 569 |
+
if _published_rev(g["ws"]) is not None:
|
| 570 |
+
workspace["rev"] = _published_rev(g["ws"])
|
| 571 |
return {"workspace": _workspace_filter_removals(
|
| 572 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 573 |
|
|
|
|
| 622 |
workspace["userAvatars"] = {}
|
| 623 |
workspace["scopeKey"] = scope
|
| 624 |
workspace["auditTimestamps"] = ({} if fields_only else
|
| 625 |
+
_audit_timestamps(scope, g["pids"], g.get('_note_hidden') or (), session.runtime))
|
| 626 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 627 |
if map_setup:
|
| 628 |
workspace["mapSetup"] = map_setup
|
| 629 |
if fields_only:
|
| 630 |
workspace["fieldsOnly"] = True
|
| 631 |
+
if _published_rev(g["ws"]) is not None:
|
| 632 |
+
workspace["rev"] = _published_rev(g["ws"])
|
| 633 |
return {"workspace": _workspace_filter_removals(
|
| 634 |
workspace, g['fields'], g['measures'], g.get('_note_hidden') or ())}
|
| 635 |
|
|
|
|
| 734 |
workspace["cohortMode"] = True
|
| 735 |
# The create pane offers "this cohort table only" (default) vs "all customer tables".
|
| 736 |
workspace["scopeChoice"] = True
|
| 737 |
+
if _published_rev(g["ws"]) is not None:
|
| 738 |
+
workspace["rev"] = _published_rev(g["ws"])
|
| 739 |
return {"workspace": _workspace_filter_removals(
|
| 740 |
workspace, g['fields'], g['measures'], g.get('hidden') or ())}
|
| 741 |
|
|
|
|
| 2114 |
Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
|
| 2115 |
same sessions the read door (`/workspace`) admits, or a tenant without the customer module
|
| 2116 |
can SEE its own user tables and not write to them."""
|
| 2117 |
+
from core import grid_events, shared_overlay, view_audit
|
| 2118 |
from routes_customers import grid_assembly
|
| 2119 |
from routes_tables import _app_write_via, _managed_or_refuse
|
| 2120 |
|
|
|
|
| 2218 |
# answer "the batch was refused" and leave the caller to guess which member, which is
|
| 2219 |
# the same class of unfalsifiable answer the channel exists to end.
|
| 2220 |
_before = len(ctx.out.refusals)
|
| 2221 |
+
try:
|
| 2222 |
+
rerender = grid_events.handle_one(one, ctx)
|
| 2223 |
+
except view_audit.ViewRevisionStale:
|
| 2224 |
+
_settle_refused_view_write(ctx, one)
|
| 2225 |
+
raise
|
| 2226 |
row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
|
| 2227 |
"rerender": bool(rerender)}
|
| 2228 |
mine = ctx.out.refusals[_before:]
|
api/routes_keychain.py
CHANGED
|
@@ -12,7 +12,7 @@ the keychain cutover wave R3 staged.
|
|
| 12 |
import os
|
| 13 |
import time
|
| 14 |
|
| 15 |
-
from fastapi import Body, Depends
|
| 16 |
from fastapi import APIRouter
|
| 17 |
|
| 18 |
from deps import Session, err, require_session
|
|
@@ -1066,6 +1066,8 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 1066 |
raise err(503, "store_unavailable",
|
| 1067 |
"the key was not saved; its sharing setting could not be stored, so "
|
| 1068 |
"nothing was kept. Try again.")
|
|
|
|
|
|
|
| 1069 |
return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
|
| 1070 |
|
| 1071 |
|
|
@@ -1275,8 +1277,18 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
|
|
| 1275 |
|
| 1276 |
@router.post("/admin/keychain/{entry_id}/test")
|
| 1277 |
def test_key(entry_id: str, session: Session = Depends(require_session)):
|
| 1278 |
-
_own_row(session, entry_id)
|
| 1279 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1280 |
|
| 1281 |
|
| 1282 |
def _unsynced_customer_records(session):
|
|
@@ -1486,6 +1498,144 @@ def _odoo_admin(session):
|
|
| 1486 |
"configure it")
|
| 1487 |
|
| 1488 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1489 |
@router.get("/admin/connectors/odoo/config")
|
| 1490 |
def odoo_config(session: Session = Depends(require_session)):
|
| 1491 |
"""Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a
|
|
@@ -1495,7 +1645,16 @@ def odoo_config(session: Session = Depends(require_session)):
|
|
| 1495 |
entry = _odoo_entry(session)
|
| 1496 |
server_db, server_url, api_user, editable = _odoo_source_fields(session, entry)
|
| 1497 |
cfg = rel.read_config(session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1498 |
return {
|
|
|
|
|
|
|
|
|
|
| 1499 |
"applicable": bool(rel.is_royal(session.tenant)),
|
| 1500 |
"source": "keychain" if entry else ("env" if env_odoo_available(session.runtime)
|
| 1501 |
else "none"),
|
|
@@ -1520,6 +1679,74 @@ def odoo_config(session: Session = Depends(require_session)):
|
|
| 1520 |
}
|
| 1521 |
|
| 1522 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1523 |
@router.put("/admin/connectors/odoo/config")
|
| 1524 |
def odoo_config_put(body: dict = Body(default=None),
|
| 1525 |
session: Session = Depends(require_session)):
|
|
@@ -1584,12 +1811,22 @@ def odoo_config_put(body: dict = Body(default=None),
|
|
| 1584 |
raise err(400, "missing_sync_tz",
|
| 1585 |
"choose a time zone for the daily sync time")
|
| 1586 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1587 |
server_db = body.get("serverDb")
|
| 1588 |
if server_db is not None:
|
| 1589 |
server_db = " ".join(str(server_db).split())[:80]
|
| 1590 |
|
| 1591 |
def _up(cur):
|
| 1592 |
cur = cur if isinstance(cur, dict) else {}
|
|
|
|
|
|
|
|
|
|
| 1593 |
if clean_grids is not None:
|
| 1594 |
cur.setdefault("grids", {}).update(clean_grids)
|
| 1595 |
if clean_every is not None:
|
|
@@ -1619,7 +1856,7 @@ def odoo_config_put(body: dict = Body(default=None),
|
|
| 1619 |
notes.append(f"the daily time {saved['syncAt']} {saved['syncTz']} is saved and "
|
| 1620 |
f"applies when the sync interval is daily")
|
| 1621 |
else:
|
| 1622 |
-
notes.append(f"the Odoo sync runs once a day at {saved['syncAt']} {saved['syncTz']}")
|
| 1623 |
|
| 1624 |
# ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE
|
| 1625 |
# PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed:
|
|
@@ -1944,6 +2181,48 @@ def pause_connector(key: str, body: dict = Body(default=None),
|
|
| 1944 |
return {"key": key, "paused": paused, "snapshots": snapshots}
|
| 1945 |
|
| 1946 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1947 |
from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
|
| 1948 |
|
| 1949 |
router.include_router(amazon_deletion_router)
|
|
|
|
| 12 |
import os
|
| 13 |
import time
|
| 14 |
|
| 15 |
+
from fastapi import Body, Depends, HTTPException
|
| 16 |
from fastapi import APIRouter
|
| 17 |
|
| 18 |
from deps import Session, err, require_session
|
|
|
|
| 1066 |
raise err(503, "store_unavailable",
|
| 1067 |
"the key was not saved; its sharing setting could not be stored, so "
|
| 1068 |
"nothing was kept. Try again.")
|
| 1069 |
+
if etype == "odoo":
|
| 1070 |
+
_business_tz_on_connect(session) # S82-T107: default the zone on connect
|
| 1071 |
return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
|
| 1072 |
|
| 1073 |
|
|
|
|
| 1277 |
|
| 1278 |
@router.post("/admin/keychain/{entry_id}/test")
|
| 1279 |
def test_key(entry_id: str, session: Session = Depends(require_session)):
|
| 1280 |
+
row = _own_row(session, entry_id) # 404 for an entry this caller may not see
|
| 1281 |
+
result = _kc().test_entry(session.runtime, entry_id)
|
| 1282 |
+
# S82-T107: a passing test of the connection that serves this tenant re-defaults the zone.
|
| 1283 |
+
if (isinstance(result, dict) and result.get("ok") and str(row.get("type") or "") == "odoo"
|
| 1284 |
+
and session.admin):
|
| 1285 |
+
try:
|
| 1286 |
+
serving = _odoo_entry(session)
|
| 1287 |
+
except Exception: # noqa: BLE001
|
| 1288 |
+
serving = None
|
| 1289 |
+
if serving is not None and str(serving.get("id")) == str(entry_id):
|
| 1290 |
+
_business_tz_on_connect(session)
|
| 1291 |
+
return result
|
| 1292 |
|
| 1293 |
|
| 1294 |
def _unsynced_customer_records(session):
|
|
|
|
| 1498 |
"configure it")
|
| 1499 |
|
| 1500 |
|
| 1501 |
+
# ═════════════════════════════════ S82-T107: THE TENANT'S BUSINESS TIMEZONE (`businessTz`, C10)
|
| 1502 |
+
#
|
| 1503 |
+
# One zone per tenant, stored beside the sync slot in the Odoo connector config. The business day
|
| 1504 |
+
# reads it. It is NEVER `syncTz`: that zone says when the sync runs, not where the business is.
|
| 1505 |
+
# It defaults from the connected Odoo user's own `tz` on connect, on test and on the first admin
|
| 1506 |
+
# config read. A blank Odoo `tz` leaves it unset, and the panel asks the admin to pick one. An
|
| 1507 |
+
# admin's pick wins over every later Odoo default.
|
| 1508 |
+
BUSINESS_TZ_KEY = "businessTz"
|
| 1509 |
+
#: Where the stored zone came from: "odoo", "admin", or "odoo-blank" (Odoo was asked, said nothing).
|
| 1510 |
+
BUSINESS_TZ_SOURCE_KEY = "businessTzSource"
|
| 1511 |
+
|
| 1512 |
+
|
| 1513 |
+
def _zone_ok(name):
|
| 1514 |
+
"""True when `name` is an IANA zone this process can load."""
|
| 1515 |
+
if not isinstance(name, str) or not name.strip():
|
| 1516 |
+
return False
|
| 1517 |
+
try:
|
| 1518 |
+
from zoneinfo import ZoneInfo
|
| 1519 |
+
ZoneInfo(name.strip())
|
| 1520 |
+
return True
|
| 1521 |
+
except Exception: # noqa: BLE001
|
| 1522 |
+
return False
|
| 1523 |
+
|
| 1524 |
+
|
| 1525 |
+
def _odoo_cfg(rt):
|
| 1526 |
+
try:
|
| 1527 |
+
cur = rt.get(_rel().CONFIG_KEY) if rt is not None else None
|
| 1528 |
+
except Exception: # noqa: BLE001
|
| 1529 |
+
cur = None
|
| 1530 |
+
return cur if isinstance(cur, dict) else {}
|
| 1531 |
+
|
| 1532 |
+
|
| 1533 |
+
def stored_business_tz(rt):
|
| 1534 |
+
"""The tenant's stored business timezone, or None. Reads `businessTz` only, never `syncTz`."""
|
| 1535 |
+
name = str(_odoo_cfg(rt).get(BUSINESS_TZ_KEY) or "").strip()
|
| 1536 |
+
return name if _zone_ok(name) else None
|
| 1537 |
+
|
| 1538 |
+
|
| 1539 |
+
def odoo_user_tz(creds, timeout=8):
|
| 1540 |
+
"""The connected Odoo user's `tz` from `res.users`.
|
| 1541 |
+
|
| 1542 |
+
Returns the zone name, "" when Odoo holds none (or one this process cannot load), and None when
|
| 1543 |
+
Odoo could not be asked. One admin act over read-only XML-RPC: `authenticate`, then one `read`
|
| 1544 |
+
of the user's own row. It goes outside every limiter and the shared client on purpose."""
|
| 1545 |
+
import xmlrpc.client
|
| 1546 |
+
creds = creds or {}
|
| 1547 |
+
url = str(creds.get("url") or "").strip().rstrip("/")
|
| 1548 |
+
db, user, key = (str(creds.get(k) or "") for k in ("db", "user", "api_key"))
|
| 1549 |
+
if not (url.startswith(("http://", "https://")) and db and user and key):
|
| 1550 |
+
return None
|
| 1551 |
+
base = xmlrpc.client.SafeTransport if url.startswith("https") else xmlrpc.client.Transport
|
| 1552 |
+
|
| 1553 |
+
class _Timed(base):
|
| 1554 |
+
def make_connection(self, host):
|
| 1555 |
+
conn = super().make_connection(host)
|
| 1556 |
+
conn.timeout = timeout
|
| 1557 |
+
return conn
|
| 1558 |
+
|
| 1559 |
+
try:
|
| 1560 |
+
common = xmlrpc.client.ServerProxy(f"{url}/xmlrpc/2/common", transport=_Timed(),
|
| 1561 |
+
allow_none=True)
|
| 1562 |
+
uid = common.authenticate(db, user, key, {})
|
| 1563 |
+
if not uid:
|
| 1564 |
+
return None
|
| 1565 |
+
models = xmlrpc.client.ServerProxy(f"{url}/xmlrpc/2/object", transport=_Timed(),
|
| 1566 |
+
allow_none=True)
|
| 1567 |
+
rows = models.execute_kw(db, uid, key, "res.users", "read", [[uid]], {"fields": ["tz"]})
|
| 1568 |
+
except Exception: # noqa: BLE001
|
| 1569 |
+
return None
|
| 1570 |
+
first = rows[0] if isinstance(rows, list) and rows and isinstance(rows[0], dict) else {}
|
| 1571 |
+
zone = str(first.get("tz") or "").strip() # Odoo sends False for a blank field
|
| 1572 |
+
return zone if _zone_ok(zone) else ""
|
| 1573 |
+
|
| 1574 |
+
|
| 1575 |
+
def _odoo_tz_creds(session, entry):
|
| 1576 |
+
"""The fields of the connection serving this tenant's Odoo, or None. Never returned to a client."""
|
| 1577 |
+
if entry is None:
|
| 1578 |
+
if not env_odoo_available(session.runtime):
|
| 1579 |
+
return None
|
| 1580 |
+
got, _why = _env_odoo_fields()
|
| 1581 |
+
return got
|
| 1582 |
+
try:
|
| 1583 |
+
return _kc().read_fields(session.runtime, entry["id"]) or None
|
| 1584 |
+
except Exception: # noqa: BLE001
|
| 1585 |
+
return None
|
| 1586 |
+
|
| 1587 |
+
|
| 1588 |
+
def default_business_tz(session, entry=None, refresh=False):
|
| 1589 |
+
"""Default `businessTz` from the connected Odoo user's `tz`, and return the zone in force.
|
| 1590 |
+
|
| 1591 |
+
`refresh=False` (the admin config read) asks Odoo only when no zone is stored and Odoo was not
|
| 1592 |
+
already asked. `refresh=True` (connect, test) asks again. An admin's pick is never replaced, a
|
| 1593 |
+
blank answer never clears a stored zone, and the default is never `syncTz` nor a literal."""
|
| 1594 |
+
rt = session.runtime
|
| 1595 |
+
cur = _odoo_cfg(rt)
|
| 1596 |
+
have = stored_business_tz(rt)
|
| 1597 |
+
source = str(cur.get(BUSINESS_TZ_SOURCE_KEY) or "")
|
| 1598 |
+
if have and source == "admin":
|
| 1599 |
+
return have
|
| 1600 |
+
if not refresh and (have or source == "odoo-blank"):
|
| 1601 |
+
return have
|
| 1602 |
+
creds = _odoo_tz_creds(session, entry)
|
| 1603 |
+
if not creds:
|
| 1604 |
+
return have
|
| 1605 |
+
zone = odoo_user_tz(creds)
|
| 1606 |
+
if zone is None or (not zone and have):
|
| 1607 |
+
return have
|
| 1608 |
+
|
| 1609 |
+
def _up(c):
|
| 1610 |
+
c = c if isinstance(c, dict) else {}
|
| 1611 |
+
if str(c.get(BUSINESS_TZ_SOURCE_KEY) or "") == "admin" and _zone_ok(c.get(BUSINESS_TZ_KEY)):
|
| 1612 |
+
return c # an admin picked a zone meanwhile
|
| 1613 |
+
if zone:
|
| 1614 |
+
c[BUSINESS_TZ_KEY] = zone
|
| 1615 |
+
c[BUSINESS_TZ_SOURCE_KEY] = "odoo"
|
| 1616 |
+
else:
|
| 1617 |
+
c.pop(BUSINESS_TZ_KEY, None)
|
| 1618 |
+
c[BUSINESS_TZ_SOURCE_KEY] = "odoo-blank"
|
| 1619 |
+
return c
|
| 1620 |
+
|
| 1621 |
+
try:
|
| 1622 |
+
rt.update(_rel().CONFIG_KEY, _up, flush="sync")
|
| 1623 |
+
except Exception: # noqa: BLE001
|
| 1624 |
+
return have
|
| 1625 |
+
return stored_business_tz(rt)
|
| 1626 |
+
|
| 1627 |
+
|
| 1628 |
+
def _business_tz_on_connect(session):
|
| 1629 |
+
"""Connect and test: re-default the zone from the serving Odoo user. Never raises."""
|
| 1630 |
+
try:
|
| 1631 |
+
entry = _odoo_entry(session)
|
| 1632 |
+
if entry is None and not env_odoo_available(session.runtime):
|
| 1633 |
+
return None
|
| 1634 |
+
return default_business_tz(session, entry, refresh=True)
|
| 1635 |
+
except Exception: # noqa: BLE001
|
| 1636 |
+
return None
|
| 1637 |
+
|
| 1638 |
+
|
| 1639 |
@router.get("/admin/connectors/odoo/config")
|
| 1640 |
def odoo_config(session: Session = Depends(require_session)):
|
| 1641 |
"""Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a
|
|
|
|
| 1645 |
entry = _odoo_entry(session)
|
| 1646 |
server_db, server_url, api_user, editable = _odoo_source_fields(session, entry)
|
| 1647 |
cfg = rel.read_config(session.runtime)
|
| 1648 |
+
# S82-T107: the first admin read of a connection with no stored zone takes the Odoo user's tz.
|
| 1649 |
+
business_tz = cfg.get(BUSINESS_TZ_KEY) if _zone_ok(cfg.get(BUSINESS_TZ_KEY)) else None
|
| 1650 |
+
if business_tz is None:
|
| 1651 |
+
live = not cfg["frozen"] and (entry is not None or env_odoo_available(session.runtime))
|
| 1652 |
+
business_tz = (default_business_tz(session, entry) if live
|
| 1653 |
+
else stored_business_tz(session.runtime))
|
| 1654 |
return {
|
| 1655 |
+
"businessTz": business_tz,
|
| 1656 |
+
"businessTzSource": str(_odoo_cfg(session.runtime).get(BUSINESS_TZ_SOURCE_KEY) or "")
|
| 1657 |
+
if business_tz else "",
|
| 1658 |
"applicable": bool(rel.is_royal(session.tenant)),
|
| 1659 |
"source": "keychain" if entry else ("env" if env_odoo_available(session.runtime)
|
| 1660 |
else "none"),
|
|
|
|
| 1679 |
}
|
| 1680 |
|
| 1681 |
|
| 1682 |
+
def _odoo_field_name_tables(session):
|
| 1683 |
+
"""Read only this connector's visible database definitions in the session's tenant."""
|
| 1684 |
+
from core import perm_scope, user_tables
|
| 1685 |
+
tables = {}
|
| 1686 |
+
for grid in _rel().grid_choices(session.runtime):
|
| 1687 |
+
key = grid["key"]
|
| 1688 |
+
if not perm_scope.may_read(session.user, key, st=session.runtime):
|
| 1689 |
+
continue
|
| 1690 |
+
definition = user_tables.get_one(key, st=session.runtime)
|
| 1691 |
+
if definition is not None:
|
| 1692 |
+
tables[key] = (definition, str(definition.get("name") or grid["label"]))
|
| 1693 |
+
return tables
|
| 1694 |
+
|
| 1695 |
+
|
| 1696 |
+
@router.get("/admin/connectors/odoo/field-names")
|
| 1697 |
+
def odoo_field_names(session: Session = Depends(require_session)):
|
| 1698 |
+
"""List the connector fields this administrator may rename without changing their definitions."""
|
| 1699 |
+
_odoo_admin(session)
|
| 1700 |
+
from core import perm_scope, user_tables
|
| 1701 |
+
try:
|
| 1702 |
+
fields = []
|
| 1703 |
+
for key, (definition, name) in _odoo_field_name_tables(session).items():
|
| 1704 |
+
hidden = perm_scope.hidden_keys(session.user, key, definition.get("fields") or [],
|
| 1705 |
+
st=session.runtime)
|
| 1706 |
+
for field in user_tables.renamable_fields([key], st=session.runtime):
|
| 1707 |
+
if field["key"] not in hidden and user_tables.may_rename_field(
|
| 1708 |
+
key, field["key"], session.uname, session.admin, st=session.runtime):
|
| 1709 |
+
fields.append({**field, "tableLabel": name})
|
| 1710 |
+
return {"fields": fields}
|
| 1711 |
+
except HTTPException:
|
| 1712 |
+
raise
|
| 1713 |
+
except Exception:
|
| 1714 |
+
raise err(503, "store_unavailable", "Field names could not be read. Try again.") from None
|
| 1715 |
+
|
| 1716 |
+
|
| 1717 |
+
@router.put("/admin/connectors/odoo/field-names")
|
| 1718 |
+
def odoo_field_names_put(body: dict = Body(default=None),
|
| 1719 |
+
session: Session = Depends(require_session)):
|
| 1720 |
+
"""Rename one connector field through the shared label-only authority."""
|
| 1721 |
+
_odoo_admin(session)
|
| 1722 |
+
from core import perm_scope, user_tables
|
| 1723 |
+
body = body or {}
|
| 1724 |
+
if not all(isinstance(body.get(key), str) and body[key].strip() for key in ("table", "key", "label")):
|
| 1725 |
+
raise err(400, "invalid_field_name", "Choose a database, column and nonblank field name.")
|
| 1726 |
+
table_key, fkey = body["table"], body["key"]
|
| 1727 |
+
try:
|
| 1728 |
+
tables = _odoo_field_name_tables(session)
|
| 1729 |
+
if table_key not in tables:
|
| 1730 |
+
raise err(404, "not_found", "This connector has no database by that name you may edit.")
|
| 1731 |
+
definition, name = tables[table_key]
|
| 1732 |
+
if fkey in perm_scope.hidden_keys(session.user, table_key, definition.get("fields") or [],
|
| 1733 |
+
st=session.runtime):
|
| 1734 |
+
raise err(403, "forbidden", "this database has no column by that name that you may edit")
|
| 1735 |
+
raw = {key: value for key, value in body.items() if key not in ("table", "key")}
|
| 1736 |
+
if not user_tables.may_rename_field(table_key, fkey, session.uname, session.admin,
|
| 1737 |
+
st=session.runtime, raw=raw):
|
| 1738 |
+
raise err(403, "preset_field_locked", "Only the name of a renamable pre-set column may change here.")
|
| 1739 |
+
saved = user_tables.rename_field(table_key, fkey, body["label"], session.uname,
|
| 1740 |
+
session.admin, st=session.runtime, by=session.uname, via="web")
|
| 1741 |
+
if saved is None:
|
| 1742 |
+
raise err(503, "store_unavailable", "The field name was not saved. Try again.")
|
| 1743 |
+
return {"field": {"table": table_key, "key": fkey, "label": saved["label"], "tableLabel": name}}
|
| 1744 |
+
except HTTPException:
|
| 1745 |
+
raise
|
| 1746 |
+
except Exception:
|
| 1747 |
+
raise err(503, "store_unavailable", "The field name was not saved. Try again.") from None
|
| 1748 |
+
|
| 1749 |
+
|
| 1750 |
@router.put("/admin/connectors/odoo/config")
|
| 1751 |
def odoo_config_put(body: dict = Body(default=None),
|
| 1752 |
session: Session = Depends(require_session)):
|
|
|
|
| 1811 |
raise err(400, "missing_sync_tz",
|
| 1812 |
"choose a time zone for the daily sync time")
|
| 1813 |
|
| 1814 |
+
# S82-T107: the business timezone. An admin's pick, kept apart from the sync slot zone.
|
| 1815 |
+
clean_business_tz = None
|
| 1816 |
+
if BUSINESS_TZ_KEY in body:
|
| 1817 |
+
clean_business_tz = str(body.get(BUSINESS_TZ_KEY) or "").strip()
|
| 1818 |
+
if not _zone_ok(clean_business_tz):
|
| 1819 |
+
raise err(400, "invalid_business_tz", "choose a business timezone from the list")
|
| 1820 |
+
|
| 1821 |
server_db = body.get("serverDb")
|
| 1822 |
if server_db is not None:
|
| 1823 |
server_db = " ".join(str(server_db).split())[:80]
|
| 1824 |
|
| 1825 |
def _up(cur):
|
| 1826 |
cur = cur if isinstance(cur, dict) else {}
|
| 1827 |
+
if clean_business_tz:
|
| 1828 |
+
cur[BUSINESS_TZ_KEY] = clean_business_tz
|
| 1829 |
+
cur[BUSINESS_TZ_SOURCE_KEY] = "admin"
|
| 1830 |
if clean_grids is not None:
|
| 1831 |
cur.setdefault("grids", {}).update(clean_grids)
|
| 1832 |
if clean_every is not None:
|
|
|
|
| 1856 |
notes.append(f"the daily time {saved['syncAt']} {saved['syncTz']} is saved and "
|
| 1857 |
f"applies when the sync interval is daily")
|
| 1858 |
else:
|
| 1859 |
+
notes.append(f"the Odoo sync runs once a day at {rel.slot_run_clock(saved['syncAt'])} {saved['syncTz']}")
|
| 1860 |
|
| 1861 |
# ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE
|
| 1862 |
# PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed:
|
|
|
|
| 2181 |
return {"key": key, "paused": paused, "snapshots": snapshots}
|
| 2182 |
|
| 2183 |
|
| 2184 |
+
def _live_limits_admin(session):
|
| 2185 |
+
"""S82-T101: live call limits are workspace wide, so only an administrator reads or sets them."""
|
| 2186 |
+
if not session.admin:
|
| 2187 |
+
raise err(403, "not_admin", "only an administrator can see or change live call limits")
|
| 2188 |
+
|
| 2189 |
+
|
| 2190 |
+
def _live_limits_payload(limits):
|
| 2191 |
+
from core import live_limiter
|
| 2192 |
+
return {**limits, "defaults": live_limiter.defaults(),
|
| 2193 |
+
"callsPerMinuteMin": live_limiter.CALLS_PER_MINUTE_MIN,
|
| 2194 |
+
"callsPerMinuteMax": live_limiter.CALLS_PER_MINUTE_MAX}
|
| 2195 |
+
|
| 2196 |
+
|
| 2197 |
+
@router.get("/admin/connectors/live-limits")
|
| 2198 |
+
def live_limits(session: Session = Depends(require_session)):
|
| 2199 |
+
"""S82-T101 (R65): the tenant's AI and metric live call limits."""
|
| 2200 |
+
_live_limits_admin(session)
|
| 2201 |
+
from core import live_limiter
|
| 2202 |
+
try:
|
| 2203 |
+
limits = live_limiter.read_limits(session.runtime)
|
| 2204 |
+
except Exception:
|
| 2205 |
+
raise err(503, "store_unavailable", "the live call limits could not be read; try again")
|
| 2206 |
+
return _live_limits_payload(limits)
|
| 2207 |
+
|
| 2208 |
+
|
| 2209 |
+
@router.put("/admin/connectors/live-limits")
|
| 2210 |
+
def live_limits_put(body: dict = Body(default=None),
|
| 2211 |
+
session: Session = Depends(require_session)):
|
| 2212 |
+
"""S82-T101 (R65): set calls per minute, one in flight, and max rows per call (null is no limit)."""
|
| 2213 |
+
_live_limits_admin(session)
|
| 2214 |
+
from core import live_limiter
|
| 2215 |
+
clean, errors = live_limiter.validate_limits(body if body is not None else {})
|
| 2216 |
+
if errors:
|
| 2217 |
+
raise err(400, "invalid_live_limits", "some live call limits are not valid",
|
| 2218 |
+
details={"fields": errors})
|
| 2219 |
+
try:
|
| 2220 |
+
limits = live_limiter.write_limits(session.runtime, clean, by=session.uname)
|
| 2221 |
+
except Exception:
|
| 2222 |
+
raise err(503, "store_unavailable", "the change was not saved; try again")
|
| 2223 |
+
return _live_limits_payload(limits)
|
| 2224 |
+
|
| 2225 |
+
|
| 2226 |
from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
|
| 2227 |
|
| 2228 |
router.include_router(amazon_deletion_router)
|
api/routes_nav.py
CHANGED
|
@@ -596,18 +596,18 @@ def _granted_view_ids(session):
|
|
| 596 |
#: ⚠ COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one
|
| 597 |
#: of the reserved words below is still a second connection, and its sibling must still be named.
|
| 598 |
#:
|
| 599 |
-
#: These
|
| 600 |
#: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
|
| 601 |
-
#: empty into the one
|
| 602 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 603 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 604 |
-
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
|
| 605 |
|
| 606 |
-
#: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "
|
| 607 |
#: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
|
| 608 |
CUSTOM_SOURCE_GROUP = "custom"
|
| 609 |
-
#: Native provenance tokens
|
| 610 |
-
_NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation"})
|
| 611 |
|
| 612 |
|
| 613 |
def _native_group(group):
|
|
@@ -1165,7 +1165,7 @@ def _user_pref_now():
|
|
| 1165 |
|
| 1166 |
@router.get("/nav/prefs/{key}")
|
| 1167 |
def get_user_pref(key: str, session: Session = Depends(require_session)):
|
| 1168 |
-
"""C5: this user's stored value for one preference key
|
| 1169 |
key = _user_pref_key(key)
|
| 1170 |
try:
|
| 1171 |
bucket = (session.runtime.get(_NAV_USER_PREFS_KEY) or {}).get(session.uname) or {}
|
|
@@ -1173,7 +1173,7 @@ def get_user_pref(key: str, session: Session = Depends(require_session)):
|
|
| 1173 |
raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
|
| 1174 |
row = bucket.get(key) if isinstance(bucket, dict) else None
|
| 1175 |
if not isinstance(row, dict) or "value" not in row:
|
| 1176 |
-
|
| 1177 |
return {"key": key, "value": row.get("value"), "updated_at": row.get("updated_at")}
|
| 1178 |
|
| 1179 |
|
|
@@ -1427,6 +1427,10 @@ def nav_schema(key: str, session: Session = Depends(require_session)):
|
|
| 1427 |
"means": means.get(f["key"], "")}
|
| 1428 |
if f.get("options"):
|
| 1429 |
entry["options"] = list(f["options"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1430 |
fields.append(entry)
|
| 1431 |
except Exception:
|
| 1432 |
fields = []
|
|
|
|
| 596 |
#: ⚠ COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one
|
| 597 |
#: of the reserved words below is still a second connection, and its sibling must still be named.
|
| 598 |
#:
|
| 599 |
+
#: These reserved words are the rail's OWN vocabulary, not a connection's to take:
|
| 600 |
#: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
|
| 601 |
+
#: `system` / empty into the one System group (S79-T60, key `CUSTOM_SOURCE_GROUP`). A connection someone named
|
| 602 |
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 603 |
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 604 |
+
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "system", "odoo"})
|
| 605 |
|
| 606 |
+
#: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "System";
|
| 607 |
#: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
|
| 608 |
CUSTOM_SOURCE_GROUP = "custom"
|
| 609 |
+
#: Native provenance tokens fold into the key. Must equal `dataPath.ts::CUSTOM_SOURCE_TOKENS`.
|
| 610 |
+
_NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation", "system"})
|
| 611 |
|
| 612 |
|
| 613 |
def _native_group(group):
|
|
|
|
| 1165 |
|
| 1166 |
@router.get("/nav/prefs/{key}")
|
| 1167 |
def get_user_pref(key: str, session: Session = Depends(require_session)):
|
| 1168 |
+
"""C5: this user's stored value for one preference key; an unset key answers 200 with null."""
|
| 1169 |
key = _user_pref_key(key)
|
| 1170 |
try:
|
| 1171 |
bucket = (session.runtime.get(_NAV_USER_PREFS_KEY) or {}).get(session.uname) or {}
|
|
|
|
| 1173 |
raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
|
| 1174 |
row = bucket.get(key) if isinstance(bucket, dict) else None
|
| 1175 |
if not isinstance(row, dict) or "value" not in row:
|
| 1176 |
+
return {"key": key, "value": None, "updated_at": None}
|
| 1177 |
return {"key": key, "value": row.get("value"), "updated_at": row.get("updated_at")}
|
| 1178 |
|
| 1179 |
|
|
|
|
| 1427 |
"means": means.get(f["key"], "")}
|
| 1428 |
if f.get("options"):
|
| 1429 |
entry["options"] = list(f["options"])
|
| 1430 |
+
elif f["key"] == "dba":
|
| 1431 |
+
# The dba vocabulary is the tenant's teams plus one multi-team value, never a contract list.
|
| 1432 |
+
from modules.customer_data import dba_options
|
| 1433 |
+
entry["options"] = dba_options()
|
| 1434 |
fields.append(entry)
|
| 1435 |
except Exception:
|
| 1436 |
fields = []
|