Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +5 -449
- VERSION +1 -1
- api/agent_tools.py +20 -0
- api/amazon_relational.py +39 -7
- api/automation_engine.py +4 -0
- api/automation_worker.py +3 -1
- api/connector_onboarding.py +124 -0
- api/formula_runtime.py +5 -2
- api/main.py +82 -12
- api/odoo_purchasing.py +188 -0
- api/odoo_relational.py +463 -24
- api/routes_activity.py +3 -0
- api/routes_admin.py +90 -37
- api/routes_alerts.py +9 -7
- api/routes_assets.py +11 -4
- api/routes_auth.py +2 -2
- api/routes_connectors.py +339 -36
- api/routes_customers.py +98 -28
- api/routes_external.py +38 -45
- api/routes_forms.py +179 -1
- api/routes_grid.py +70 -21
- api/routes_keychain.py +35 -12
- api/routes_nav.py +10 -3
- api/routes_odoo_tables.py +83 -78
- api/routes_products.py +26 -18
- api/routes_sheet.py +323 -63
- api/routes_tables.py +25 -11
- api/telnyx_whatsapp.py +105 -0
- api/test_odoo_rank_refresh.py +7 -3
- api/whatsapp_door.py +12 -2
- api/whatsapp_identity.py +13 -0
- platform/aios_grid.py +6 -1
- platform/core/audit.py +3 -0
- platform/core/data_binding.py +2 -0
- platform/core/external_oauth.py +82 -14
- platform/core/field_permissions.py +26 -1
- platform/core/freeze_notice.py +49 -0
- platform/core/grid_events.py +12 -2
- platform/core/keychain.py +38 -10
- platform/core/route_system.py +7 -1
- platform/core/setup_links.py +162 -0
- platform/core/shared_overlay.py +28 -3
- platform/core/sheet_import.py +55 -12
- platform/core/sheets.py +82 -37
- platform/core/store.py +33 -0
- platform/core/store_backend.py +5 -0
- platform/core/store_pg.py +14 -13
- platform/core/table_store.py +6 -2
- platform/core/user_tables.py +54 -14
- platform/core/users.py +23 -1
RELEASES.json
CHANGED
|
@@ -1,455 +1,11 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
-
"version": "
|
| 6 |
-
"sha": "
|
| 7 |
-
"date": "2026-10-
|
| 8 |
-
"subject": "release
|
| 9 |
-
},
|
| 10 |
-
{
|
| 11 |
-
"version": "v75",
|
| 12 |
-
"sha": "2c5146981c",
|
| 13 |
-
"date": "2026-09-19",
|
| 14 |
-
"subject": "release v75"
|
| 15 |
-
},
|
| 16 |
-
{
|
| 17 |
-
"version": "v74",
|
| 18 |
-
"sha": "40dceba926",
|
| 19 |
-
"date": "2026-09-14",
|
| 20 |
-
"subject": "release v74: waves 58+59 incl. the Amazon credential door (W59-T21/22/23) and sync producer (W59-T42); hotfix promote so"
|
| 21 |
-
},
|
| 22 |
-
{
|
| 23 |
-
"version": "v73",
|
| 24 |
-
"sha": "bb94de5665",
|
| 25 |
-
"date": "2026-09-13",
|
| 26 |
-
"subject": "release v73"
|
| 27 |
-
},
|
| 28 |
-
{
|
| 29 |
-
"version": "v71",
|
| 30 |
-
"sha": "96143dc6fb",
|
| 31 |
-
"date": "2026-09-04",
|
| 32 |
-
"subject": "release v71 - wave 51, QA'd 24 PASS / 6 FAIL / 5 UNPROVEN"
|
| 33 |
-
},
|
| 34 |
-
{
|
| 35 |
-
"version": "v70",
|
| 36 |
-
"sha": "eedfb6a001",
|
| 37 |
-
"date": "2026-09-03",
|
| 38 |
-
"subject": "release v70 - wave 50 (grid analytics, the Sheet as a spreadsheet, view trash and audit)"
|
| 39 |
-
},
|
| 40 |
-
{
|
| 41 |
-
"version": "v69",
|
| 42 |
-
"sha": "8598f56293",
|
| 43 |
-
"date": "2026-09-02",
|
| 44 |
-
"subject": "release v69: wave48-continuation validated + hotfixes v67/v68/d4e3e31 + go-live QA repairs (Supplier option limit, optio"
|
| 45 |
-
},
|
| 46 |
-
{
|
| 47 |
-
"version": "v68",
|
| 48 |
-
"sha": "62b065a261",
|
| 49 |
-
"date": "2026-09-01",
|
| 50 |
-
"subject": "release v68: unified holiday date offsets"
|
| 51 |
-
},
|
| 52 |
-
{
|
| 53 |
-
"version": "v67",
|
| 54 |
-
"sha": "3b2241982c",
|
| 55 |
-
"date": "2026-09-01",
|
| 56 |
-
"subject": "release v67"
|
| 57 |
-
},
|
| 58 |
-
{
|
| 59 |
-
"version": "v66",
|
| 60 |
-
"sha": "73c94f0767",
|
| 61 |
-
"date": "2026-08-31",
|
| 62 |
-
"subject": "release v66"
|
| 63 |
-
},
|
| 64 |
-
{
|
| 65 |
-
"version": "v65",
|
| 66 |
-
"sha": "9f6cfedb8a",
|
| 67 |
-
"date": "2026-08-31",
|
| 68 |
-
"subject": "release v65"
|
| 69 |
-
},
|
| 70 |
-
{
|
| 71 |
-
"version": "v64",
|
| 72 |
-
"sha": "bb8985c6e7",
|
| 73 |
-
"date": "2026-08-31",
|
| 74 |
-
"subject": "release v64"
|
| 75 |
-
},
|
| 76 |
-
{
|
| 77 |
-
"version": "v63",
|
| 78 |
-
"sha": "ec75cb8f34",
|
| 79 |
-
"date": "2026-08-31",
|
| 80 |
-
"subject": "release v63"
|
| 81 |
-
},
|
| 82 |
-
{
|
| 83 |
-
"version": "v62",
|
| 84 |
-
"sha": "2c8f702c96",
|
| 85 |
-
"date": "2026-08-31",
|
| 86 |
-
"subject": "release v62"
|
| 87 |
-
},
|
| 88 |
-
{
|
| 89 |
-
"version": "v61",
|
| 90 |
-
"sha": "bbc4c0760a",
|
| 91 |
-
"date": "2026-08-30",
|
| 92 |
-
"subject": "v61 - wave 46 hotfix: D-590 schema overlay reaches the grid read (all three assemblies), D-599 route to zone association"
|
| 93 |
-
},
|
| 94 |
-
{
|
| 95 |
-
"version": "v60",
|
| 96 |
-
"sha": "dd1b062021",
|
| 97 |
-
"date": "2026-08-30",
|
| 98 |
-
"subject": "v60 - wave 46"
|
| 99 |
-
},
|
| 100 |
-
{
|
| 101 |
-
"version": "v59",
|
| 102 |
-
"sha": "3cb73ce802",
|
| 103 |
-
"date": "2026-08-29",
|
| 104 |
-
"subject": "release v59"
|
| 105 |
-
},
|
| 106 |
-
{
|
| 107 |
-
"version": "v58",
|
| 108 |
-
"sha": "c526cede82",
|
| 109 |
-
"date": "2026-08-29",
|
| 110 |
-
"subject": "release v58"
|
| 111 |
-
},
|
| 112 |
-
{
|
| 113 |
-
"version": "v57",
|
| 114 |
-
"sha": "4859c035e6",
|
| 115 |
-
"date": "2026-08-29",
|
| 116 |
-
"subject": "release v57"
|
| 117 |
-
},
|
| 118 |
-
{
|
| 119 |
-
"version": "v56",
|
| 120 |
-
"sha": "2970b729e8",
|
| 121 |
-
"date": "2026-08-28",
|
| 122 |
-
"subject": "release v56"
|
| 123 |
-
},
|
| 124 |
-
{
|
| 125 |
-
"version": "v55",
|
| 126 |
-
"sha": "b766b904b8",
|
| 127 |
-
"date": "2026-08-28",
|
| 128 |
-
"subject": "release v55"
|
| 129 |
-
},
|
| 130 |
-
{
|
| 131 |
-
"version": "v54",
|
| 132 |
-
"sha": "6a8e99a064",
|
| 133 |
-
"date": "2026-08-27",
|
| 134 |
-
"subject": "release v54"
|
| 135 |
-
},
|
| 136 |
-
{
|
| 137 |
-
"version": "v53",
|
| 138 |
-
"sha": "89ed2cb867",
|
| 139 |
-
"date": "2026-08-24",
|
| 140 |
-
"subject": "release v53"
|
| 141 |
-
},
|
| 142 |
-
{
|
| 143 |
-
"version": "v52",
|
| 144 |
-
"sha": "8263493762",
|
| 145 |
-
"date": "2026-08-23",
|
| 146 |
-
"subject": "release v52"
|
| 147 |
-
},
|
| 148 |
-
{
|
| 149 |
-
"version": "v51",
|
| 150 |
-
"sha": "5740d20fa5",
|
| 151 |
-
"date": "2026-08-23",
|
| 152 |
-
"subject": "release v51"
|
| 153 |
-
},
|
| 154 |
-
{
|
| 155 |
-
"version": "v50",
|
| 156 |
-
"sha": "8027c83d1d",
|
| 157 |
-
"date": "2026-08-23",
|
| 158 |
-
"subject": "release v50"
|
| 159 |
-
},
|
| 160 |
-
{
|
| 161 |
-
"version": "v49",
|
| 162 |
-
"sha": "9a14c01a48",
|
| 163 |
-
"date": "2026-08-23",
|
| 164 |
-
"subject": "release v49"
|
| 165 |
-
},
|
| 166 |
-
{
|
| 167 |
-
"version": "v48",
|
| 168 |
-
"sha": "3c69fd93da",
|
| 169 |
-
"date": "2026-08-23",
|
| 170 |
-
"subject": "release v48"
|
| 171 |
-
},
|
| 172 |
-
{
|
| 173 |
-
"version": "v47",
|
| 174 |
-
"sha": "917f9a0eb5",
|
| 175 |
-
"date": "2026-08-23",
|
| 176 |
-
"subject": "release v47"
|
| 177 |
-
},
|
| 178 |
-
{
|
| 179 |
-
"version": "v46",
|
| 180 |
-
"sha": "a8dc531b50",
|
| 181 |
-
"date": "2026-08-23",
|
| 182 |
-
"subject": "release v46"
|
| 183 |
-
},
|
| 184 |
-
{
|
| 185 |
-
"version": "v45",
|
| 186 |
-
"sha": "0b2b5010d8",
|
| 187 |
-
"date": "2026-08-23",
|
| 188 |
-
"subject": "release v45"
|
| 189 |
-
},
|
| 190 |
-
{
|
| 191 |
-
"version": "v44",
|
| 192 |
-
"sha": "a7877aaf20",
|
| 193 |
-
"date": "2026-08-23",
|
| 194 |
-
"subject": "Hotfix: allow renaming the active Map view"
|
| 195 |
-
},
|
| 196 |
-
{
|
| 197 |
-
"version": "v43",
|
| 198 |
-
"sha": "e2ba4616fa",
|
| 199 |
-
"date": "2026-08-23",
|
| 200 |
-
"subject": "release v43"
|
| 201 |
-
},
|
| 202 |
-
{
|
| 203 |
-
"version": "v42",
|
| 204 |
-
"sha": "25b885e637",
|
| 205 |
-
"date": "2026-08-23",
|
| 206 |
-
"subject": "release v42"
|
| 207 |
-
},
|
| 208 |
-
{
|
| 209 |
-
"version": "v41",
|
| 210 |
-
"sha": "1d6adcc3c3",
|
| 211 |
-
"date": "2026-08-22",
|
| 212 |
-
"subject": "v41 vertical route navigation"
|
| 213 |
-
},
|
| 214 |
-
{
|
| 215 |
-
"version": "v40",
|
| 216 |
-
"sha": "1451837168",
|
| 217 |
-
"date": "2026-08-22",
|
| 218 |
-
"subject": "release v40 route UI correction"
|
| 219 |
-
},
|
| 220 |
-
{
|
| 221 |
-
"version": "v39",
|
| 222 |
-
"sha": "a78bdf00e3",
|
| 223 |
-
"date": "2026-08-21",
|
| 224 |
-
"subject": "W39 clean stale migration targets"
|
| 225 |
-
},
|
| 226 |
-
{
|
| 227 |
-
"version": "v38",
|
| 228 |
-
"sha": "a556631cf9",
|
| 229 |
-
"date": "2026-08-21",
|
| 230 |
-
"subject": "W39 reversible Live store cutover"
|
| 231 |
-
},
|
| 232 |
-
{
|
| 233 |
-
"version": "v37",
|
| 234 |
-
"sha": "1ddd8cf823",
|
| 235 |
-
"date": "2026-08-21",
|
| 236 |
-
"subject": "W39: coalesce derived workspace reads"
|
| 237 |
-
},
|
| 238 |
-
{
|
| 239 |
-
"version": "v36",
|
| 240 |
-
"sha": "51db3b7fee",
|
| 241 |
-
"date": "2026-08-21",
|
| 242 |
-
"subject": "W39 share-field oid repair"
|
| 243 |
-
},
|
| 244 |
-
{
|
| 245 |
-
"version": "v35",
|
| 246 |
-
"sha": "0f642129c8",
|
| 247 |
-
"date": "2026-08-21",
|
| 248 |
-
"subject": "W39 visual QA fixture"
|
| 249 |
-
},
|
| 250 |
-
{
|
| 251 |
-
"version": "v34",
|
| 252 |
-
"sha": "cb4caa188e",
|
| 253 |
-
"date": "2026-08-21",
|
| 254 |
-
"subject": "W39 Staging PostgreSQL egress controls"
|
| 255 |
-
},
|
| 256 |
-
{
|
| 257 |
-
"version": "v33",
|
| 258 |
-
"sha": "2b115e573d",
|
| 259 |
-
"date": "2026-08-21",
|
| 260 |
-
"subject": "W39: stop nonproduction Postgres background egress"
|
| 261 |
-
},
|
| 262 |
-
{
|
| 263 |
-
"version": "v32",
|
| 264 |
-
"sha": "b47b27ca7b",
|
| 265 |
-
"date": "2026-08-21",
|
| 266 |
-
"subject": "W39: repair Staging Postgres projection"
|
| 267 |
-
},
|
| 268 |
-
{
|
| 269 |
-
"version": "v31",
|
| 270 |
-
"sha": "8880a49a24",
|
| 271 |
-
"date": "2026-08-21",
|
| 272 |
-
"subject": "v31: Wave 39 staging candidate"
|
| 273 |
-
},
|
| 274 |
-
{
|
| 275 |
-
"version": "v30",
|
| 276 |
-
"sha": "71776ca6c8",
|
| 277 |
-
"date": "2026-08-20",
|
| 278 |
-
"subject": "v30: W39-T29 Member-owned View first-share hotfix"
|
| 279 |
-
},
|
| 280 |
-
{
|
| 281 |
-
"version": "v29",
|
| 282 |
-
"sha": "55ced5018f",
|
| 283 |
-
"date": "2026-08-18",
|
| 284 |
-
"subject": "v29: the build staging has been running since 2026-08-18 (55ced50)."
|
| 285 |
-
},
|
| 286 |
-
{
|
| 287 |
-
"version": "v28",
|
| 288 |
-
"sha": "2793e4ba53",
|
| 289 |
-
"date": "2026-08-18",
|
| 290 |
-
"subject": "release v28"
|
| 291 |
-
},
|
| 292 |
-
{
|
| 293 |
-
"version": "v27",
|
| 294 |
-
"sha": "c05a794837",
|
| 295 |
-
"date": "2026-08-18",
|
| 296 |
-
"subject": "release v27"
|
| 297 |
-
},
|
| 298 |
-
{
|
| 299 |
-
"version": "v25",
|
| 300 |
-
"sha": "bc248295ed",
|
| 301 |
-
"date": "2026-08-14",
|
| 302 |
-
"subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)"
|
| 303 |
-
},
|
| 304 |
-
{
|
| 305 |
-
"version": "v26",
|
| 306 |
-
"sha": "6a642e139c",
|
| 307 |
-
"date": "2026-08-14",
|
| 308 |
-
"subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)"
|
| 309 |
-
},
|
| 310 |
-
{
|
| 311 |
-
"version": "v24",
|
| 312 |
-
"sha": "514d8292ce",
|
| 313 |
-
"date": "2026-08-09",
|
| 314 |
-
"subject": "release v24"
|
| 315 |
-
},
|
| 316 |
-
{
|
| 317 |
-
"version": "v23",
|
| 318 |
-
"sha": "44cc28380f",
|
| 319 |
-
"date": "2026-08-09",
|
| 320 |
-
"subject": "release v23"
|
| 321 |
-
},
|
| 322 |
-
{
|
| 323 |
-
"version": "v22",
|
| 324 |
-
"sha": "f14e199ff3",
|
| 325 |
-
"date": "2026-08-09",
|
| 326 |
-
"subject": "release v22"
|
| 327 |
-
},
|
| 328 |
-
{
|
| 329 |
-
"version": "v21",
|
| 330 |
-
"sha": "5fdffce8b5",
|
| 331 |
-
"date": "2026-08-09",
|
| 332 |
-
"subject": "release v21"
|
| 333 |
-
},
|
| 334 |
-
{
|
| 335 |
-
"version": "v20",
|
| 336 |
-
"sha": "442e4a1430",
|
| 337 |
-
"date": "2026-08-09",
|
| 338 |
-
"subject": "release v20"
|
| 339 |
-
},
|
| 340 |
-
{
|
| 341 |
-
"version": "v19",
|
| 342 |
-
"sha": "94cb4a4dfd",
|
| 343 |
-
"date": "2026-08-09",
|
| 344 |
-
"subject": "release v19"
|
| 345 |
-
},
|
| 346 |
-
{
|
| 347 |
-
"version": "v18",
|
| 348 |
-
"sha": "0106ef9498",
|
| 349 |
-
"date": "2026-08-09",
|
| 350 |
-
"subject": "release v18"
|
| 351 |
-
},
|
| 352 |
-
{
|
| 353 |
-
"version": "v17",
|
| 354 |
-
"sha": "308240cc75",
|
| 355 |
-
"date": "2026-08-09",
|
| 356 |
-
"subject": "release v17"
|
| 357 |
-
},
|
| 358 |
-
{
|
| 359 |
-
"version": "v16",
|
| 360 |
-
"sha": "a804df5443",
|
| 361 |
-
"date": "2026-08-09",
|
| 362 |
-
"subject": "release v16"
|
| 363 |
-
},
|
| 364 |
-
{
|
| 365 |
-
"version": "v15",
|
| 366 |
-
"sha": "6b84d7264c",
|
| 367 |
-
"date": "2026-08-09",
|
| 368 |
-
"subject": "release v15"
|
| 369 |
-
},
|
| 370 |
-
{
|
| 371 |
-
"version": "v14",
|
| 372 |
-
"sha": "bee5eaf5b5",
|
| 373 |
-
"date": "2026-08-07",
|
| 374 |
-
"subject": "release v14"
|
| 375 |
-
},
|
| 376 |
-
{
|
| 377 |
-
"version": "v13",
|
| 378 |
-
"sha": "314c0e368b",
|
| 379 |
-
"date": "2026-08-06",
|
| 380 |
-
"subject": "release v13"
|
| 381 |
-
},
|
| 382 |
-
{
|
| 383 |
-
"version": "v12",
|
| 384 |
-
"sha": "4908f39978",
|
| 385 |
-
"date": "2026-08-06",
|
| 386 |
-
"subject": "release v12"
|
| 387 |
-
},
|
| 388 |
-
{
|
| 389 |
-
"version": "v11",
|
| 390 |
-
"sha": "312661712b",
|
| 391 |
-
"date": "2026-08-05",
|
| 392 |
-
"subject": "release v11"
|
| 393 |
-
},
|
| 394 |
-
{
|
| 395 |
-
"version": "v10",
|
| 396 |
-
"sha": "43f8042874",
|
| 397 |
-
"date": "2026-08-05",
|
| 398 |
-
"subject": "release v10"
|
| 399 |
-
},
|
| 400 |
-
{
|
| 401 |
-
"version": "v9",
|
| 402 |
-
"sha": "2e26f57396",
|
| 403 |
-
"date": "2026-08-05",
|
| 404 |
-
"subject": "release v9"
|
| 405 |
-
},
|
| 406 |
-
{
|
| 407 |
-
"version": "v8",
|
| 408 |
-
"sha": "8073b3ecba",
|
| 409 |
-
"date": "2026-08-05",
|
| 410 |
-
"subject": "release v8"
|
| 411 |
-
},
|
| 412 |
-
{
|
| 413 |
-
"version": "v7",
|
| 414 |
-
"sha": "b1c5a7e9b8",
|
| 415 |
-
"date": "2026-08-05",
|
| 416 |
-
"subject": "release v7"
|
| 417 |
-
},
|
| 418 |
-
{
|
| 419 |
-
"version": "v6",
|
| 420 |
-
"sha": "a344020f11",
|
| 421 |
-
"date": "2026-08-05",
|
| 422 |
-
"subject": "release v6"
|
| 423 |
-
},
|
| 424 |
-
{
|
| 425 |
-
"version": "v5",
|
| 426 |
-
"sha": "c0b0fbb40a",
|
| 427 |
-
"date": "2026-08-05",
|
| 428 |
-
"subject": "release v5"
|
| 429 |
-
},
|
| 430 |
-
{
|
| 431 |
-
"version": "v4",
|
| 432 |
-
"sha": "de5037ff1c",
|
| 433 |
-
"date": "2026-08-05",
|
| 434 |
-
"subject": "release v4"
|
| 435 |
-
},
|
| 436 |
-
{
|
| 437 |
-
"version": "v3",
|
| 438 |
-
"sha": "6b9fa62d93",
|
| 439 |
-
"date": "2026-08-05",
|
| 440 |
-
"subject": "release v3"
|
| 441 |
-
},
|
| 442 |
-
{
|
| 443 |
-
"version": "v2",
|
| 444 |
-
"sha": "fd05861998",
|
| 445 |
-
"date": "2026-08-04",
|
| 446 |
-
"subject": "release v2"
|
| 447 |
-
},
|
| 448 |
-
{
|
| 449 |
-
"version": "v1",
|
| 450 |
-
"sha": "5b4e2c4bc3",
|
| 451 |
-
"date": "2026-08-04",
|
| 452 |
-
"subject": "release v1"
|
| 453 |
}
|
| 454 |
]
|
| 455 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "v77 (e741e5ca)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
+
"version": "v77",
|
| 6 |
+
"sha": "e741e5ca",
|
| 7 |
+
"date": "2026-10-05",
|
| 8 |
+
"subject": "release v77"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
}
|
| 10 |
]
|
| 11 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
v77 (e741e5ca)
|
api/agent_tools.py
CHANGED
|
@@ -34,6 +34,20 @@ def register(name, schema, fn, *, effect):
|
|
| 34 |
|
| 35 |
|
| 36 |
def load_modules():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
for name in AGENT_TOOL_MODULES:
|
| 38 |
try:
|
| 39 |
module = importlib.import_module(name)
|
|
@@ -48,6 +62,12 @@ def load_modules():
|
|
| 48 |
_LOG.info("Assistant tool entry in %s was skipped (%s)", name, type(exc).__name__)
|
| 49 |
|
| 50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
def registered():
|
| 52 |
return {name: {**entry, "schema": copy.deepcopy(entry["schema"])}
|
| 53 |
for name, entry in _TOOLS.items()}
|
|
|
|
| 34 |
|
| 35 |
|
| 36 |
def load_modules():
|
| 37 |
+
from core import keychain
|
| 38 |
+
|
| 39 |
+
# Minting changes only capability metadata. Credentials are entered through the
|
| 40 |
+
# public form, so every channel may return this link without a write confirmation.
|
| 41 |
+
register("connector_setup_link", {
|
| 42 |
+
"type": "object",
|
| 43 |
+
"description": "Create a one-time HTTPS setup link for your connector. Enter credentials only through the link.",
|
| 44 |
+
"properties": {
|
| 45 |
+
"connector": {"type": "string", "enum": sorted(keychain.ENTRY_TYPES)},
|
| 46 |
+
"scope": {"type": "string", "enum": ["personal", "business"]},
|
| 47 |
+
},
|
| 48 |
+
"required": ["connector", "scope"],
|
| 49 |
+
"additionalProperties": False,
|
| 50 |
+
}, _connector_setup_link, effect="read")
|
| 51 |
for name in AGENT_TOOL_MODULES:
|
| 52 |
try:
|
| 53 |
module = importlib.import_module(name)
|
|
|
|
| 62 |
_LOG.info("Assistant tool entry in %s was skipped (%s)", name, type(exc).__name__)
|
| 63 |
|
| 64 |
|
| 65 |
+
def _connector_setup_link(session, args):
|
| 66 |
+
from routes_forms import mint_setup_link
|
| 67 |
+
|
| 68 |
+
return mint_setup_link(args, session=session)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
def registered():
|
| 72 |
return {name: {**entry, "schema": copy.deepcopy(entry["schema"])}
|
| 73 |
for name, entry in _TOOLS.items()}
|
api/amazon_relational.py
CHANGED
|
@@ -139,6 +139,14 @@ _MONEY.update({"estimated_fee_total", "estimated_referral_fee_per_unit",
|
|
| 139 |
#: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
|
| 140 |
_RATE_SUFFIX = ("_percentage", "_percentage_b2_b", "_rate", "_ctr")
|
| 141 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
def _label(col):
|
| 144 |
return col.replace("_", " ").strip().capitalize()
|
|
@@ -157,10 +165,10 @@ def _ftype(col, duck_type):
|
|
| 157 |
if col in _MONEY or col.endswith(("_amount", "_fee")) or any(col.endswith(s) for s in _RATE_SUFFIX):
|
| 158 |
return "currency"
|
| 159 |
base = str(duck_type or "").upper().split("(")[0]
|
| 160 |
-
if base in ("
|
| 161 |
-
return "int"
|
| 162 |
-
if base in ("DOUBLE", "FLOAT"):
|
| 163 |
return "currency"
|
|
|
|
|
|
|
| 164 |
return "text"
|
| 165 |
|
| 166 |
|
|
@@ -521,9 +529,10 @@ def grid_source(cur, key):
|
|
| 521 |
"needs_excluded": False, "cols": cols}
|
| 522 |
|
| 523 |
|
| 524 |
-
def grid_sources():
|
| 525 |
"""Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
|
| 526 |
-
|
|
|
|
| 527 |
try:
|
| 528 |
present = {key for key, table in SOURCE.items() if mirror_columns(cur, table)}
|
| 529 |
if not present:
|
|
@@ -535,7 +544,8 @@ def grid_sources():
|
|
| 535 |
keys.add(key)
|
| 536 |
return {key: grid_source(cur, key) for key in SOURCE if key in keys}
|
| 537 |
finally:
|
| 538 |
-
|
|
|
|
| 539 |
|
| 540 |
|
| 541 |
def _row_id(row, key):
|
|
@@ -584,8 +594,12 @@ def _read(cur, key):
|
|
| 584 |
cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
|
| 585 |
if not cols:
|
| 586 |
return []
|
| 587 |
-
quoted = ", ".join(f'"{c}"' for c in cols)
|
| 588 |
spec = grid_source(cur, key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 589 |
# S67-T30: a stored snapshot carries the same product lookups, computed by the SAME SQL
|
| 590 |
# expressions the read-through binding uses, in this one SELECT (never per row).
|
| 591 |
derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
|
|
@@ -596,6 +610,9 @@ def _read(cur, key):
|
|
| 596 |
for r in rows:
|
| 597 |
row = {}
|
| 598 |
for name, val in zip(names, r[1:]):
|
|
|
|
|
|
|
|
|
|
| 599 |
# Postgres JSON cannot hold U+0000, and one such byte refused the whole Amazon write.
|
| 600 |
row[name] = "" if val is None else (val if isinstance(val, (int, float))
|
| 601 |
else str(val).replace("\x00", ""))
|
|
@@ -858,6 +875,17 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 858 |
written[key].update(rows=rows["count"], removed=removed, readThrough=True)
|
| 859 |
else:
|
| 860 |
doc[key].pop("readThrough", None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 861 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 862 |
# mutable display label or from the process having imported this module.
|
| 863 |
doc[key]["connector"] = source_attribution(key)
|
|
@@ -913,6 +941,10 @@ def _refresh_from_cursor(cur, rt, why, tenant, previous_fingerprint, log, state)
|
|
| 913 |
f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
|
| 914 |
return state
|
| 915 |
state["present"] = True
|
|
|
|
|
|
|
|
|
|
|
|
|
| 916 |
built = plan(cur, rt)
|
| 917 |
problems = built.get("problems") or []
|
| 918 |
for p in problems:
|
|
|
|
| 139 |
#: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
|
| 140 |
_RATE_SUFFIX = ("_percentage", "_percentage_b2_b", "_rate", "_ctr")
|
| 141 |
|
| 142 |
+
# Inventory reports remain VARCHAR when a quantity column mixes numbers and blanks.
|
| 143 |
+
# Names identify units; identifiers such as SKU and FNSKU must stay text.
|
| 144 |
+
_QUANTITY = {"quantity", "quantity_purchased", "starting_warehouse_balance",
|
| 145 |
+
"in_transit_between_warehouses", "receipts", "customer_shipments",
|
| 146 |
+
"customer_returns", "vendor_returns", "warehouse_transfer_in",
|
| 147 |
+
"warehouse_transfer_out", "found", "lost", "damaged", "disposed",
|
| 148 |
+
"other_events", "ending_warehouse_balance", "unknown_events"}
|
| 149 |
+
|
| 150 |
|
| 151 |
def _label(col):
|
| 152 |
return col.replace("_", " ").strip().capitalize()
|
|
|
|
| 165 |
if col in _MONEY or col.endswith(("_amount", "_fee")) or any(col.endswith(s) for s in _RATE_SUFFIX):
|
| 166 |
return "currency"
|
| 167 |
base = str(duck_type or "").upper().split("(")[0]
|
| 168 |
+
if base in ("DOUBLE", "FLOAT", "DECIMAL"):
|
|
|
|
|
|
|
| 169 |
return "currency"
|
| 170 |
+
if col in _QUANTITY or base in ("BIGINT", "INTEGER", "HUGEINT"):
|
| 171 |
+
return "int"
|
| 172 |
return "text"
|
| 173 |
|
| 174 |
|
|
|
|
| 529 |
"needs_excluded": False, "cols": cols}
|
| 530 |
|
| 531 |
|
| 532 |
+
def grid_sources(cur=None):
|
| 533 |
"""Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
|
| 534 |
+
own_cursor = cur is None
|
| 535 |
+
cur = datastore.ro_cursor() if own_cursor else cur
|
| 536 |
try:
|
| 537 |
present = {key for key, table in SOURCE.items() if mirror_columns(cur, table)}
|
| 538 |
if not present:
|
|
|
|
| 544 |
keys.add(key)
|
| 545 |
return {key: grid_source(cur, key) for key in SOURCE if key in keys}
|
| 546 |
finally:
|
| 547 |
+
if own_cursor:
|
| 548 |
+
cur.close()
|
| 549 |
|
| 550 |
|
| 551 |
def _row_id(row, key):
|
|
|
|
| 594 |
cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
|
| 595 |
if not cols:
|
| 596 |
return []
|
|
|
|
| 597 |
spec = grid_source(cur, key)
|
| 598 |
+
# Stored snapshots and mirror windows share their numeric casts. Empty report cells
|
| 599 |
+
# become null; decimals remain numbers instead of text that defeats aggregation.
|
| 600 |
+
quoted = ", ".join(spec["cols"][c][0] for c in cols)
|
| 601 |
+
numeric = {name for name, dtype in mirror_columns(cur, SOURCE[key])
|
| 602 |
+
if _ftype(name, dtype) in ("currency", "int")}
|
| 603 |
# S67-T30: a stored snapshot carries the same product lookups, computed by the SAME SQL
|
| 604 |
# expressions the read-through binding uses, in this one SELECT (never per row).
|
| 605 |
derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
|
|
|
|
| 610 |
for r in rows:
|
| 611 |
row = {}
|
| 612 |
for name, val in zip(names, r[1:]):
|
| 613 |
+
if name in numeric:
|
| 614 |
+
row[name] = spec["cols"][name][1](val)
|
| 615 |
+
continue
|
| 616 |
# Postgres JSON cannot hold U+0000, and one such byte refused the whole Amazon write.
|
| 617 |
row[name] = "" if val is None else (val if isinstance(val, (int, float))
|
| 618 |
else str(val).replace("\x00", ""))
|
|
|
|
| 875 |
written[key].update(rows=rows["count"], removed=removed, readThrough=True)
|
| 876 |
else:
|
| 877 |
doc[key].pop("readThrough", None)
|
| 878 |
+
# The shared merger compares string spellings, so an old "2" and a new
|
| 879 |
+
# integer 2 otherwise compare equal and leave the old cell untyped.
|
| 880 |
+
numeric = {f["key"] for f in doc[key].get("fields", [])
|
| 881 |
+
if f.get("type") in ("currency", "int")
|
| 882 |
+
and (f.get("automation") or {}).get("flowId") == "amazon_relational"
|
| 883 |
+
and not _odoo._ut().user_edited(f)}
|
| 884 |
+
stored = doc[key].get("rows") or {}
|
| 885 |
+
for row in rows:
|
| 886 |
+
saved = stored.get(row["_id"])
|
| 887 |
+
if numeric and isinstance(saved, dict):
|
| 888 |
+
saved.update({name: row[name] for name in numeric if name in row})
|
| 889 |
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 890 |
# mutable display label or from the process having imported this module.
|
| 891 |
doc[key]["connector"] = source_attribution(key)
|
|
|
|
| 941 |
f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
|
| 942 |
return state
|
| 943 |
state["present"] = True
|
| 944 |
+
# S70-T12 / D-3495: publish immediately from the restored cursor, before planning or
|
| 945 |
+
# the unchanged-fingerprint return. Registry readiness must not wait for a store write.
|
| 946 |
+
import routes_odoo_tables
|
| 947 |
+
routes_odoo_tables._learn_amazon_sources(cur)
|
| 948 |
built = plan(cur, rt)
|
| 949 |
problems = built.get("problems") or []
|
| 950 |
for p in problems:
|
api/automation_engine.py
CHANGED
|
@@ -16506,6 +16506,10 @@ def start_scheduler(log=print):
|
|
| 16506 |
external EventBridge tick (R5) does not depend on it either way, since that POSTs the
|
| 16507 |
endpoint rather than riding this thread.
|
| 16508 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16509 |
if os.environ.get("AIOS_AUTOMATIONS") != "1":
|
| 16510 |
return False
|
| 16511 |
# PostgreSQL production has one durable wake-up: EventBridge -> Lambda -> protected endpoint.
|
|
|
|
| 16506 |
external EventBridge tick (R5) does not depend on it either way, since that POSTs the
|
| 16507 |
endpoint rather than riding this thread.
|
| 16508 |
"""
|
| 16509 |
+
from core import freeze_notice
|
| 16510 |
+
if freeze_notice.enabled():
|
| 16511 |
+
log("[aios-auto] maintenance freeze holds automation scheduler")
|
| 16512 |
+
return False
|
| 16513 |
if os.environ.get("AIOS_AUTOMATIONS") != "1":
|
| 16514 |
return False
|
| 16515 |
# PostgreSQL production has one durable wake-up: EventBridge -> Lambda -> protected endpoint.
|
api/automation_worker.py
CHANGED
|
@@ -67,7 +67,9 @@ def _claim(job: dict):
|
|
| 67 |
url = str(os.environ.get("DATABASE_URL") or "").strip()
|
| 68 |
if not url:
|
| 69 |
raise RuntimeError("DATABASE_URL is required")
|
| 70 |
-
|
|
|
|
|
|
|
| 71 |
lock_name = f"aios-automation:{job['tenant']}:{job['automationId'] or job['mode']}"
|
| 72 |
with con.cursor() as cur:
|
| 73 |
cur.execute("SELECT pg_try_advisory_lock(hashtextextended(%s, 0))", (lock_name,))
|
|
|
|
| 67 |
url = str(os.environ.get("DATABASE_URL") or "").strip()
|
| 68 |
if not url:
|
| 69 |
raise RuntimeError("DATABASE_URL is required")
|
| 70 |
+
ca = os.environ.get("AIOS_DATABASE_CA_PEM_B64")
|
| 71 |
+
options = {"sslrootcert": store_pg._database_ca_file(ca)} if ca is not None else {}
|
| 72 |
+
con = psycopg.connect(url, **options)
|
| 73 |
lock_name = f"aios-automation:{job['tenant']}:{job['automationId'] or job['mode']}"
|
| 74 |
with con.cursor() as cur:
|
| 75 |
cur.execute("SELECT pg_try_advisory_lock(hashtextextended(%s, 0))", (lock_name,))
|
api/connector_onboarding.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""First-connection choices, stored before a credential can dispatch a sync."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def choices(provider):
|
| 6 |
+
if provider == "odoo":
|
| 7 |
+
import odoo_relational as rel
|
| 8 |
+
defaults = {"invoices", "orders", "order_lines", "stock_moves", "stock_valuation", "stock_quant"}
|
| 9 |
+
return ([{"key": "customer_data", "label": "Customers", "selected": True},
|
| 10 |
+
{"key": "product_data", "label": "Products", "selected": True}]
|
| 11 |
+
+ [{"key": key, "label": label, "selected": bucket in defaults}
|
| 12 |
+
for bucket, key, label, _ in rel.TABLES])
|
| 13 |
+
if provider == "amazon_sp":
|
| 14 |
+
from harness import amazon_store
|
| 15 |
+
return [{"key": "ut_amzn_" + key, "label": key.replace("_", " ").capitalize(),
|
| 16 |
+
"selected": key in {"orders", "order_items", "inventory", "settlements"}}
|
| 17 |
+
for key in amazon_store.TABLES]
|
| 18 |
+
return []
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def install_selection(rt, provider, selected, *, existing=False):
|
| 22 |
+
"""Explicit first-save intent only; preserve every pre-existing connector configuration.
|
| 23 |
+
|
| 24 |
+
This runs before add_entry, so an automatic runner cannot see a credential with all-on defaults.
|
| 25 |
+
A failed credential save leaves a safe, resumable choice document, never an unselected sync.
|
| 26 |
+
"""
|
| 27 |
+
offered = {row["key"] for row in choices(provider)}
|
| 28 |
+
if not offered or not isinstance(selected, list) or any(type(key) is not str for key in selected):
|
| 29 |
+
raise ValueError("Choose databases from this connector's setup list.")
|
| 30 |
+
if len(set(selected)) != len(selected) or not set(selected) <= offered:
|
| 31 |
+
raise ValueError("The database selection contains an unknown or duplicate database.")
|
| 32 |
+
if existing:
|
| 33 |
+
raise ValueError("This connection already exists. Change its databases in Settings.")
|
| 34 |
+
if provider == "odoo":
|
| 35 |
+
import odoo_relational as rel
|
| 36 |
+
key, member = rel.CONFIG_KEY, "grids"
|
| 37 |
+
else:
|
| 38 |
+
import connectors_amazon as amazon
|
| 39 |
+
key, member = amazon.ENABLEMENT_KEY, "databases"
|
| 40 |
+
|
| 41 |
+
def install(current):
|
| 42 |
+
if current is None:
|
| 43 |
+
current = {}
|
| 44 |
+
if not isinstance(current, dict):
|
| 45 |
+
raise ValueError("The stored connector settings are unavailable.")
|
| 46 |
+
# Only our own incomplete setup can be retried. Older selections are never repurposed.
|
| 47 |
+
if current and current.get("initialSetup") != "selected":
|
| 48 |
+
raise ValueError("This workspace already has database choices. Use Settings to change them.")
|
| 49 |
+
return {**current, member: {name: name in selected for name in sorted(offered)},
|
| 50 |
+
"initialSetup": "selected"}
|
| 51 |
+
|
| 52 |
+
rt.update(key, install, flush="sync")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def seal_selection(rt, provider):
|
| 56 |
+
from odoo_relational import CONFIG_KEY
|
| 57 |
+
from connectors_amazon import ENABLEMENT_KEY
|
| 58 |
+
key = CONFIG_KEY if provider == "odoo" else ENABLEMENT_KEY
|
| 59 |
+
|
| 60 |
+
def seal(current):
|
| 61 |
+
if not isinstance(current, dict) or current.get("initialSetup") != "selected":
|
| 62 |
+
raise ValueError("The first-connection selection was not saved.")
|
| 63 |
+
return {**current, "initialSetup": "configured"}
|
| 64 |
+
|
| 65 |
+
rt.update(key, seal, flush="sync")
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def selected_mirror_entities(rt):
|
| 69 |
+
"""None preserves old sync policy; an explicit setup limits source pulls to selected dependencies."""
|
| 70 |
+
import odoo_relational as rel
|
| 71 |
+
config = rt.get(rel.CONFIG_KEY)
|
| 72 |
+
if not isinstance(config, dict) or config.get("initialSetup") not in {"selected", "configured"}:
|
| 73 |
+
return None
|
| 74 |
+
grids = config.get("grids")
|
| 75 |
+
if not isinstance(grids, dict) or any(type(v) is not bool for v in grids.values()):
|
| 76 |
+
raise ValueError("The first-connection database selection is unavailable.")
|
| 77 |
+
import routes_odoo_tables as route
|
| 78 |
+
from harness import datastore
|
| 79 |
+
sources = route._odoo_sources()
|
| 80 |
+
entities = set()
|
| 81 |
+
for key, enabled in grids.items():
|
| 82 |
+
if not enabled:
|
| 83 |
+
continue
|
| 84 |
+
if key in {"customer_data", "product_data"}:
|
| 85 |
+
entities.add({"customer_data": "res_partner", "product_data": "product_product"}[key])
|
| 86 |
+
continue
|
| 87 |
+
if key not in sources:
|
| 88 |
+
continue
|
| 89 |
+
source = sources[key]
|
| 90 |
+
if source.get("table"):
|
| 91 |
+
entities.add(source["table"])
|
| 92 |
+
entities.update(source.get("tables", {}).values())
|
| 93 |
+
if source.get("needs_excluded"):
|
| 94 |
+
entities.add("res_partner")
|
| 95 |
+
return sorted(entities & set(datastore.ENTITIES))
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def mirror_cursor(rt, table_key=None):
|
| 99 |
+
"""A selected source waits for its dependencies, without declaring the entire mirror ready."""
|
| 100 |
+
from harness import datastore
|
| 101 |
+
if rt is None:
|
| 102 |
+
return datastore.ro_con()
|
| 103 |
+
rt.assert_datastore_matches()
|
| 104 |
+
entities = selected_mirror_entities(rt)
|
| 105 |
+
if entities is None:
|
| 106 |
+
return datastore.ro_con()
|
| 107 |
+
if table_key is not None:
|
| 108 |
+
import odoo_relational as rel
|
| 109 |
+
import routes_odoo_tables as route
|
| 110 |
+
config = rt.get(rel.CONFIG_KEY)
|
| 111 |
+
if not config.get("grids", {}).get(table_key, False):
|
| 112 |
+
raise RuntimeError("Enable this database in connection settings before reading it.")
|
| 113 |
+
source = route._odoo_sources().get(table_key)
|
| 114 |
+
if source is None:
|
| 115 |
+
return datastore.ro_con()
|
| 116 |
+
entities = set(source.get("tables", {}).values())
|
| 117 |
+
if source.get("table"):
|
| 118 |
+
entities.add(source["table"])
|
| 119 |
+
if source.get("needs_excluded"):
|
| 120 |
+
entities.add("res_partner")
|
| 121 |
+
if any(not datastore.entity_live(key) for key in entities):
|
| 122 |
+
raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
|
| 123 |
+
# An empty first selection has no readers to run and need not create a mirror file.
|
| 124 |
+
return datastore.ro_cursor() if entities else None
|
api/formula_runtime.py
CHANGED
|
@@ -3110,7 +3110,7 @@ class _SheetInputs(dict):
|
|
| 3110 |
self.engine._consume(super().get(SHEET_ADDR_KEY), key)
|
| 3111 |
owner = self.engine.spill_parents.get(key)
|
| 3112 |
if owner in self.engine._pending: raise _NeedReference(owner)
|
| 3113 |
-
fault = key if key in self.engine.errors else None
|
| 3114 |
if not _occupied(self.engine.specs.get(key)):
|
| 3115 |
producers = self.engine.prospective.at(key, self.engine.stats)
|
| 3116 |
pending = producers & self.engine._pending
|
|
@@ -3123,6 +3123,8 @@ class _SheetInputs(dict):
|
|
| 3123 |
raise _ReferenceFault(fault)
|
| 3124 |
self.engine.stats["reads"] += 1
|
| 3125 |
if self.format_stack: self.format_stack[-1].append(self.engine.formats.get(key))
|
|
|
|
|
|
|
| 3126 |
return self.engine.values.get(key, default)
|
| 3127 |
|
| 3128 |
def range_values(self, bounds):
|
|
@@ -3783,7 +3785,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
|
|
| 3783 |
projected += 1
|
| 3784 |
code = engine.errors.get(key)
|
| 3785 |
cell["value"] = engine.values.get(key)
|
| 3786 |
-
if code:
|
| 3787 |
message = _sheet_error_message(code)
|
| 3788 |
cell.update(state="refused", display="", refusal={"code": code, "message": message},
|
| 3789 |
error={"code": code, "message": message,
|
|
@@ -3791,6 +3793,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
|
|
| 3791 |
excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
|
| 3792 |
if excel: cell["error"]["excelCode"] = excel
|
| 3793 |
else:
|
|
|
|
| 3794 |
cell.update(state="fresh", display=_display_text(cell["value"]))
|
| 3795 |
cell.pop("refusal", None)
|
| 3796 |
cell.pop("error", None)
|
|
|
|
| 3110 |
self.engine._consume(super().get(SHEET_ADDR_KEY), key)
|
| 3111 |
owner = self.engine.spill_parents.get(key)
|
| 3112 |
if owner in self.engine._pending: raise _NeedReference(owner)
|
| 3113 |
+
fault = key if key in self.engine.errors and self.engine.errors[key] != "blank" else None
|
| 3114 |
if not _occupied(self.engine.specs.get(key)):
|
| 3115 |
producers = self.engine.prospective.at(key, self.engine.stats)
|
| 3116 |
pending = producers & self.engine._pending
|
|
|
|
| 3123 |
raise _ReferenceFault(fault)
|
| 3124 |
self.engine.stats["reads"] += 1
|
| 3125 |
if self.format_stack: self.format_stack[-1].append(self.engine.formats.get(key))
|
| 3126 |
+
if self.engine.errors.get(key) == "blank":
|
| 3127 |
+
return ""
|
| 3128 |
return self.engine.values.get(key, default)
|
| 3129 |
|
| 3130 |
def range_values(self, bounds):
|
|
|
|
| 3785 |
projected += 1
|
| 3786 |
code = engine.errors.get(key)
|
| 3787 |
cell["value"] = engine.values.get(key)
|
| 3788 |
+
if code and code != "blank":
|
| 3789 |
message = _sheet_error_message(code)
|
| 3790 |
cell.update(state="refused", display="", refusal={"code": code, "message": message},
|
| 3791 |
error={"code": code, "message": message,
|
|
|
|
| 3793 |
excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
|
| 3794 |
if excel: cell["error"]["excelCode"] = excel
|
| 3795 |
else:
|
| 3796 |
+
if code == "blank": cell["value"] = ""
|
| 3797 |
cell.update(state="fresh", display=_display_text(cell["value"]))
|
| 3798 |
cell.pop("refusal", None)
|
| 3799 |
cell.pop("error", None)
|
api/main.py
CHANGED
|
@@ -89,6 +89,7 @@ from core import grid_events # noqa: E402
|
|
| 89 |
# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
| 90 |
# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not.
|
| 91 |
from core import data_binding # noqa: E402
|
|
|
|
| 92 |
from core import store # noqa: E402
|
| 93 |
# W50-T28 clause 2 (C3 / R9) — the THIRD store refusal, and the only one of the three that is a
|
| 94 |
# 409: a view write whose caller was looking at an older revision. See its handler below.
|
|
@@ -182,6 +183,32 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
|
|
| 182 |
status_code=503)
|
| 183 |
|
| 184 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
@app.exception_handler(data_binding.StoreWriteRefused)
|
| 186 |
def _store_write_refused(request: Request, exc: data_binding.StoreWriteRefused):
|
| 187 |
"""D-315 — this deployment may not write that store. 503, with the REASON.
|
|
@@ -266,6 +293,12 @@ def _view_revision_unverifiable(request: Request, exc: view_audit.RevisionUnveri
|
|
| 266 |
status_code=503)
|
| 267 |
|
| 268 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
@app.get("/api/health")
|
| 270 |
def health(challenge: str = ""):
|
| 271 |
"""Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
|
|
@@ -641,6 +674,14 @@ if _WEB_DIST.is_dir():
|
|
| 641 |
# default envelope; scoped users still pay their own scope's first build, once.
|
| 642 |
# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
|
| 643 |
# run against fakes) can never fire a live Odoo pull.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
def _seed_and_sync_store():
|
| 645 |
"""Bring the analytical store (harness.datastore) LIVE for this container — the app.py
|
| 646 |
bootstrap, mirrored (2026-07-31, owner item 1).
|
|
@@ -656,6 +697,8 @@ def _seed_and_sync_store():
|
|
| 656 |
passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still
|
| 657 |
serve.
|
| 658 |
"""
|
|
|
|
|
|
|
| 659 |
import time as _t
|
| 660 |
t0 = _t.time()
|
| 661 |
try:
|
|
@@ -669,9 +712,10 @@ def _seed_and_sync_store():
|
|
| 669 |
_odoo._tlocal.client = _odoo.OdooClient()
|
| 670 |
except Exception:
|
| 671 |
pass
|
|
|
|
| 672 |
res = {}
|
| 673 |
for i in range(12):
|
| 674 |
-
res = _ds.sync_all(log=lambda *a, **k: None)
|
| 675 |
print(f"[aios-api] datastore sync pass {i + 1}: "
|
| 676 |
+ ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
|
| 677 |
if all(v.get("phase") == "live" for v in res.values()):
|
|
@@ -682,7 +726,7 @@ def _seed_and_sync_store():
|
|
| 682 |
# measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of
|
| 683 |
# phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran.
|
| 684 |
try:
|
| 685 |
-
_ds.reconcile_deletes(log=lambda *a, **k: None)
|
| 686 |
except Exception:
|
| 687 |
pass
|
| 688 |
print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
|
|
@@ -1007,11 +1051,14 @@ def _run_sync_requests(outcome=None):
|
|
| 1007 |
# transport. Do not borrow a request's cached XML-RPC client.
|
| 1008 |
odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
|
| 1009 |
odoo._tlocal.tenant_ctx = None
|
| 1010 |
-
|
|
|
|
|
|
|
|
|
|
| 1011 |
finally:
|
| 1012 |
odoo._tlocal.client = previous_client
|
| 1013 |
odoo._tlocal.tenant_ctx = previous_context
|
| 1014 |
-
if not report:
|
| 1015 |
raise _SyncNowRefused(
|
| 1016 |
"Odoo did not run a sync. Check whether the connection is paused.")
|
| 1017 |
if any(row.get("phase") != "live" for row in report.values()):
|
|
@@ -1844,7 +1891,13 @@ _neon_usage.set_guards(STORE_GUARDS)
|
|
| 1844 |
_startup_notes()
|
| 1845 |
|
| 1846 |
|
| 1847 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1848 |
"""ONE bounded mirror pass: DuckDB and the Odoo XML-RPC client, and nothing else.
|
| 1849 |
|
| 1850 |
⛔⛔ NO `Store`, NO `store_kv`, NO `user_tables` IN THIS BODY, AND THAT IS A CONTRACT RATHER
|
|
@@ -1873,7 +1926,7 @@ def _resync_mirror_pass(manual, passes):
|
|
| 1873 |
# relational rebuild, which `manual` says nothing about.
|
| 1874 |
if manual:
|
| 1875 |
return None, None, None
|
| 1876 |
-
sync_report = _ds.sync_all(log=lambda *a, **k: None)
|
| 1877 |
# Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
|
| 1878 |
# the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
|
| 1879 |
# ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
|
|
@@ -1881,7 +1934,7 @@ def _resync_mirror_pass(manual, passes):
|
|
| 1881 |
# HOTFIX 2026-09-29: EVERY pass, not every fourth. At a daily cadence a fourth pass is four
|
| 1882 |
# days, and the first pass after a reseed never reconciled at all. `reconcile_deletes`
|
| 1883 |
# compares counts first and walks ids only for an entity whose count moved.
|
| 1884 |
-
delete_report = _ds.reconcile_deletes(log=lambda *a, **k: None)
|
| 1885 |
return sync_report, delete_report, None
|
| 1886 |
except Exception as e: # noqa: BLE001
|
| 1887 |
# A partial report must never be mistaken for a confirmed zero-change pass.
|
|
@@ -2019,6 +2072,8 @@ def _store_resync_loop():
|
|
| 2019 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 2020 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 2021 |
refreshed pool re-reads the freshly synced store."""
|
|
|
|
|
|
|
| 2022 |
import time as _t
|
| 2023 |
from core import connector_sync
|
| 2024 |
passes = 0
|
|
@@ -2156,7 +2211,11 @@ def _store_resync_loop():
|
|
| 2156 |
RESYNC_STATE["passes"] = passes
|
| 2157 |
if _odoo_requested is None:
|
| 2158 |
_odoo_requested = _run_sync_requests()
|
| 2159 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2160 |
source_fingerprints = _resync_source_fingerprints(sync_report)
|
| 2161 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 2162 |
RESYNC_STATE["lastPassOk"] = _err is None
|
|
@@ -2322,6 +2381,8 @@ def _store_resync_loop():
|
|
| 2322 |
|
| 2323 |
def _boot_app_migrations(why):
|
| 2324 |
"""Apply tenant App retirement migrations independently of mirror sync and reads."""
|
|
|
|
|
|
|
| 2325 |
try:
|
| 2326 |
import application_registry
|
| 2327 |
import timeseries_retirement
|
|
@@ -2345,6 +2406,8 @@ def _boot_app_migrations(why):
|
|
| 2345 |
|
| 2346 |
|
| 2347 |
def _prewarm():
|
|
|
|
|
|
|
| 2348 |
import time as _t
|
| 2349 |
t0 = _t.time()
|
| 2350 |
# The Odoo seed/sync can run for minutes and is separate from the finite cache warm below.
|
|
@@ -2443,6 +2506,8 @@ def _provision_qa_sandbox(pg=None):
|
|
| 2443 |
does not open a PostgreSQL connection merely to learn it should refuse. Live has neither
|
| 2444 |
deployment flag and returns before importing the PostgreSQL backend.
|
| 2445 |
"""
|
|
|
|
|
|
|
| 2446 |
if os.environ.get("AIOS_ENABLE_QA_TENANT") != "1":
|
| 2447 |
return False
|
| 2448 |
granted = {s.strip().lower() for s in
|
|
@@ -2478,18 +2543,23 @@ except Exception as e: # noqa: BLE001
|
|
| 2478 |
# never blocks health/readiness; the SQL procedure is idempotent, and `_provision_qa_sandbox`
|
| 2479 |
# itself refuses before opening a pool on every other environment.
|
| 2480 |
if (os.environ.get("AIOS_ENABLE_QA_TENANT") == "1"
|
| 2481 |
-
and os.environ.get("STORE_BACKEND") == "pg"
|
|
|
|
| 2482 |
import threading as _threading_qa
|
| 2483 |
_threading_qa.Thread(target=_provision_qa_sandbox, daemon=True,
|
| 2484 |
name="qa-sandbox-schema").start()
|
| 2485 |
|
| 2486 |
if (os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 2487 |
-
and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
|
|
|
| 2488 |
import threading as _threading_apps
|
| 2489 |
_threading_apps.Thread(target=_boot_app_migrations, args=("boot",), daemon=True,
|
| 2490 |
name="app-migrations").start()
|
| 2491 |
|
| 2492 |
-
if os.environ.get("
|
|
|
|
|
|
|
|
|
|
| 2493 |
import threading as _threading
|
| 2494 |
_threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
|
| 2495 |
|
|
@@ -2524,6 +2594,6 @@ if os.environ.get("AIOS_PREWARM") == "1":
|
|
| 2524 |
# The other structural cost remains outside that skip policy:
|
| 2525 |
# * the whole-document store write itself. A partial-JSONB write is a wave.
|
| 2526 |
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 2527 |
-
if os.environ.get("AIOS_STORE_RESYNC") == "1":
|
| 2528 |
import threading as _threading_resync
|
| 2529 |
_threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
|
|
|
|
| 89 |
# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
|
| 90 |
# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not.
|
| 91 |
from core import data_binding # noqa: E402
|
| 92 |
+
from core import freeze_notice # noqa: E402
|
| 93 |
from core import store # noqa: E402
|
| 94 |
# W50-T28 clause 2 (C3 / R9) — the THIRD store refusal, and the only one of the three that is a
|
| 95 |
# 409: a view write whose caller was looking at an older revision. See its handler below.
|
|
|
|
| 183 |
status_code=503)
|
| 184 |
|
| 185 |
|
| 186 |
+
@app.exception_handler(freeze_notice.DataInputFrozen)
|
| 187 |
+
def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen):
|
| 188 |
+
"""Use the existing readable error envelope for every maintenance refusal."""
|
| 189 |
+
return JSONResponse(
|
| 190 |
+
{"error": {"code": freeze_notice.ERROR_CODE, "message": freeze_notice.REFUSAL_TEXT}},
|
| 191 |
+
status_code=freeze_notice.STATUS_CODE)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
@app.middleware("http")
|
| 195 |
+
async def _freeze_response(request: Request, call_next):
|
| 196 |
+
"""Keep a refused write readable even when a route catches its store error."""
|
| 197 |
+
if request.method in {"GET", "HEAD", "OPTIONS"}:
|
| 198 |
+
return await call_next(request)
|
| 199 |
+
with freeze_notice.refusal_scope() as refused:
|
| 200 |
+
try:
|
| 201 |
+
response = await call_next(request)
|
| 202 |
+
except Exception:
|
| 203 |
+
if not refused[0]:
|
| 204 |
+
raise
|
| 205 |
+
return _data_input_frozen(request, freeze_notice.DataInputFrozen())
|
| 206 |
+
if refused[0]:
|
| 207 |
+
# Replace the response, including any cookie minted from unpersisted auth state.
|
| 208 |
+
return _data_input_frozen(request, freeze_notice.DataInputFrozen())
|
| 209 |
+
return response
|
| 210 |
+
|
| 211 |
+
|
| 212 |
@app.exception_handler(data_binding.StoreWriteRefused)
|
| 213 |
def _store_write_refused(request: Request, exc: data_binding.StoreWriteRefused):
|
| 214 |
"""D-315 — this deployment may not write that store. 503, with the REASON.
|
|
|
|
| 293 |
status_code=503)
|
| 294 |
|
| 295 |
|
| 296 |
+
@app.get("/api/v1/freeze-notice")
|
| 297 |
+
def public_freeze_notice():
|
| 298 |
+
"""Report public maintenance state without resolving a session or store."""
|
| 299 |
+
return JSONResponse(freeze_notice.notice(), headers={"Cache-Control": "no-store"})
|
| 300 |
+
|
| 301 |
+
|
| 302 |
@app.get("/api/health")
|
| 303 |
def health(challenge: str = ""):
|
| 304 |
"""Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
|
|
|
|
| 674 |
# default envelope; scoped users still pay their own scope's first build, once.
|
| 675 |
# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
|
| 676 |
# run against fakes) can never fire a live Odoo pull.
|
| 677 |
+
def _freeze_holds(writer):
|
| 678 |
+
"""Log each held writer before it can start work."""
|
| 679 |
+
if not freeze_notice.enabled():
|
| 680 |
+
return False
|
| 681 |
+
print(f"[aios-api] maintenance freeze holds {writer}")
|
| 682 |
+
return True
|
| 683 |
+
|
| 684 |
+
|
| 685 |
def _seed_and_sync_store():
|
| 686 |
"""Bring the analytical store (harness.datastore) LIVE for this container — the app.py
|
| 687 |
bootstrap, mirrored (2026-07-31, owner item 1).
|
|
|
|
| 697 |
passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still
|
| 698 |
serve.
|
| 699 |
"""
|
| 700 |
+
if _freeze_holds("boot store sync"):
|
| 701 |
+
return
|
| 702 |
import time as _t
|
| 703 |
t0 = _t.time()
|
| 704 |
try:
|
|
|
|
| 712 |
_odoo._tlocal.client = _odoo.OdooClient()
|
| 713 |
except Exception:
|
| 714 |
pass
|
| 715 |
+
selected_entities = _configured_mirror_entities()
|
| 716 |
res = {}
|
| 717 |
for i in range(12):
|
| 718 |
+
res = _ds.sync_all(entities=selected_entities, log=lambda *a, **k: None)
|
| 719 |
print(f"[aios-api] datastore sync pass {i + 1}: "
|
| 720 |
+ ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
|
| 721 |
if all(v.get("phase") == "live" for v in res.values()):
|
|
|
|
| 726 |
# measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of
|
| 727 |
# phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran.
|
| 728 |
try:
|
| 729 |
+
_ds.reconcile_deletes(entities=selected_entities, log=lambda *a, **k: None)
|
| 730 |
except Exception:
|
| 731 |
pass
|
| 732 |
print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
|
|
|
|
| 1051 |
# transport. Do not borrow a request's cached XML-RPC client.
|
| 1052 |
odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
|
| 1053 |
odoo._tlocal.tenant_ctx = None
|
| 1054 |
+
from connector_onboarding import selected_mirror_entities
|
| 1055 |
+
selected_entities = selected_mirror_entities(rt)
|
| 1056 |
+
report = datastore.sync_all(entities=selected_entities,
|
| 1057 |
+
log=lambda *a, **k: None)
|
| 1058 |
finally:
|
| 1059 |
odoo._tlocal.client = previous_client
|
| 1060 |
odoo._tlocal.tenant_ctx = previous_context
|
| 1061 |
+
if not report and selected_entities != []:
|
| 1062 |
raise _SyncNowRefused(
|
| 1063 |
"Odoo did not run a sync. Check whether the connection is paused.")
|
| 1064 |
if any(row.get("phase") != "live" for row in report.values()):
|
|
|
|
| 1891 |
_startup_notes()
|
| 1892 |
|
| 1893 |
|
| 1894 |
+
def _configured_mirror_entities(tenant=None):
|
| 1895 |
+
from harness import runtime
|
| 1896 |
+
from connector_onboarding import selected_mirror_entities
|
| 1897 |
+
return selected_mirror_entities(runtime.get_runtime(tenant or _resync_tenant()[0]))
|
| 1898 |
+
|
| 1899 |
+
|
| 1900 |
+
def _resync_mirror_pass(manual, passes, entities=None):
|
| 1901 |
"""ONE bounded mirror pass: DuckDB and the Odoo XML-RPC client, and nothing else.
|
| 1902 |
|
| 1903 |
⛔⛔ NO `Store`, NO `store_kv`, NO `user_tables` IN THIS BODY, AND THAT IS A CONTRACT RATHER
|
|
|
|
| 1926 |
# relational rebuild, which `manual` says nothing about.
|
| 1927 |
if manual:
|
| 1928 |
return None, None, None
|
| 1929 |
+
sync_report = _ds.sync_all(entities=entities, log=lambda *a, **k: None)
|
| 1930 |
# Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
|
| 1931 |
# the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
|
| 1932 |
# ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
|
|
|
|
| 1934 |
# HOTFIX 2026-09-29: EVERY pass, not every fourth. At a daily cadence a fourth pass is four
|
| 1935 |
# days, and the first pass after a reseed never reconciled at all. `reconcile_deletes`
|
| 1936 |
# compares counts first and walks ids only for an entity whose count moved.
|
| 1937 |
+
delete_report = _ds.reconcile_deletes(entities=entities, log=lambda *a, **k: None)
|
| 1938 |
return sync_report, delete_report, None
|
| 1939 |
except Exception as e: # noqa: BLE001
|
| 1940 |
# A partial report must never be mistaken for a confirmed zero-change pass.
|
|
|
|
| 2072 |
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
|
| 2073 |
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 2074 |
refreshed pool re-reads the freshly synced store."""
|
| 2075 |
+
if _freeze_holds("store resync"):
|
| 2076 |
+
return
|
| 2077 |
import time as _t
|
| 2078 |
from core import connector_sync
|
| 2079 |
passes = 0
|
|
|
|
| 2211 |
RESYNC_STATE["passes"] = passes
|
| 2212 |
if _odoo_requested is None:
|
| 2213 |
_odoo_requested = _run_sync_requests()
|
| 2214 |
+
try:
|
| 2215 |
+
selected_entities = _configured_mirror_entities(_cad["tenant"])
|
| 2216 |
+
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
|
| 2217 |
+
except Exception as exc:
|
| 2218 |
+
sync_report, delete_report, _err = None, None, _classify_resync_error(exc)
|
| 2219 |
source_fingerprints = _resync_source_fingerprints(sync_report)
|
| 2220 |
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
|
| 2221 |
RESYNC_STATE["lastPassOk"] = _err is None
|
|
|
|
| 2381 |
|
| 2382 |
def _boot_app_migrations(why):
|
| 2383 |
"""Apply tenant App retirement migrations independently of mirror sync and reads."""
|
| 2384 |
+
if _freeze_holds("app migrations"):
|
| 2385 |
+
return
|
| 2386 |
try:
|
| 2387 |
import application_registry
|
| 2388 |
import timeseries_retirement
|
|
|
|
| 2406 |
|
| 2407 |
|
| 2408 |
def _prewarm():
|
| 2409 |
+
if _freeze_holds("prewarm"):
|
| 2410 |
+
return
|
| 2411 |
import time as _t
|
| 2412 |
t0 = _t.time()
|
| 2413 |
# The Odoo seed/sync can run for minutes and is separate from the finite cache warm below.
|
|
|
|
| 2506 |
does not open a PostgreSQL connection merely to learn it should refuse. Live has neither
|
| 2507 |
deployment flag and returns before importing the PostgreSQL backend.
|
| 2508 |
"""
|
| 2509 |
+
if _freeze_holds("QA schema provisioning"):
|
| 2510 |
+
return False
|
| 2511 |
if os.environ.get("AIOS_ENABLE_QA_TENANT") != "1":
|
| 2512 |
return False
|
| 2513 |
granted = {s.strip().lower() for s in
|
|
|
|
| 2543 |
# never blocks health/readiness; the SQL procedure is idempotent, and `_provision_qa_sandbox`
|
| 2544 |
# itself refuses before opening a pool on every other environment.
|
| 2545 |
if (os.environ.get("AIOS_ENABLE_QA_TENANT") == "1"
|
| 2546 |
+
and os.environ.get("STORE_BACKEND") == "pg"
|
| 2547 |
+
and not _freeze_holds("QA schema provisioning")):
|
| 2548 |
import threading as _threading_qa
|
| 2549 |
_threading_qa.Thread(target=_provision_qa_sandbox, daemon=True,
|
| 2550 |
name="qa-sandbox-schema").start()
|
| 2551 |
|
| 2552 |
if (os.environ.get("AIOS_BOOT_APP_MIGRATIONS") != "0"
|
| 2553 |
+
and (data_binding.is_space() or os.environ.get("AIOS_BOOT_APP_MIGRATIONS") == "1")
|
| 2554 |
+
and not _freeze_holds("app migrations")):
|
| 2555 |
import threading as _threading_apps
|
| 2556 |
_threading_apps.Thread(target=_boot_app_migrations, args=("boot",), daemon=True,
|
| 2557 |
name="app-migrations").start()
|
| 2558 |
|
| 2559 |
+
if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
|
| 2560 |
+
_freeze_holds("boot store sync")
|
| 2561 |
+
|
| 2562 |
+
if os.environ.get("AIOS_PREWARM") == "1" and not _freeze_holds("prewarm"):
|
| 2563 |
import threading as _threading
|
| 2564 |
_threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
|
| 2565 |
|
|
|
|
| 2594 |
# The other structural cost remains outside that skip policy:
|
| 2595 |
# * the whole-document store write itself. A partial-JSONB write is a wave.
|
| 2596 |
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
|
| 2597 |
+
if os.environ.get("AIOS_STORE_RESYNC") == "1" and not _freeze_holds("store resync"):
|
| 2598 |
import threading as _threading_resync
|
| 2599 |
_threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
|
api/odoo_purchasing.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Read-only purchasing projections and source reconciliation for the Odoo connector."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import datetime as dt
|
| 5 |
+
import math
|
| 6 |
+
|
| 7 |
+
ORDERS_KEY = "ut_odoo_purchase_orders"
|
| 8 |
+
LINES_KEY = "ut_odoo_purchase_lines"
|
| 9 |
+
CONFIRMED = "po.state IN ('purchase','done')"
|
| 10 |
+
|
| 11 |
+
MIRROR_ENTITIES = {
|
| 12 |
+
"purchase_order": {
|
| 13 |
+
"model": "purchase.order", "archivable": False, "optional": True,
|
| 14 |
+
"fields": ["name", "partner_id", "date_approve", "date_order", "state",
|
| 15 |
+
"amount_untaxed", "amount_total", "currency_id", "write_date"],
|
| 16 |
+
},
|
| 17 |
+
"purchase_order_line": {
|
| 18 |
+
"model": "purchase.order.line", "archivable": False, "optional": True,
|
| 19 |
+
"fields": ["order_id", "product_id", "product_uom", "product_uom_qty", "product_qty",
|
| 20 |
+
"qty_received", "price_unit", "price_subtotal", "date_planned", "write_date"],
|
| 21 |
+
},
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _text(value):
|
| 26 |
+
return str(value or "")
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _id(value):
|
| 30 |
+
return "" if value in (None, False, "") else int(value)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _number(value):
|
| 34 |
+
return float(value or 0)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _date(value):
|
| 38 |
+
return str(value or "")[:10]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _fields(items):
|
| 42 |
+
return [{"source": "overlay", "default": True,
|
| 43 |
+
"automation": {"flowId": "odoo_relational", "preset": True}, **field} for field in items]
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def order_fields():
|
| 47 |
+
return _fields([
|
| 48 |
+
{"key": "order_no", "label": "Purchase order", "type": "text", "pinned": True},
|
| 49 |
+
{"key": "odoo_id", "label": "Odoo ID", "type": "int", "default": False,
|
| 50 |
+
"description": "The `purchase.order` id. Also this row's id."},
|
| 51 |
+
{"key": "vendor", "label": "Vendor", "type": "text"},
|
| 52 |
+
{"key": "vendor_id", "label": "Odoo vendor id", "type": "int", "default": False},
|
| 53 |
+
{"key": "confirmation_date", "label": "Confirmation date", "type": "date"},
|
| 54 |
+
{"key": "order_date", "label": "Order date", "type": "date"},
|
| 55 |
+
{"key": "state", "label": "State", "type": "text"},
|
| 56 |
+
{"key": "currency", "label": "Currency", "type": "text"},
|
| 57 |
+
{"key": "amount_untaxed", "label": "Untaxed amount", "type": "currency", "agg": "sum"},
|
| 58 |
+
{"key": "amount_total", "label": "Total amount", "type": "currency", "agg": "sum"},
|
| 59 |
+
])
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def line_fields():
|
| 63 |
+
return _fields([
|
| 64 |
+
{"key": "order_no", "label": "Purchase order", "type": "text", "pinned": True},
|
| 65 |
+
{"key": "odoo_id", "label": "Odoo ID", "type": "int", "default": False,
|
| 66 |
+
"description": "The `purchase.order.line` id. Also this row's id."},
|
| 67 |
+
{"key": "order_id", "label": "Odoo order id", "type": "int", "default": False},
|
| 68 |
+
{"key": "vendor", "label": "Vendor", "type": "text"},
|
| 69 |
+
{"key": "vendor_id", "label": "Odoo vendor id", "type": "int", "default": False},
|
| 70 |
+
{"key": "confirmation_date", "label": "Confirmation date", "type": "date"},
|
| 71 |
+
{"key": "state", "label": "State", "type": "text"},
|
| 72 |
+
{"key": "currency", "label": "Currency", "type": "text"},
|
| 73 |
+
{"key": "product", "label": "Product", "type": "text"},
|
| 74 |
+
{"key": "product_id", "label": "Odoo product id", "type": "int", "default": False},
|
| 75 |
+
{"key": "sku", "label": "SKU", "type": "text"},
|
| 76 |
+
{"key": "uom", "label": "Product unit of measure", "type": "text"},
|
| 77 |
+
{"key": "purchase_uom", "label": "Purchase unit of measure", "type": "text"},
|
| 78 |
+
{"key": "standard_price", "label": "Standard cost", "type": "currency"},
|
| 79 |
+
{"key": "category", "label": "Category", "type": "text"},
|
| 80 |
+
{"key": "qty", "label": "Quantity in product units", "type": "int", "agg": "sum"},
|
| 81 |
+
{"key": "purchase_qty", "label": "Quantity in purchase units", "type": "int", "agg": "sum"},
|
| 82 |
+
{"key": "received_qty", "label": "Received in purchase units", "type": "int", "agg": "sum"},
|
| 83 |
+
{"key": "price_unit", "label": "Unit price", "type": "currency"},
|
| 84 |
+
{"key": "price_subtotal", "label": "Untaxed amount", "type": "currency", "agg": "sum"},
|
| 85 |
+
{"key": "planned_date", "label": "Planned date", "type": "date"},
|
| 86 |
+
])
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def source_specs():
|
| 90 |
+
common = {
|
| 91 |
+
"order_no": ("po.name", _text), "vendor": ("po.partner_name", _text),
|
| 92 |
+
"vendor_id": ("po.partner_id", _id), "confirmation_date": ("po.date_approve", _date),
|
| 93 |
+
"state": ("po.state", _text), "currency": ("po.currency_name", _text),
|
| 94 |
+
}
|
| 95 |
+
return {
|
| 96 |
+
ORDERS_KEY: {
|
| 97 |
+
"from_sql": "purchase_order po", "tables": {"po": "purchase_order"},
|
| 98 |
+
"where": CONFIRMED, "id": "po.id",
|
| 99 |
+
"cols": {**common, "odoo_id": ("po.id", _id), "order_date": ("po.date_order", _date),
|
| 100 |
+
"amount_untaxed": ("po.amount_untaxed", _number),
|
| 101 |
+
"amount_total": ("po.amount_total", _number)},
|
| 102 |
+
},
|
| 103 |
+
LINES_KEY: {
|
| 104 |
+
"from_sql": "(purchase_order_line pol JOIN purchase_order po ON po.id = pol.order_id "
|
| 105 |
+
"LEFT JOIN product_product pp ON pp.id = pol.product_id)",
|
| 106 |
+
"tables": {"pol": "purchase_order_line", "po": "purchase_order", "pp": "product_product"},
|
| 107 |
+
"where": CONFIRMED, "id": "pol.id",
|
| 108 |
+
"cols": {**common, "odoo_id": ("pol.id", _id), "order_id": ("pol.order_id", _id),
|
| 109 |
+
"product": ("pol.product_name", _text), "product_id": ("pol.product_id", _id),
|
| 110 |
+
"sku": ("pp.default_code", _text), "uom": ("pp.uom_name", _text),
|
| 111 |
+
"purchase_uom": ("pol.product_uom_name", _text),
|
| 112 |
+
"standard_price": ("pp.standard_price", _number), "category": ("pp.categ_name", _text),
|
| 113 |
+
"qty": ("pol.product_uom_qty", _number), "purchase_qty": ("pol.product_qty", _number),
|
| 114 |
+
"received_qty": ("pol.qty_received", _number), "price_unit": ("pol.price_unit", _number),
|
| 115 |
+
"price_subtotal": ("pol.price_subtotal", _number), "planned_date": ("pol.date_planned", _date)},
|
| 116 |
+
},
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _row_totals(client, model, domain, field):
|
| 121 |
+
last, count, total = 0, 0, 0.0
|
| 122 |
+
while True:
|
| 123 |
+
rows = client.search_read(model, domain + [("id", ">", last)], ["id", field],
|
| 124 |
+
limit=2000, order="id asc")
|
| 125 |
+
if not rows:
|
| 126 |
+
return count, total
|
| 127 |
+
values = []
|
| 128 |
+
for row in rows:
|
| 129 |
+
rid = int(row["id"])
|
| 130 |
+
value = float(row.get(field) or 0)
|
| 131 |
+
if rid <= last or not math.isfinite(value):
|
| 132 |
+
raise ValueError("purchasing validation source did not advance or returned a non-finite amount")
|
| 133 |
+
last = rid
|
| 134 |
+
values.append(value)
|
| 135 |
+
count += len(rows)
|
| 136 |
+
total = math.fsum([total, *values])
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def validate(client, cur=None, month="2026-06"):
|
| 140 |
+
"""Tie a closed confirmation month to independent Odoo line aggregates, without printing totals.
|
| 141 |
+
|
| 142 |
+
A supplied cursor must be admitted for the same tenant/connection as the read-only client.
|
| 143 |
+
Without it, this validates live row totals only and makes no production-mirror claim.
|
| 144 |
+
"""
|
| 145 |
+
start = dt.date.fromisoformat(month + "-01")
|
| 146 |
+
end = (start.replace(day=28) + dt.timedelta(days=4)).replace(day=1)
|
| 147 |
+
if end > dt.date.today().replace(day=1):
|
| 148 |
+
raise ValueError("purchasing validation requires a closed month")
|
| 149 |
+
checks, totals = [], {}
|
| 150 |
+
plane = "mirror" if cur is not None else "live_rows"
|
| 151 |
+
periods = [("date_approve", ">=", start.isoformat()), ("date_approve", "<", end.isoformat())]
|
| 152 |
+
for key, model, field in ((ORDERS_KEY, "purchase.order", "amount_untaxed"),
|
| 153 |
+
(LINES_KEY, "purchase.order.line", "price_subtotal")):
|
| 154 |
+
prefix = "order_id." if key == LINES_KEY else ""
|
| 155 |
+
domain = [(prefix + "state", "in", ["purchase", "done"])]
|
| 156 |
+
domain += [(prefix + f, op, value) for f, op, value in periods]
|
| 157 |
+
try:
|
| 158 |
+
if cur is None:
|
| 159 |
+
count, total = _row_totals(client, model, domain, field)
|
| 160 |
+
else:
|
| 161 |
+
spec = source_specs()[key]
|
| 162 |
+
count, distinct, total = cur.execute(
|
| 163 |
+
"SELECT COUNT(*), COUNT(DISTINCT " + spec["id"] + "), COALESCE(SUM("
|
| 164 |
+
+ spec["cols"][field][0] + "), 0) FROM " + spec["from_sql"] + " WHERE "
|
| 165 |
+
+ spec["where"] + " AND po.date_approve >= ? AND po.date_approve < ?",
|
| 166 |
+
[start.isoformat(), end.isoformat()]).fetchone()
|
| 167 |
+
if count != distinct:
|
| 168 |
+
raise ValueError("purchasing projection duplicated identities")
|
| 169 |
+
expected = int(client.search_count(model, domain))
|
| 170 |
+
groups = client.read_group(model, domain, [field + ":sum"],
|
| 171 |
+
["order_id" if key == LINES_KEY else "partner_id"], lazy=False)
|
| 172 |
+
summary = math.fsum(float(row.get(field) or 0) for row in groups)
|
| 173 |
+
checks.append({"check": key + " complete population", "ok": count == expected,
|
| 174 |
+
"observation": plane, "observed_rows": count, "source_rows": expected})
|
| 175 |
+
checks.append({"check": key + " vs Odoo read_group",
|
| 176 |
+
"ok": math.isfinite(total) and math.isfinite(summary)
|
| 177 |
+
and abs(total - summary) <= 0.01,
|
| 178 |
+
"observation": plane})
|
| 179 |
+
totals[key] = (total, summary, count)
|
| 180 |
+
except Exception as exc:
|
| 181 |
+
checks.append({"check": key + " reconciliation", "ok": False,
|
| 182 |
+
"error_type": type(exc).__name__, "observation": plane})
|
| 183 |
+
if len(totals) == 2:
|
| 184 |
+
checks.append({"check": "confirmed purchase orders vs independent Odoo line total",
|
| 185 |
+
"ok": totals[ORDERS_KEY][2] > 0 and totals[LINES_KEY][2] > 0
|
| 186 |
+
and abs(totals[ORDERS_KEY][0] - totals[LINES_KEY][1]) <= 0.01,
|
| 187 |
+
"observation": plane})
|
| 188 |
+
return checks
|
api/odoo_relational.py
CHANGED
|
@@ -91,6 +91,14 @@ VENDORS_KEY = "ut_odoo_vendors"
|
|
| 91 |
#: grid; that is the whole shape of the conversion.
|
| 92 |
ORDER_LINES_KEY = "ut_odoo_order_lines"
|
| 93 |
GL_LINES_KEY = "ut_odoo_gl_lines"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
|
| 95 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 96 |
JOIN_KEY = "partner_id"
|
|
@@ -166,6 +174,7 @@ def _scope_field():
|
|
| 166 |
|
| 167 |
|
| 168 |
def _refreshed_field():
|
|
|
|
| 169 |
return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay",
|
| 170 |
"default": False, "description": "When this row was last reconciled against Odoo."}
|
| 171 |
|
|
@@ -713,6 +722,69 @@ def gl_line_fields():
|
|
| 713 |
)]
|
| 714 |
|
| 715 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 716 |
def product_fields():
|
| 717 |
"""One row per `product.product`, keyed on its id — EVERY product, archived ones included.
|
| 718 |
|
|
@@ -1886,6 +1958,12 @@ TABLES = (
|
|
| 1886 |
# wave's own [[reachable-is-not-the-same-as-built]] shape.
|
| 1887 |
("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields),
|
| 1888 |
("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1889 |
)
|
| 1890 |
|
| 1891 |
# The grid declarations above are the stable mirror contract, not Odoo's schema. T03 snapshots
|
|
@@ -1895,10 +1973,14 @@ ODOO_TABLE_MODELS = {
|
|
| 1895 |
INVOICES_KEY: "account.move", ORDERS_KEY: "sale.order", AGENTS_KEY: "res.partner",
|
| 1896 |
ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move",
|
| 1897 |
ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line",
|
|
|
|
|
|
|
|
|
|
| 1898 |
}
|
| 1899 |
# Shipments deliberately stay outside this map and TABLES: its consumers route mapped models
|
| 1900 |
# through the DuckDB refresh and suppress their discovered-model reads. There is no picking
|
| 1901 |
# mirror. The live shipment writer below stamps its model directly on the stored connector.
|
|
|
|
| 1902 |
|
| 1903 |
|
| 1904 |
def authoritative_schema_snapshot():
|
|
@@ -1913,7 +1995,7 @@ def authoritative_schema_snapshot():
|
|
| 1913 |
seen.add(model)
|
| 1914 |
try:
|
| 1915 |
models[model] = get_odoo().fields_get(
|
| 1916 |
-
model, attributes=
|
| 1917 |
except Exception as exc:
|
| 1918 |
errors.append(f"{model}: {type(exc).__name__}: {exc}")
|
| 1919 |
return models, errors
|
|
@@ -1930,10 +2012,45 @@ MODEL_PUBLISH_SECONDS = 10
|
|
| 1930 |
MODEL_CATALOG_TTL = _dt.timedelta(hours=6)
|
| 1931 |
_MODEL_FIELD_TYPES = {
|
| 1932 |
"boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
|
| 1933 |
-
"date": "date", "datetime": "datetime",
|
| 1934 |
}
|
| 1935 |
|
| 1936 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1937 |
def discover_readable_models(client):
|
| 1938 |
"""Read-only introspection over an injected Odoo client.
|
| 1939 |
|
|
@@ -1962,7 +2079,7 @@ def discover_readable_models(client):
|
|
| 1962 |
continue
|
| 1963 |
try:
|
| 1964 |
fields = client.fields_get(
|
| 1965 |
-
model, attributes=
|
| 1966 |
except Exception as exc: # noqa: BLE001
|
| 1967 |
fields = {}
|
| 1968 |
errors.append(f"{model}: fields: {type(exc).__name__}: {exc}")
|
|
@@ -1970,9 +2087,7 @@ def discover_readable_models(client):
|
|
| 1970 |
for name, meta in (fields or {}).items():
|
| 1971 |
if not str(name).strip() or not isinstance(meta, dict):
|
| 1972 |
continue
|
| 1973 |
-
clean_fields[str(name)] = {k: meta.get(k) for k in
|
| 1974 |
-
("string", "type", "required", "readonly", "relation")
|
| 1975 |
-
if k in meta}
|
| 1976 |
rows.append({"model": model,
|
| 1977 |
"name": " ".join(str((item or {}).get("name") or model).split()),
|
| 1978 |
"fields": clean_fields})
|
|
@@ -2005,7 +2120,8 @@ def readable_model_catalog(rt, connection_id=None):
|
|
| 2005 |
def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False):
|
| 2006 |
"""Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure."""
|
| 2007 |
current = readable_model_catalog(rt, connection_id)
|
| 2008 |
-
if current["capturedAt"] and not force
|
|
|
|
| 2009 |
try:
|
| 2010 |
captured = _dt.datetime.fromisoformat(current["capturedAt"])
|
| 2011 |
if _dt.datetime.now(_dt.timezone.utc) - captured < MODEL_CATALOG_TTL:
|
|
@@ -2063,14 +2179,12 @@ def refresh_model_catalog_entry(rt, model, connection_id=None, client=None):
|
|
| 2063 |
if not bool(readable):
|
| 2064 |
return None
|
| 2065 |
fields = bound_client.fields_get(
|
| 2066 |
-
model, attributes=
|
| 2067 |
clean_fields = {}
|
| 2068 |
for name, meta in (fields if isinstance(fields, dict) else {}).items():
|
| 2069 |
if not str(name).strip() or not isinstance(meta, dict):
|
| 2070 |
continue
|
| 2071 |
-
clean_fields[str(name)] = {k: meta.get(k) for k in
|
| 2072 |
-
("string", "type", "required", "readonly", "relation")
|
| 2073 |
-
if k in meta}
|
| 2074 |
return clean_fields
|
| 2075 |
|
| 2076 |
try:
|
|
@@ -2539,6 +2653,17 @@ ODOO_SPEC = (
|
|
| 2539 |
"links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),
|
| 2540 |
("move_id", INVOICES_KEY, _ROW_ID_COL, _M2O),
|
| 2541 |
(ACCOUNT_JOIN_KEY, ACCOUNTS_KEY, ACCOUNT_JOIN_KEY, _M2O))},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2542 |
)
|
| 2543 |
|
| 2544 |
|
|
@@ -2826,6 +2951,11 @@ TABLE_SUBJECTS = {
|
|
| 2826 |
BILLS_KEY: "odoo:account.move.vendor",
|
| 2827 |
ORDER_LINES_KEY: "odoo:sale.order.line",
|
| 2828 |
GL_LINES_KEY: "odoo:account.move.line",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2829 |
}
|
| 2830 |
|
| 2831 |
# One receipt identity per curated database. The model is not enough: invoices/bills share
|
|
@@ -2840,6 +2970,11 @@ TABLE_TOPICS = {
|
|
| 2840 |
BILLS_KEY: "odoo_bills",
|
| 2841 |
ORDER_LINES_KEY: "sales_lines",
|
| 2842 |
GL_LINES_KEY: "gl_lines",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2843 |
}
|
| 2844 |
|
| 2845 |
_SOURCE_COUNT_SPECS = {
|
|
@@ -2860,6 +2995,14 @@ _SOURCE_COUNT_SPECS = {
|
|
| 2860 |
"order_lines": {"model": "sale.order.line", "mode": "count", "domain": [
|
| 2861 |
("order_id.state", "in", ["sale", "done"])]},
|
| 2862 |
"gl_lines": {"model": "account.move.line", "mode": "count", "domain": []},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2863 |
}
|
| 2864 |
|
| 2865 |
# The two canonical registry databases replaced the retired ``ut_odoo_customers`` and
|
|
@@ -3174,6 +3317,271 @@ _READERS = {
|
|
| 3174 |
READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS)
|
| 3175 |
|
| 3176 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3177 |
# ═════════════════════════════════════════════════════════════════════════════════════════════
|
| 3178 |
# ⭐⭐ W54-T46 / R4 / R7 / CONTRACT C4 — A TABLE BELONGS TO EXACTLY ONE CONNECTION
|
| 3179 |
# ═════════════════════════════════════════════════════════════════════════════════════════════
|
|
@@ -4249,9 +4657,11 @@ def apply_plan(rt, built, username="automation", today=None, report=None,
|
|
| 4249 |
# keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does.
|
| 4250 |
retired = []
|
| 4251 |
conn_label = connection_label(rt, connection_id)
|
|
|
|
| 4252 |
plans = [(table_key_for(key, connection_id),
|
| 4253 |
table_label_for(label, conn_label),
|
| 4254 |
-
namespaced_fields(fields(), connection_id),
|
|
|
|
| 4255 |
built[bucket])
|
| 4256 |
for bucket, key, label, fields in TABLES if bucket in built]
|
| 4257 |
|
|
@@ -4380,6 +4790,17 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
|
|
| 4380 |
# an empty grid.
|
| 4381 |
if is_read_through_key(key):
|
| 4382 |
table["readThrough"] = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4383 |
# W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same
|
| 4384 |
# reason `readThrough` is: it is derived from this module's own structure, so the code is what
|
| 4385 |
# it has to agree with, and a definition that lost the stamp should regain it rather than go
|
|
@@ -4425,8 +4846,13 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
|
|
| 4425 |
if _ut().user_edited(stored):
|
| 4426 |
continue
|
| 4427 |
if (stored.get("automation") or {}).get("preset"):
|
| 4428 |
-
for prop in ("label", "type", "options", "link", "rollup", "description",
|
| 4429 |
"agg", "pinned", "default"):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4430 |
if prop in field:
|
| 4431 |
stored[prop] = field[prop]
|
| 4432 |
else:
|
|
@@ -4465,6 +4891,8 @@ def _model_field_specs(catalogue_row):
|
|
| 4465 |
"label": str(meta.get("string") or raw_name),
|
| 4466 |
"type": _MODEL_FIELD_TYPES.get(raw_type, "text"),
|
| 4467 |
"source": "odoo",
|
|
|
|
|
|
|
| 4468 |
**({"description": "Binary content is not loaded into this database."}
|
| 4469 |
if raw_type == "binary" else {}),
|
| 4470 |
"automation": {"preset": True}})
|
|
@@ -4482,7 +4910,7 @@ def _model_field_specs(catalogue_row):
|
|
| 4482 |
def publish_database_schema(rt, table_key, connection_id=None, username="automation", today=None):
|
| 4483 |
"""Publish one permission-visible database definition without row work.
|
| 4484 |
|
| 4485 |
-
Make at most two source calls on one model
|
| 4486 |
Enable owns the choice and a durable sync job owns materialization. This seam only merges the
|
| 4487 |
schema, so retrying it preserves existing rows, authored fields and the navigation rename.
|
| 4488 |
"""
|
|
@@ -4504,15 +4932,17 @@ def publish_database_schema(rt, table_key, connection_id=None, username="automat
|
|
| 4504 |
raw = (_ut().odoo_schema_snapshot(rt).get("models") or {}).get(model)
|
| 4505 |
if isinstance(raw, dict):
|
| 4506 |
row = {"model": model, "fields": raw}
|
| 4507 |
-
if not row or not row.get("fields"):
|
| 4508 |
catalogue = refresh_model_catalog_entry(rt, model, connection_id=namespace)
|
| 4509 |
row = next((row for row in catalogue["models"] if row.get("model") == model), None)
|
| 4510 |
if not row or not row.get("fields"):
|
| 4511 |
raise Refused("the field catalogue is unavailable for this database. Refresh the catalogue first.")
|
|
|
|
|
|
|
| 4512 |
raw_fields = _model_field_specs(row)
|
| 4513 |
declared = next((fields() for _bucket, known_key, _label, fields in TABLES
|
| 4514 |
if known_key == (base or table_key)), [])
|
| 4515 |
-
fields = namespaced_fields(declared, namespace)
|
| 4516 |
have = {field["key"] for field in fields}
|
| 4517 |
fields.extend(field for field in raw_fields if field["key"] not in have)
|
| 4518 |
from core import connector_sync
|
|
@@ -4834,8 +5264,8 @@ def reconciliation_drift(receipt):
|
|
| 4834 |
def blocking_failures(database_failures, report):
|
| 4835 |
"""The database failures that make a sync NOT good, PURE.
|
| 4836 |
|
| 4837 |
-
A reconciliation failure
|
| 4838 |
-
|
| 4839 |
"""
|
| 4840 |
receipts = {}
|
| 4841 |
for name in ("databases", "canonicalDatabases"):
|
|
@@ -4845,8 +5275,10 @@ def blocking_failures(database_failures, report):
|
|
| 4845 |
out = {}
|
| 4846 |
for key, failure in (database_failures or {}).items():
|
| 4847 |
if isinstance(failure, dict) and failure.get("code") == "reconciliation_incomplete":
|
| 4848 |
-
|
| 4849 |
-
|
|
|
|
|
|
|
| 4850 |
continue
|
| 4851 |
out[key] = failure
|
| 4852 |
return out
|
|
@@ -4982,6 +5414,13 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 4982 |
return built, written, errors, receipts, report
|
| 4983 |
|
| 4984 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4985 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
| 4986 |
all_connections=False, table_keys=None, on_apply=None):
|
| 4987 |
"""THE entry point — the store-resync path and the route both call this.
|
|
@@ -5073,8 +5512,8 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 5073 |
if ns is not None:
|
| 5074 |
cur = None
|
| 5075 |
elif cur is None:
|
| 5076 |
-
from
|
| 5077 |
-
cur =
|
| 5078 |
requested = set(table_keys) if table_keys is not None else None
|
| 5079 |
if requested is not None:
|
| 5080 |
for key in requested:
|
|
@@ -5098,7 +5537,7 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 5098 |
built, written, database_errors, database_receipts, plan_report = refresh_curated(
|
| 5099 |
rt, cur, keep, connection_id=ns, username=username, today=today,
|
| 5100 |
retire=requested is None, on_apply=on_apply)
|
| 5101 |
-
if ns is None and requested is None and cur is not None:
|
| 5102 |
canonical_receipts = canonical_reconciliation_receipts(cur)
|
| 5103 |
else:
|
| 5104 |
# The binding selects this connection's verified read-only client. Authentication failure
|
|
@@ -5112,7 +5551,7 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 5112 |
built, written, database_errors, database_receipts, plan_report = refresh_curated(
|
| 5113 |
rt, cur, keep, connection_id=ns, username=username, today=today,
|
| 5114 |
retire=requested is None, on_apply=on_apply, source_client=source_client)
|
| 5115 |
-
if ns is None and requested is None and cur is not None:
|
| 5116 |
canonical_receipts = canonical_reconciliation_receipts(
|
| 5117 |
cur, source_client=source_client)
|
| 5118 |
# ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing.
|
|
|
|
| 91 |
#: grid; that is the whole shape of the conversion.
|
| 92 |
ORDER_LINES_KEY = "ut_odoo_order_lines"
|
| 93 |
GL_LINES_KEY = "ut_odoo_gl_lines"
|
| 94 |
+
STOCK_MOVES_KEY = "ut_odoo_stock_moves"
|
| 95 |
+
STOCK_VALUATION_KEY = "ut_odoo_stock_valuation"
|
| 96 |
+
STOCK_QUANT_KEY = "ut_odoo_stock_quant"
|
| 97 |
+
from odoo_purchasing import (ORDERS_KEY as PURCHASE_ORDERS_KEY, LINES_KEY as PURCHASE_LINES_KEY,
|
| 98 |
+
order_fields as purchase_order_fields,
|
| 99 |
+
line_fields as purchase_line_fields,
|
| 100 |
+
source_specs as purchase_source_specs,
|
| 101 |
+
validate as validate_purchasing)
|
| 102 |
|
| 103 |
#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
|
| 104 |
JOIN_KEY = "partner_id"
|
|
|
|
| 174 |
|
| 175 |
|
| 176 |
def _refreshed_field():
|
| 177 |
+
"""Materialized row stamp; the definition writer omits it on read-through tables."""
|
| 178 |
return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay",
|
| 179 |
"default": False, "description": "When this row was last reconciled against Odoo."}
|
| 180 |
|
|
|
|
| 722 |
)]
|
| 723 |
|
| 724 |
|
| 725 |
+
def _stock_product_fields():
|
| 726 |
+
"""D-2632 product master columns, joined by product id including archived products."""
|
| 727 |
+
return [
|
| 728 |
+
{"key": "product", "label": "Product", "type": "text", "default": True},
|
| 729 |
+
{"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "default": False},
|
| 730 |
+
{"key": "sku", "label": "SKU", "type": "text", "default": True},
|
| 731 |
+
{"key": "uom", "label": "Product unit of measure", "type": "text", "default": True},
|
| 732 |
+
{"key": "standard_price", "label": "Standard cost", "type": "currency", "default": True},
|
| 733 |
+
{"key": "category", "label": "Category", "type": "text", "default": True},
|
| 734 |
+
]
|
| 735 |
+
|
| 736 |
+
|
| 737 |
+
def _stock_location_fields(prefix, label):
|
| 738 |
+
return [
|
| 739 |
+
{"key": prefix + "_id", "label": label + " id", "type": "int", "default": False},
|
| 740 |
+
{"key": prefix, "label": label, "type": "text", "default": True},
|
| 741 |
+
{"key": prefix + "_usage", "label": label + " usage", "type": "text", "default": True},
|
| 742 |
+
{"key": prefix + "_fba", "label": label + " FBA", "type": "checkbox", "default": True},
|
| 743 |
+
]
|
| 744 |
+
|
| 745 |
+
|
| 746 |
+
def stock_move_fields():
|
| 747 |
+
return [_preset({"source": "overlay", **field}) for field in (
|
| 748 |
+
[{"key": "move", "label": "Stock move", "type": "text", "pinned": True, "default": True},
|
| 749 |
+
{"key": "odoo_id", "label": "Odoo ID", "type": "int", "default": False,
|
| 750 |
+
"description": "The `stock.move` id. Also this row's id."}]
|
| 751 |
+
+ _stock_product_fields()
|
| 752 |
+
+ [{"key": "qty", "label": "Moved quantity", "type": "int", "agg": "sum", "default": True,
|
| 753 |
+
"description": "Quantity completed in the move's unit of measure."},
|
| 754 |
+
{"key": "move_uom", "label": "Move unit of measure", "type": "text", "default": True},
|
| 755 |
+
{"key": "date", "label": "Date", "type": "date", "default": True},
|
| 756 |
+
{"key": "state", "label": "State", "type": "text", "default": True},
|
| 757 |
+
{"key": "picking_id", "label": "Odoo picking id", "type": "int", "default": False}]
|
| 758 |
+
+ _stock_location_fields("source_location", "Source location")
|
| 759 |
+
+ _stock_location_fields("destination_location", "Destination location")
|
| 760 |
+
)]
|
| 761 |
+
|
| 762 |
+
|
| 763 |
+
def stock_valuation_fields():
|
| 764 |
+
return [_preset({"source": "overlay", **field}) for field in (
|
| 765 |
+
[{"key": "odoo_id", "label": "Odoo ID", "type": "int", "pinned": True, "default": True,
|
| 766 |
+
"description": "The `stock.valuation.layer` id. Also this row's id."}]
|
| 767 |
+
+ _stock_product_fields()
|
| 768 |
+
+ [{"key": "qty", "label": "Quantity", "type": "int", "agg": "sum", "default": True},
|
| 769 |
+
{"key": "value", "label": "Value", "type": "currency", "agg": "sum", "default": True},
|
| 770 |
+
{"key": "date", "label": "Date", "type": "date", "default": True},
|
| 771 |
+
{"key": "stock_move_id", "label": "Odoo stock move id", "type": "int", "default": False}]
|
| 772 |
+
+ _stock_location_fields("source_location", "Source location")
|
| 773 |
+
+ _stock_location_fields("destination_location", "Destination location")
|
| 774 |
+
)]
|
| 775 |
+
|
| 776 |
+
|
| 777 |
+
def stock_quant_fields():
|
| 778 |
+
return [_preset({"source": "overlay", **field}) for field in (
|
| 779 |
+
[{"key": "odoo_id", "label": "Odoo ID", "type": "int", "pinned": True, "default": True,
|
| 780 |
+
"description": "The `stock.quant` id. Also this row's id."}]
|
| 781 |
+
+ _stock_product_fields() + _stock_location_fields("location", "Location")
|
| 782 |
+
+ [{"key": "qty", "label": "On hand", "type": "int", "agg": "sum", "default": True},
|
| 783 |
+
{"key": "reserved_qty", "label": "Reserved quantity", "type": "int",
|
| 784 |
+
"agg": "sum", "default": True}]
|
| 785 |
+
)]
|
| 786 |
+
|
| 787 |
+
|
| 788 |
def product_fields():
|
| 789 |
"""One row per `product.product`, keyed on its id — EVERY product, archived ones included.
|
| 790 |
|
|
|
|
| 1958 |
# wave's own [[reachable-is-not-the-same-as-built]] shape.
|
| 1959 |
("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields),
|
| 1960 |
("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields),
|
| 1961 |
+
# S70-T19: definitions only. Every stock population stays in the windowed mirror.
|
| 1962 |
+
("stock_moves", STOCK_MOVES_KEY, "Odoo stock moves", stock_move_fields),
|
| 1963 |
+
("stock_valuation", STOCK_VALUATION_KEY, "Odoo stock valuation", stock_valuation_fields),
|
| 1964 |
+
("stock_quant", STOCK_QUANT_KEY, "Odoo stock on hand", stock_quant_fields),
|
| 1965 |
+
("purchase_orders", PURCHASE_ORDERS_KEY, "Odoo purchase orders", purchase_order_fields),
|
| 1966 |
+
("purchase_lines", PURCHASE_LINES_KEY, "Odoo purchase order lines", purchase_line_fields),
|
| 1967 |
)
|
| 1968 |
|
| 1969 |
# The grid declarations above are the stable mirror contract, not Odoo's schema. T03 snapshots
|
|
|
|
| 1973 |
INVOICES_KEY: "account.move", ORDERS_KEY: "sale.order", AGENTS_KEY: "res.partner",
|
| 1974 |
ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move",
|
| 1975 |
ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line",
|
| 1976 |
+
STOCK_MOVES_KEY: "stock.move", STOCK_VALUATION_KEY: "stock.valuation.layer",
|
| 1977 |
+
STOCK_QUANT_KEY: "stock.quant",
|
| 1978 |
+
PURCHASE_ORDERS_KEY: "purchase.order", PURCHASE_LINES_KEY: "purchase.order.line",
|
| 1979 |
}
|
| 1980 |
# Shipments deliberately stay outside this map and TABLES: its consumers route mapped models
|
| 1981 |
# through the DuckDB refresh and suppress their discovered-model reads. There is no picking
|
| 1982 |
# mirror. The live shipment writer below stamps its model directly on the stored connector.
|
| 1983 |
+
_ODOO_FIELD_ATTRIBUTES = ("string", "type", "required", "readonly", "relation", "selection")
|
| 1984 |
|
| 1985 |
|
| 1986 |
def authoritative_schema_snapshot():
|
|
|
|
| 1995 |
seen.add(model)
|
| 1996 |
try:
|
| 1997 |
models[model] = get_odoo().fields_get(
|
| 1998 |
+
model, attributes=_ODOO_FIELD_ATTRIBUTES)
|
| 1999 |
except Exception as exc:
|
| 2000 |
errors.append(f"{model}: {type(exc).__name__}: {exc}")
|
| 2001 |
return models, errors
|
|
|
|
| 2012 |
MODEL_CATALOG_TTL = _dt.timedelta(hours=6)
|
| 2013 |
_MODEL_FIELD_TYPES = {
|
| 2014 |
"boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
|
| 2015 |
+
"date": "date", "datetime": "datetime", "selection": "select",
|
| 2016 |
}
|
| 2017 |
|
| 2018 |
|
| 2019 |
+
def _selection_options(meta):
|
| 2020 |
+
"""Odoo selection keys, not display labels or the values on a sampled page."""
|
| 2021 |
+
selection = meta.get("selection") if isinstance(meta, dict) else None
|
| 2022 |
+
out = []
|
| 2023 |
+
if isinstance(selection, (list, tuple)):
|
| 2024 |
+
for pair in selection:
|
| 2025 |
+
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
| 2026 |
+
continue
|
| 2027 |
+
value = pair[0]
|
| 2028 |
+
if isinstance(value, bool) or not isinstance(value, (str, int)):
|
| 2029 |
+
continue
|
| 2030 |
+
key = str(value)
|
| 2031 |
+
if key and key not in out:
|
| 2032 |
+
out.append(key)
|
| 2033 |
+
return out
|
| 2034 |
+
|
| 2035 |
+
|
| 2036 |
+
def _missing_selection_metadata(row):
|
| 2037 |
+
fields = (row or {}).get("fields") or {}
|
| 2038 |
+
return any(isinstance(meta, dict) and meta.get("type") == "selection"
|
| 2039 |
+
and not _selection_options(meta) for meta in fields.values())
|
| 2040 |
+
|
| 2041 |
+
|
| 2042 |
+
def _selection_field_contracts(fields, catalogue_row):
|
| 2043 |
+
"""Enrich curated columns without changing their row source, links or namespace."""
|
| 2044 |
+
metadata = (catalogue_row or {}).get("fields") or {}
|
| 2045 |
+
out = []
|
| 2046 |
+
for field in fields:
|
| 2047 |
+
meta = metadata.get(field["key"]) or {}
|
| 2048 |
+
options = _selection_options(meta) if meta.get("type") == "selection" else []
|
| 2049 |
+
out.append({**field, "type": "select", "options": options, "choicesFrom": "odoo"}
|
| 2050 |
+
if options else dict(field))
|
| 2051 |
+
return out
|
| 2052 |
+
|
| 2053 |
+
|
| 2054 |
def discover_readable_models(client):
|
| 2055 |
"""Read-only introspection over an injected Odoo client.
|
| 2056 |
|
|
|
|
| 2079 |
continue
|
| 2080 |
try:
|
| 2081 |
fields = client.fields_get(
|
| 2082 |
+
model, attributes=_ODOO_FIELD_ATTRIBUTES)
|
| 2083 |
except Exception as exc: # noqa: BLE001
|
| 2084 |
fields = {}
|
| 2085 |
errors.append(f"{model}: fields: {type(exc).__name__}: {exc}")
|
|
|
|
| 2087 |
for name, meta in (fields or {}).items():
|
| 2088 |
if not str(name).strip() or not isinstance(meta, dict):
|
| 2089 |
continue
|
| 2090 |
+
clean_fields[str(name)] = {k: meta.get(k) for k in _ODOO_FIELD_ATTRIBUTES if k in meta}
|
|
|
|
|
|
|
| 2091 |
rows.append({"model": model,
|
| 2092 |
"name": " ".join(str((item or {}).get("name") or model).split()),
|
| 2093 |
"fields": clean_fields})
|
|
|
|
| 2120 |
def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False):
|
| 2121 |
"""Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure."""
|
| 2122 |
current = readable_model_catalog(rt, connection_id)
|
| 2123 |
+
if (current["capturedAt"] and not force
|
| 2124 |
+
and not any(_missing_selection_metadata(row) for row in current["models"])):
|
| 2125 |
try:
|
| 2126 |
captured = _dt.datetime.fromisoformat(current["capturedAt"])
|
| 2127 |
if _dt.datetime.now(_dt.timezone.utc) - captured < MODEL_CATALOG_TTL:
|
|
|
|
| 2179 |
if not bool(readable):
|
| 2180 |
return None
|
| 2181 |
fields = bound_client.fields_get(
|
| 2182 |
+
model, attributes=_ODOO_FIELD_ATTRIBUTES)
|
| 2183 |
clean_fields = {}
|
| 2184 |
for name, meta in (fields if isinstance(fields, dict) else {}).items():
|
| 2185 |
if not str(name).strip() or not isinstance(meta, dict):
|
| 2186 |
continue
|
| 2187 |
+
clean_fields[str(name)] = {k: meta.get(k) for k in _ODOO_FIELD_ATTRIBUTES if k in meta}
|
|
|
|
|
|
|
| 2188 |
return clean_fields
|
| 2189 |
|
| 2190 |
try:
|
|
|
|
| 2653 |
"links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),
|
| 2654 |
("move_id", INVOICES_KEY, _ROW_ID_COL, _M2O),
|
| 2655 |
(ACCOUNT_JOIN_KEY, ACCOUNTS_KEY, ACCOUNT_JOIN_KEY, _M2O))},
|
| 2656 |
+
{"key": STOCK_MOVES_KEY, "source": "stock_move", "label": "Odoo stock moves",
|
| 2657 |
+
"pinned": "move", "identity": (_ROW_ID_COL, "odoo_id"), "links": ()},
|
| 2658 |
+
{"key": STOCK_VALUATION_KEY, "source": "stock_valuation_layer", "label": "Odoo stock valuation",
|
| 2659 |
+
"pinned": "odoo_id", "identity": (_ROW_ID_COL, "odoo_id"), "links": ()},
|
| 2660 |
+
{"key": STOCK_QUANT_KEY, "source": "stock_quant", "label": "Odoo stock on hand",
|
| 2661 |
+
"pinned": "odoo_id", "identity": (_ROW_ID_COL, "odoo_id"), "links": ()},
|
| 2662 |
+
{"key": PURCHASE_ORDERS_KEY, "source": "purchase_order", "label": "Odoo purchase orders",
|
| 2663 |
+
"pinned": "order_no", "identity": (_ROW_ID_COL, "odoo_id"), "links": ()},
|
| 2664 |
+
{"key": PURCHASE_LINES_KEY, "source": "purchase_order_line", "label": "Odoo purchase order lines",
|
| 2665 |
+
"pinned": "order_no", "identity": (_ROW_ID_COL, "odoo_id"),
|
| 2666 |
+
"links": (("order_id", PURCHASE_ORDERS_KEY, _ROW_ID_COL, _M2O),)},
|
| 2667 |
)
|
| 2668 |
|
| 2669 |
|
|
|
|
| 2951 |
BILLS_KEY: "odoo:account.move.vendor",
|
| 2952 |
ORDER_LINES_KEY: "odoo:sale.order.line",
|
| 2953 |
GL_LINES_KEY: "odoo:account.move.line",
|
| 2954 |
+
STOCK_MOVES_KEY: "odoo:stock.move",
|
| 2955 |
+
STOCK_VALUATION_KEY: "odoo:stock.valuation.layer",
|
| 2956 |
+
STOCK_QUANT_KEY: "odoo:stock.quant.internal",
|
| 2957 |
+
PURCHASE_ORDERS_KEY: "odoo:purchase.order.confirmed",
|
| 2958 |
+
PURCHASE_LINES_KEY: "odoo:purchase.order.line.confirmed",
|
| 2959 |
}
|
| 2960 |
|
| 2961 |
# One receipt identity per curated database. The model is not enough: invoices/bills share
|
|
|
|
| 2970 |
BILLS_KEY: "odoo_bills",
|
| 2971 |
ORDER_LINES_KEY: "sales_lines",
|
| 2972 |
GL_LINES_KEY: "gl_lines",
|
| 2973 |
+
STOCK_MOVES_KEY: "odoo_stock_moves",
|
| 2974 |
+
STOCK_VALUATION_KEY: "odoo_stock_valuation",
|
| 2975 |
+
STOCK_QUANT_KEY: "odoo_stock_quant",
|
| 2976 |
+
PURCHASE_ORDERS_KEY: "odoo_purchase_orders",
|
| 2977 |
+
PURCHASE_LINES_KEY: "odoo_purchase_lines",
|
| 2978 |
}
|
| 2979 |
|
| 2980 |
_SOURCE_COUNT_SPECS = {
|
|
|
|
| 2995 |
"order_lines": {"model": "sale.order.line", "mode": "count", "domain": [
|
| 2996 |
("order_id.state", "in", ["sale", "done"])]},
|
| 2997 |
"gl_lines": {"model": "account.move.line", "mode": "count", "domain": []},
|
| 2998 |
+
"stock_moves": {"model": "stock.move", "mode": "count", "domain": []},
|
| 2999 |
+
"stock_valuation": {"model": "stock.valuation.layer", "mode": "count", "domain": []},
|
| 3000 |
+
"stock_quant": {"model": "stock.quant", "mode": "count", "domain": [
|
| 3001 |
+
("location_id.usage", "=", "internal")]},
|
| 3002 |
+
"purchase_orders": {"model": "purchase.order", "mode": "count", "domain": [
|
| 3003 |
+
("state", "in", ["purchase", "done"])]},
|
| 3004 |
+
"purchase_lines": {"model": "purchase.order.line", "mode": "count", "domain": [
|
| 3005 |
+
("order_id.state", "in", ["purchase", "done"])]},
|
| 3006 |
}
|
| 3007 |
|
| 3008 |
# The two canonical registry databases replaced the retired ``ut_odoo_customers`` and
|
|
|
|
| 3317 |
READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS)
|
| 3318 |
|
| 3319 |
|
| 3320 |
+
# S70-T19 integration exports. These are declarations, not an import-time registration seam.
|
| 3321 |
+
# COMBINE-AMEND: copy the two new specs into harness.datastore.ENTITIES, merge the field additions
|
| 3322 |
+
# into each existing entity's fields AND backfill_fields, and extend NUMERIC_FIELDS. Do not import
|
| 3323 |
+
# this API module from harness or mutate ENTITIES at runtime. Stock move/location already exist.
|
| 3324 |
+
STOCK_MIRROR_ENTITIES = {
|
| 3325 |
+
"stock_valuation_layer": {
|
| 3326 |
+
"model": "stock.valuation.layer", "archivable": False, "optional": True,
|
| 3327 |
+
"fields": ["product_id", "quantity", "value", "create_date", "stock_move_id", "write_date"],
|
| 3328 |
+
},
|
| 3329 |
+
"stock_quant": {
|
| 3330 |
+
"model": "stock.quant", "archivable": False, "optional": True,
|
| 3331 |
+
"fields": ["product_id", "location_id", "quantity", "reserved_quantity", "write_date"],
|
| 3332 |
+
},
|
| 3333 |
+
}
|
| 3334 |
+
STOCK_MIRROR_FIELD_ADDITIONS = {
|
| 3335 |
+
"stock_move": ("name", "product_uom"),
|
| 3336 |
+
"stock_location": ("parent_path",),
|
| 3337 |
+
"product_product": ("uom_id",),
|
| 3338 |
+
}
|
| 3339 |
+
STOCK_MIRROR_NUMERIC_FIELDS = ("quantity", "value", "reserved_quantity")
|
| 3340 |
+
|
| 3341 |
+
|
| 3342 |
+
def stock_mirror_entity_specs():
|
| 3343 |
+
"""Detached source declarations for the integrator; this never modifies the datastore."""
|
| 3344 |
+
from copy import deepcopy
|
| 3345 |
+
return deepcopy(STOCK_MIRROR_ENTITIES)
|
| 3346 |
+
|
| 3347 |
+
|
| 3348 |
+
def _stock_text(value):
|
| 3349 |
+
return str(value or "")
|
| 3350 |
+
|
| 3351 |
+
|
| 3352 |
+
def _stock_id(value):
|
| 3353 |
+
return "" if value in (None, False, "") else int(value)
|
| 3354 |
+
|
| 3355 |
+
|
| 3356 |
+
def _stock_number(value):
|
| 3357 |
+
# The existing quantity field dialect is `int`, but values remain fractional, as on order
|
| 3358 |
+
# lines. Never cast a quantity to int or round signed valuation values during projection.
|
| 3359 |
+
return float(value or 0.0)
|
| 3360 |
+
|
| 3361 |
+
|
| 3362 |
+
def _stock_flag(value):
|
| 3363 |
+
return "1" if value else ""
|
| 3364 |
+
|
| 3365 |
+
|
| 3366 |
+
def _stock_product_cols(alias):
|
| 3367 |
+
return {
|
| 3368 |
+
"product": (alias + ".product_name", _stock_text),
|
| 3369 |
+
PRODUCT_JOIN_KEY: (alias + ".product_id", _stock_id),
|
| 3370 |
+
"sku": ("pp.default_code", _stock_text),
|
| 3371 |
+
"uom": ("pp.uom_name", _stock_text),
|
| 3372 |
+
"standard_price": ("pp.standard_price", _stock_number),
|
| 3373 |
+
"category": ("pp.categ_name", _stock_text),
|
| 3374 |
+
}
|
| 3375 |
+
|
| 3376 |
+
|
| 3377 |
+
def _stock_location_cols(prefix, alias, id_sql):
|
| 3378 |
+
# Find FBA in THIS mirror's location hierarchy. The reference tenant's root is 18423, but
|
| 3379 |
+
# that id cannot define FBA in another tenant. Descendants include all nested internal stock.
|
| 3380 |
+
fba_sql = (
|
| 3381 |
+
"EXISTS (SELECT 1 FROM stock_location fba WHERE fba.name = 'FBA' "
|
| 3382 |
+
"AND fba.usage = 'view' AND strpos('/' || COALESCE(" + alias + ".parent_path, ''), "
|
| 3383 |
+
"'/' || CAST(fba.id AS VARCHAR) || '/') > 0)")
|
| 3384 |
+
return {
|
| 3385 |
+
prefix + "_id": (id_sql, _stock_id),
|
| 3386 |
+
prefix: (alias + ".complete_name", _stock_text),
|
| 3387 |
+
prefix + "_usage": (alias + ".usage", _stock_text),
|
| 3388 |
+
prefix + "_fba": (fba_sql, _stock_flag),
|
| 3389 |
+
}
|
| 3390 |
+
|
| 3391 |
+
|
| 3392 |
+
def stock_source_specs():
|
| 3393 |
+
"""Windowed SQL bindings for routes_odoo_tables._odoo_sources to merge before its return.
|
| 3394 |
+
|
| 3395 |
+
No route import, registry mutation, tenant cursor or store read occurs here. Existing route
|
| 3396 |
+
admission, tenant assertions, connection ownership, row permissions and aggregate doors
|
| 3397 |
+
remain the consumers' responsibility. A declaration alone does not make a grid readable.
|
| 3398 |
+
"""
|
| 3399 |
+
locations = (" LEFT JOIN stock_location sl ON sl.id = sm.location_id"
|
| 3400 |
+
" LEFT JOIN stock_location sd ON sd.id = sm.location_dest_id")
|
| 3401 |
+
move_cols = {
|
| 3402 |
+
"move": ("sm.name", _stock_text), "odoo_id": ("sm.id", _stock_id),
|
| 3403 |
+
**_stock_product_cols("sm"),
|
| 3404 |
+
"qty": ("sm.quantity_done", _stock_number),
|
| 3405 |
+
"move_uom": ("sm.product_uom_name", _stock_text), "date": ("sm.date", _as_date),
|
| 3406 |
+
"state": ("sm.state", _stock_text), "picking_id": ("sm.picking_id", _stock_id),
|
| 3407 |
+
**_stock_location_cols("source_location", "sl", "sm.location_id"),
|
| 3408 |
+
**_stock_location_cols("destination_location", "sd", "sm.location_dest_id"),
|
| 3409 |
+
}
|
| 3410 |
+
# A mirror whose stock_location predates the parent_path backfill serves a blank FBA flag; the
|
| 3411 |
+
# EXISTS over an absent column is a DuckDB Binder error, which reached Data as a bare 500.
|
| 3412 |
+
no_ancestry = {"absent": (("stock_location", "parent_path"),)}
|
| 3413 |
+
blank_fba = ("''", _stock_flag)
|
| 3414 |
+
valuation_cols = {
|
| 3415 |
+
"odoo_id": ("svl.id", _stock_id), **_stock_product_cols("svl"),
|
| 3416 |
+
"qty": ("svl.quantity", _stock_number), "value": ("svl.value", _stock_number),
|
| 3417 |
+
"date": ("svl.create_date", _as_date), "stock_move_id": ("svl.stock_move_id", _stock_id),
|
| 3418 |
+
**_stock_location_cols("source_location", "sl", "sm.location_id"),
|
| 3419 |
+
**_stock_location_cols("destination_location", "sd", "sm.location_dest_id"),
|
| 3420 |
+
}
|
| 3421 |
+
return {
|
| 3422 |
+
STOCK_MOVES_KEY: {
|
| 3423 |
+
"from_sql": "(stock_move sm LEFT JOIN product_product pp ON pp.id = sm.product_id"
|
| 3424 |
+
+ locations + ")",
|
| 3425 |
+
"tables": {"sm": "stock_move", "pp": "product_product", "sl": "stock_location",
|
| 3426 |
+
"sd": "stock_location"},
|
| 3427 |
+
"where": "", "id": "sm.id", "cols": move_cols,
|
| 3428 |
+
"optional": ({**no_ancestry, "without": {"cols": {
|
| 3429 |
+
"source_location_fba": blank_fba, "destination_location_fba": blank_fba}}},),
|
| 3430 |
+
},
|
| 3431 |
+
STOCK_VALUATION_KEY: {
|
| 3432 |
+
"from_sql": "(stock_valuation_layer svl LEFT JOIN product_product pp "
|
| 3433 |
+
"ON pp.id = svl.product_id LEFT JOIN stock_move sm "
|
| 3434 |
+
"ON sm.id = svl.stock_move_id" + locations + ")",
|
| 3435 |
+
"tables": {"svl": "stock_valuation_layer", "pp": "product_product", "sm": "stock_move",
|
| 3436 |
+
"sl": "stock_location", "sd": "stock_location"},
|
| 3437 |
+
"where": "", "id": "svl.id", "cols": valuation_cols,
|
| 3438 |
+
"optional": ({**no_ancestry, "without": {"cols": {
|
| 3439 |
+
"source_location_fba": blank_fba, "destination_location_fba": blank_fba}}},),
|
| 3440 |
+
},
|
| 3441 |
+
STOCK_QUANT_KEY: {
|
| 3442 |
+
"from_sql": "(stock_quant sq LEFT JOIN product_product pp ON pp.id = sq.product_id "
|
| 3443 |
+
"LEFT JOIN stock_location sl ON sl.id = sq.location_id)",
|
| 3444 |
+
"tables": {"sq": "stock_quant", "pp": "product_product", "sl": "stock_location"},
|
| 3445 |
+
"where": "sl.usage = 'internal'", "id": "sq.id",
|
| 3446 |
+
"cols": {"odoo_id": ("sq.id", _stock_id), **_stock_product_cols("sq"),
|
| 3447 |
+
**_stock_location_cols("location", "sl", "sq.location_id"),
|
| 3448 |
+
"qty": ("sq.quantity", _stock_number),
|
| 3449 |
+
"reserved_qty": ("sq.reserved_quantity", _stock_number)},
|
| 3450 |
+
"optional": ({**no_ancestry, "without": {"cols": {"location_fba": blank_fba}}},),
|
| 3451 |
+
},
|
| 3452 |
+
}
|
| 3453 |
+
|
| 3454 |
+
|
| 3455 |
+
def _stock_period(date_from, date_to, field):
|
| 3456 |
+
start = _dt.date.fromisoformat(date_from)
|
| 3457 |
+
end = _dt.date.fromisoformat(date_to)
|
| 3458 |
+
if start > end:
|
| 3459 |
+
raise Refused("the stock validation period is reversed")
|
| 3460 |
+
return [(field, ">=", start.isoformat() + " 00:00:00"),
|
| 3461 |
+
(field, "<=", end.isoformat() + " 23:59:59")]
|
| 3462 |
+
|
| 3463 |
+
|
| 3464 |
+
def _stock_live_totals(client, model, domain, measures):
|
| 3465 |
+
"""Uncapped, bounded-memory row read; a transport page is never a population limit."""
|
| 3466 |
+
import math
|
| 3467 |
+
last_id, count = 0, 0
|
| 3468 |
+
totals = {key: 0.0 for key in measures}
|
| 3469 |
+
while True:
|
| 3470 |
+
page = client.search_read(model, list(domain) + [("id", ">", last_id)],
|
| 3471 |
+
["id", *measures.values()], limit=2000, order="id asc")
|
| 3472 |
+
if not page:
|
| 3473 |
+
break
|
| 3474 |
+
for row in page:
|
| 3475 |
+
rid = _relation_id(row.get("id"))
|
| 3476 |
+
if rid is None or rid <= last_id:
|
| 3477 |
+
raise Refused("the stock validation source ids did not advance")
|
| 3478 |
+
last_id = rid
|
| 3479 |
+
count += len(page)
|
| 3480 |
+
for key, field in measures.items():
|
| 3481 |
+
values = [float(row.get(field) or 0.0) for row in page]
|
| 3482 |
+
if not all(math.isfinite(value) for value in values):
|
| 3483 |
+
raise Refused("the stock validation source returned a non-finite value")
|
| 3484 |
+
totals[key] = math.fsum([totals[key], *values])
|
| 3485 |
+
return count, totals
|
| 3486 |
+
|
| 3487 |
+
|
| 3488 |
+
def _stock_mirror_totals(cur, spec, measures, extra_where="", params=()):
|
| 3489 |
+
"""Aggregate the exported observation projection, never the independent source summary."""
|
| 3490 |
+
expressions = ["COUNT(*)", "COUNT(DISTINCT " + spec["id"] + ")"]
|
| 3491 |
+
expressions += ["COALESCE(SUM(" + spec["cols"][key][0] + "), 0)" for key in measures]
|
| 3492 |
+
where = " AND ".join("(" + term + ")" for term in (spec["where"], extra_where) if term)
|
| 3493 |
+
row = cur.execute("SELECT " + ", ".join(expressions) + " FROM " + spec["from_sql"]
|
| 3494 |
+
+ (" WHERE " + where if where else ""), params).fetchone()
|
| 3495 |
+
if int(row[0]) != int(row[1]):
|
| 3496 |
+
raise Refused("the stock mirror projection duplicated or lost record identities")
|
| 3497 |
+
return int(row[0]), dict(zip(measures, (float(value) for value in row[2:])))
|
| 3498 |
+
|
| 3499 |
+
|
| 3500 |
+
def validate(client, cur=None, date_from="2025-07-01", date_to="2026-06-30"):
|
| 3501 |
+
"""Read-only stock reconciliation, using the caller's own Odoo client and optional mirror.
|
| 3502 |
+
|
| 3503 |
+
The caller must lend a cursor admitted for the SAME tenant/default connection as the client.
|
| 3504 |
+
This helper opens no global client/cursor and writes nothing. Without a cursor it validates
|
| 3505 |
+
live row transformations only; that posture never claims Data, Sheet or mirror observation.
|
| 3506 |
+
Results contain errors/counts, never source rows or absolute monetary totals.
|
| 3507 |
+
"""
|
| 3508 |
+
import math
|
| 3509 |
+
specs = stock_source_specs()
|
| 3510 |
+
checks = []
|
| 3511 |
+
plane = "mirror" if cur is not None else "live_rows"
|
| 3512 |
+
|
| 3513 |
+
def tie(name, actual, expected, tolerance=1e-9, absolute=1e-6, nonempty=True):
|
| 3514 |
+
difference = abs(actual - expected)
|
| 3515 |
+
error = difference / abs(expected) if expected else (0.0 if difference == 0 else None)
|
| 3516 |
+
ok = (nonempty and math.isfinite(actual) and math.isfinite(expected)
|
| 3517 |
+
and (difference <= absolute or (error is not None and error <= tolerance)))
|
| 3518 |
+
checks.append({"check": name, "ok": ok, "relative_error": error, "observation": plane})
|
| 3519 |
+
|
| 3520 |
+
period = _stock_period(date_from, date_to, "create_date")
|
| 3521 |
+
move_period = _stock_period(date_from, date_to, "date")
|
| 3522 |
+
cases = (
|
| 3523 |
+
(STOCK_MOVES_KEY, "stock.move", move_period, {"qty": "quantity_done"},
|
| 3524 |
+
"sm.date >= ? AND sm.date <= ?", tuple(term[2] for term in move_period)),
|
| 3525 |
+
(STOCK_VALUATION_KEY, "stock.valuation.layer", period, {"qty": "quantity", "value": "value"},
|
| 3526 |
+
"svl.create_date >= ? AND svl.create_date <= ?", tuple(term[2] for term in period)),
|
| 3527 |
+
(STOCK_QUANT_KEY, "stock.quant", [("location_id.usage", "=", "internal")],
|
| 3528 |
+
{"qty": "quantity", "reserved_qty": "reserved_quantity"}, "", ()),
|
| 3529 |
+
)
|
| 3530 |
+
for key, model, domain, measures, where, params in cases:
|
| 3531 |
+
try:
|
| 3532 |
+
n, totals = (_stock_mirror_totals(cur, specs[key], measures, where, params)
|
| 3533 |
+
if cur is not None else _stock_live_totals(client, model, domain, measures))
|
| 3534 |
+
source_count = int(client.search_count(model, domain))
|
| 3535 |
+
checks.append({"check": key + " complete population", "ok": n == source_count,
|
| 3536 |
+
"observed_rows": n, "source_rows": source_count, "observation": plane})
|
| 3537 |
+
groups = client.read_group(model, domain,
|
| 3538 |
+
[field + ":sum" for field in measures.values()],
|
| 3539 |
+
["product_id"], lazy=False)
|
| 3540 |
+
for field_key, field in measures.items():
|
| 3541 |
+
summary = math.fsum(float(row.get(field) or 0.0) for row in groups)
|
| 3542 |
+
tie(key + " " + field_key + " vs Odoo read_group", totals[field_key], summary,
|
| 3543 |
+
absolute=0.01 if field_key == "value" else 1e-6,
|
| 3544 |
+
nonempty=bool(groups) or source_count == 0)
|
| 3545 |
+
except Exception as exc: # noqa: BLE001
|
| 3546 |
+
checks.append({"check": key + " reconciliation", "ok": False,
|
| 3547 |
+
"error_type": type(exc).__name__, "observation": plane})
|
| 3548 |
+
|
| 3549 |
+
try:
|
| 3550 |
+
# Gross CUSTOMER-DESTINATION layers only. Customer-source returns remain in the full
|
| 3551 |
+
# valuation tie above, but must never net against this GL cost tie (inventory-flow §1).
|
| 3552 |
+
shipment_domain = period + [("stock_move_id.location_dest_id.usage", "=", "customer")]
|
| 3553 |
+
n, shipment_totals = (
|
| 3554 |
+
_stock_mirror_totals(cur, specs[STOCK_VALUATION_KEY], {"value": "value"},
|
| 3555 |
+
"svl.create_date >= ? AND svl.create_date <= ? AND sd.usage = 'customer'",
|
| 3556 |
+
tuple(term[2] for term in period)) if cur is not None else
|
| 3557 |
+
_stock_live_totals(client, "stock.valuation.layer", shipment_domain, {"value": "value"}))
|
| 3558 |
+
shipment_count = int(client.search_count("stock.valuation.layer", shipment_domain))
|
| 3559 |
+
checks.append({"check": "customer shipments complete population",
|
| 3560 |
+
"ok": n == shipment_count and n > 0, "observed_rows": n,
|
| 3561 |
+
"source_rows": shipment_count, "observation": plane})
|
| 3562 |
+
shipment_groups = client.read_group("stock.valuation.layer", shipment_domain,
|
| 3563 |
+
["value:sum"], ["product_id"], lazy=False)
|
| 3564 |
+
tie("customer shipment value vs Odoo read_group", shipment_totals["value"],
|
| 3565 |
+
math.fsum(float(row.get("value") or 0.0) for row in shipment_groups),
|
| 3566 |
+
absolute=0.01, nonempty=n > 0 and bool(shipment_groups))
|
| 3567 |
+
accounts = client.search_read("account.account", [("code", "=", "600000")], ["id"], limit=0)
|
| 3568 |
+
account_ids = [_relation_id(row.get("id")) for row in accounts]
|
| 3569 |
+
account_ids = [rid for rid in account_ids if rid is not None]
|
| 3570 |
+
if not account_ids:
|
| 3571 |
+
raise Refused("the product cost GL account is unavailable")
|
| 3572 |
+
gl_domain = [("account_id", "in", account_ids), ("parent_state", "=", "posted"),
|
| 3573 |
+
("date", ">=", date_from), ("date", "<=", date_to)]
|
| 3574 |
+
gl_groups = client.read_group("account.move.line", gl_domain,
|
| 3575 |
+
["balance:sum"], ["account_id"], lazy=False)
|
| 3576 |
+
gl_cost = math.fsum(float(row.get("balance") or 0.0) for row in gl_groups)
|
| 3577 |
+
tie("customer shipments at cost vs GL 600000", -shipment_totals["value"], gl_cost,
|
| 3578 |
+
tolerance=0.005, absolute=0.0, nonempty=n > 0 and bool(gl_groups) and gl_cost > 0)
|
| 3579 |
+
except Exception as exc: # noqa: BLE001
|
| 3580 |
+
checks.append({"check": "customer shipments at cost vs GL 600000", "ok": False,
|
| 3581 |
+
"error_type": type(exc).__name__, "observation": plane})
|
| 3582 |
+
return checks
|
| 3583 |
+
|
| 3584 |
+
|
| 3585 |
# ═════════════════════════════════════════════════════════════════════════════════════════════
|
| 3586 |
# ⭐⭐ W54-T46 / R4 / R7 / CONTRACT C4 — A TABLE BELONGS TO EXACTLY ONE CONNECTION
|
| 3587 |
# ═════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
| 4657 |
# keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does.
|
| 4658 |
retired = []
|
| 4659 |
conn_label = connection_label(rt, connection_id)
|
| 4660 |
+
catalogue = {row["model"]: row for row in readable_model_catalog(rt, connection_id)["models"]}
|
| 4661 |
plans = [(table_key_for(key, connection_id),
|
| 4662 |
table_label_for(label, conn_label),
|
| 4663 |
+
_selection_field_contracts(namespaced_fields(fields(), connection_id),
|
| 4664 |
+
catalogue.get(ODOO_TABLE_MODELS.get(key))),
|
| 4665 |
built[bucket])
|
| 4666 |
for bucket, key, label, fields in TABLES if bucket in built]
|
| 4667 |
|
|
|
|
| 4790 |
# an empty grid.
|
| 4791 |
if is_read_through_key(key):
|
| 4792 |
table["readThrough"] = True
|
| 4793 |
+
if table.get("readThrough") is True and str(key).startswith("ut_odoo_"):
|
| 4794 |
+
# A definition publication date is not a mirror sync. Read-through definitions omit
|
| 4795 |
+
# this machine column; materialized rows retain their per-pass reconciliation stamp.
|
| 4796 |
+
def unbacked_refreshed(field):
|
| 4797 |
+
return (field.get("key") == "refreshed"
|
| 4798 |
+
and (field.get("automation") or {}).get("preset")
|
| 4799 |
+
and not ut.user_edited(field))
|
| 4800 |
+
|
| 4801 |
+
fields = [field for field in fields if not unbacked_refreshed(field)]
|
| 4802 |
+
table["fields"] = [field for field in (table.get("fields") or [])
|
| 4803 |
+
if not unbacked_refreshed(field)]
|
| 4804 |
# W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same
|
| 4805 |
# reason `readThrough` is: it is derived from this module's own structure, so the code is what
|
| 4806 |
# it has to agree with, and a definition that lost the stamp should regain it rather than go
|
|
|
|
| 4846 |
if _ut().user_edited(stored):
|
| 4847 |
continue
|
| 4848 |
if (stored.get("automation") or {}).get("preset"):
|
| 4849 |
+
for prop in ("label", "type", "options", "choicesFrom", "link", "rollup", "description",
|
| 4850 |
"agg", "pinned", "default"):
|
| 4851 |
+
# A subsequent row-only refresh must not replace authoritative selections with
|
| 4852 |
+
# an older compiled vocabulary while its metadata catalogue is unavailable.
|
| 4853 |
+
if (stored.get("choicesFrom") == "odoo" and field.get("choicesFrom") != "odoo"
|
| 4854 |
+
and prop in ("type", "options", "choicesFrom")):
|
| 4855 |
+
continue
|
| 4856 |
if prop in field:
|
| 4857 |
stored[prop] = field[prop]
|
| 4858 |
else:
|
|
|
|
| 4891 |
"label": str(meta.get("string") or raw_name),
|
| 4892 |
"type": _MODEL_FIELD_TYPES.get(raw_type, "text"),
|
| 4893 |
"source": "odoo",
|
| 4894 |
+
**({"choicesFrom": "odoo", "options": _selection_options(meta)}
|
| 4895 |
+
if raw_type == "selection" else {}),
|
| 4896 |
**({"description": "Binary content is not loaded into this database."}
|
| 4897 |
if raw_type == "binary" else {}),
|
| 4898 |
"automation": {"preset": True}})
|
|
|
|
| 4910 |
def publish_database_schema(rt, table_key, connection_id=None, username="automation", today=None):
|
| 4911 |
"""Publish one permission-visible database definition without row work.
|
| 4912 |
|
| 4913 |
+
Make at most two source calls on one model when fields or selection metadata are missing.
|
| 4914 |
Enable owns the choice and a durable sync job owns materialization. This seam only merges the
|
| 4915 |
schema, so retrying it preserves existing rows, authored fields and the navigation rename.
|
| 4916 |
"""
|
|
|
|
| 4932 |
raw = (_ut().odoo_schema_snapshot(rt).get("models") or {}).get(model)
|
| 4933 |
if isinstance(raw, dict):
|
| 4934 |
row = {"model": model, "fields": raw}
|
| 4935 |
+
if not row or not row.get("fields") or _missing_selection_metadata(row):
|
| 4936 |
catalogue = refresh_model_catalog_entry(rt, model, connection_id=namespace)
|
| 4937 |
row = next((row for row in catalogue["models"] if row.get("model") == model), None)
|
| 4938 |
if not row or not row.get("fields"):
|
| 4939 |
raise Refused("the field catalogue is unavailable for this database. Refresh the catalogue first.")
|
| 4940 |
+
if _missing_selection_metadata(row):
|
| 4941 |
+
raise Refused("Odoo selection choices are unavailable. Refresh the catalogue first.")
|
| 4942 |
raw_fields = _model_field_specs(row)
|
| 4943 |
declared = next((fields() for _bucket, known_key, _label, fields in TABLES
|
| 4944 |
if known_key == (base or table_key)), [])
|
| 4945 |
+
fields = _selection_field_contracts(namespaced_fields(declared, namespace), row)
|
| 4946 |
have = {field["key"] for field in fields}
|
| 4947 |
fields.extend(field for field in raw_fields if field["key"] not in have)
|
| 4948 |
from core import connector_sync
|
|
|
|
| 5264 |
def blocking_failures(database_failures, report):
|
| 5265 |
"""The database failures that make a sync NOT good, PURE.
|
| 5266 |
|
| 5267 |
+
A reconciliation failure within `reconciliation_drift`'s tolerance is informational unless
|
| 5268 |
+
its receipt carries a source failure. Original counts stay intact; every other failure blocks.
|
| 5269 |
"""
|
| 5270 |
receipts = {}
|
| 5271 |
for name in ("databases", "canonicalDatabases"):
|
|
|
|
| 5275 |
out = {}
|
| 5276 |
for key, failure in (database_failures or {}).items():
|
| 5277 |
if isinstance(failure, dict) and failure.get("code") == "reconciliation_incomplete":
|
| 5278 |
+
receipt = receipts.get(key)
|
| 5279 |
+
drift = reconciliation_drift(receipt)
|
| 5280 |
+
if (drift is not None and drift[0] <= drift[1]
|
| 5281 |
+
and not receipt.get("sourceError")):
|
| 5282 |
continue
|
| 5283 |
out[key] = failure
|
| 5284 |
return out
|
|
|
|
| 5414 |
return built, written, errors, receipts, report
|
| 5415 |
|
| 5416 |
|
| 5417 |
+
def _canonical_mirror_ready(rt):
|
| 5418 |
+
# Initial source selection does not make unrelated canonical totals authoritative.
|
| 5419 |
+
from connector_onboarding import selected_mirror_entities
|
| 5420 |
+
from harness import datastore
|
| 5421 |
+
return rt is None or selected_mirror_entities(rt) is None or datastore.ready()
|
| 5422 |
+
|
| 5423 |
+
|
| 5424 |
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
|
| 5425 |
all_connections=False, table_keys=None, on_apply=None):
|
| 5426 |
"""THE entry point — the store-resync path and the route both call this.
|
|
|
|
| 5512 |
if ns is not None:
|
| 5513 |
cur = None
|
| 5514 |
elif cur is None:
|
| 5515 |
+
from connector_onboarding import mirror_cursor
|
| 5516 |
+
cur = mirror_cursor(rt)
|
| 5517 |
requested = set(table_keys) if table_keys is not None else None
|
| 5518 |
if requested is not None:
|
| 5519 |
for key in requested:
|
|
|
|
| 5537 |
built, written, database_errors, database_receipts, plan_report = refresh_curated(
|
| 5538 |
rt, cur, keep, connection_id=ns, username=username, today=today,
|
| 5539 |
retire=requested is None, on_apply=on_apply)
|
| 5540 |
+
if ns is None and requested is None and cur is not None and _canonical_mirror_ready(rt):
|
| 5541 |
canonical_receipts = canonical_reconciliation_receipts(cur)
|
| 5542 |
else:
|
| 5543 |
# The binding selects this connection's verified read-only client. Authentication failure
|
|
|
|
| 5551 |
built, written, database_errors, database_receipts, plan_report = refresh_curated(
|
| 5552 |
rt, cur, keep, connection_id=ns, username=username, today=today,
|
| 5553 |
retire=requested is None, on_apply=on_apply, source_client=source_client)
|
| 5554 |
+
if ns is None and requested is None and cur is not None and _canonical_mirror_ready(rt):
|
| 5555 |
canonical_receipts = canonical_reconciliation_receipts(
|
| 5556 |
cur, source_client=source_client)
|
| 5557 |
# ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing.
|
api/routes_activity.py
CHANGED
|
@@ -331,6 +331,9 @@ async def ingest_activity(request: Request, session: Session = Depends(require_s
|
|
| 331 |
clean.append({**event, "eventId": uuid.uuid4().hex})
|
| 332 |
dropped = len(events) - len(clean)
|
| 333 |
clean.sort(key=lambda event: event["ts"])
|
|
|
|
|
|
|
|
|
|
| 334 |
if clean:
|
| 335 |
try:
|
| 336 |
await run_in_threadpool(_record, session, session_id, clean,
|
|
|
|
| 331 |
clean.append({**event, "eventId": uuid.uuid4().hex})
|
| 332 |
dropped = len(events) - len(clean)
|
| 333 |
clean.sort(key=lambda event: event["ts"])
|
| 334 |
+
from core import freeze_notice
|
| 335 |
+
if freeze_notice.enabled():
|
| 336 |
+
return {"accepted": 0, "dropped": len(events)}
|
| 337 |
if clean:
|
| 338 |
try:
|
| 339 |
await run_in_threadpool(_record, session, session_id, clean,
|
api/routes_admin.py
CHANGED
|
@@ -72,7 +72,7 @@ def system_data_inspector(session: Session = Depends(require_session)):
|
|
| 72 |
#: ⚠ DELIBERATELY PERMISSIVE, because the store key is the thing being constrained and not the
|
| 73 |
#: deliverability of the address: one `@`, a dot-bearing domain, and the same character class the
|
| 74 |
#: slug already trusts. A stricter grammar here would refuse addresses that exist.
|
| 75 |
-
_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$")
|
| 76 |
#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`.
|
| 77 |
_UNAME_MAX = 254
|
| 78 |
#: Password policy lives in core.users so this route and self-service rotation cannot disagree.
|
|
@@ -115,7 +115,7 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
|
| 115 |
resolved once for the whole request; the single-record routes pass neither, because their
|
| 116 |
caller fetches the editor payload separately and `GET /perms` is the authority there.
|
| 117 |
"""
|
| 118 |
-
return {"username": uname,
|
| 119 |
"name": rec.get("name") or uname,
|
| 120 |
"role": rec.get("role", "user"),
|
| 121 |
"bus": rec.get("bus", "all"),
|
|
@@ -208,12 +208,15 @@ def _clean_username(v):
|
|
| 208 |
accounts above are administrable today despite being unconstructable. Arming this validator on
|
| 209 |
those routes would look like hardening and would break five live accounts.
|
| 210 |
"""
|
| 211 |
-
uname = str(v or "").strip()
|
| 212 |
-
|
|
|
|
| 213 |
raise err(400, "bad_username",
|
| 214 |
"a username is either 2 to 32 characters of lowercase letters, digits, dot, "
|
| 215 |
"dash or underscore, or a full email address of up to 254 characters")
|
| 216 |
-
|
|
|
|
|
|
|
| 217 |
|
| 218 |
|
| 219 |
def _clean_password(v, username=None, name=None, email=None, record=None,
|
|
@@ -334,7 +337,10 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi
|
|
| 334 |
"""
|
| 335 |
body = body or {}
|
| 336 |
_require_store()
|
| 337 |
-
|
|
|
|
|
|
|
|
|
|
| 338 |
name = str(body.get("name") or "").strip() or uname
|
| 339 |
email = str(body.get("email") or "").strip() or None
|
| 340 |
pw = _clean_password(body.get("password"), username=uname, name=name, email=email)
|
|
@@ -375,7 +381,7 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi
|
|
| 375 |
# Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's —
|
| 376 |
# the tenant is stamped from the session, not taken from the body.
|
| 377 |
try:
|
| 378 |
-
users.create_user(
|
| 379 |
agent=agent, email=email, tenant=_tenant_of(session.user))
|
| 380 |
except users.PasswordPolicyError as exc:
|
| 381 |
raise err(400, "weak_password", "That password does not meet the policy: " + str(exc))
|
|
@@ -585,7 +591,9 @@ def _wall_database_name(key, session):
|
|
| 585 |
return module["label"]
|
| 586 |
try:
|
| 587 |
from core import user_tables
|
| 588 |
-
label = ((
|
|
|
|
|
|
|
| 589 |
except Exception: # noqa: BLE001
|
| 590 |
label = None
|
| 591 |
return label.strip() if isinstance(label, str) and label.strip() else "This database"
|
|
@@ -734,7 +742,7 @@ def _apply_admin_only_filter_rule(row):
|
|
| 734 |
**{_FILTER_REASON_KEY: _ADMIN_ONLY_FILTER_REASON.format(owner=owner)})
|
| 735 |
|
| 736 |
|
| 737 |
-
def _admin_usernames():
|
| 738 |
"""Every account holding the `admin` role, lowercased — or `None` when that set could not be
|
| 739 |
established.
|
| 740 |
|
|
@@ -753,7 +761,7 @@ def _admin_usernames():
|
|
| 753 |
answer.
|
| 754 |
"""
|
| 755 |
try:
|
| 756 |
-
reg = users.registry() or {}
|
| 757 |
except Exception: # noqa: BLE001
|
| 758 |
return None
|
| 759 |
if not isinstance(reg, dict) or not reg:
|
|
@@ -808,7 +816,7 @@ def _field_provenance(src, key, session):
|
|
| 808 |
classes = {}
|
| 809 |
admins = None
|
| 810 |
if session is not None:
|
| 811 |
-
admins = _admin_usernames()
|
| 812 |
try:
|
| 813 |
# ⛔ `grant_topic` IS THE MODULE KEY, NOT THE STORE BUCKET. `shares.field_oid` namespaces
|
| 814 |
# a grant by the topic the write doors already use — `"customer_data"` /
|
|
@@ -840,7 +848,7 @@ def _field_provenance(src, key, session):
|
|
| 840 |
return out
|
| 841 |
|
| 842 |
|
| 843 |
-
def _module_fields(key, session=None, wall_verdicts=None):
|
| 844 |
"""The field schema the editor builds its pickers from — the SAME vocabulary the grid uses,
|
| 845 |
so an admin filters on exactly what the user sees.
|
| 846 |
|
|
@@ -901,7 +909,8 @@ def _module_fields(key, session=None, wall_verdicts=None):
|
|
| 901 |
f"{key!r} is not a database in this workspace")
|
| 902 |
src = defn.get("fields") or []
|
| 903 |
else:
|
| 904 |
-
|
|
|
|
| 905 |
if provider is None:
|
| 906 |
# Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative
|
| 907 |
# (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is
|
|
@@ -940,7 +949,14 @@ def _module_fields(key, session=None, wall_verdicts=None):
|
|
| 940 |
if isinstance(_uf, dict) and _uf.get("key") and _uf["key"] not in _have:
|
| 941 |
_have.add(_uf["key"])
|
| 942 |
src.append(_uf)
|
| 943 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 944 |
out = []
|
| 945 |
for f in src:
|
| 946 |
if not isinstance(f, dict) or not f.get("key"):
|
|
@@ -1591,7 +1607,7 @@ def _stale_hidden_keys(key, hidden, session=None, fields=None):
|
|
| 1591 |
- {str(h) for h in (hidden or ()) if str(h).startswith(_mpre)})
|
| 1592 |
|
| 1593 |
|
| 1594 |
-
def _prune_stale_filters(block, session=None):
|
| 1595 |
"""`(block, {module: [dropped keys]})` — owner I16's cascade over a whole stored perms block.
|
| 1596 |
|
| 1597 |
⭐⭐ *"If the field is deleted, its permission filter goes with it."* Run at the ADMIN DOOR,
|
|
@@ -1610,7 +1626,8 @@ def _prune_stale_filters(block, session=None):
|
|
| 1610 |
for key, entry in (block or {}).items():
|
| 1611 |
if not isinstance(entry, dict) or not entry.get("filter"):
|
| 1612 |
continue
|
| 1613 |
-
vocab = _prunable_vocabulary(key, session=session
|
|
|
|
| 1614 |
if vocab is None:
|
| 1615 |
continue
|
| 1616 |
tree, gone = perm_scope.prune_filter_to_fields(entry["filter"], vocab)
|
|
@@ -2195,6 +2212,19 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2195 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 2196 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 2197 |
import core.perm_scope as perm_scope
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2198 |
|
| 2199 |
rec = reg[uname]
|
| 2200 |
stored = rec.get("perms") or {}
|
|
@@ -2216,7 +2246,9 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2216 |
# ⭐ ONE `_module_fields` PASS, SHARED BY THE PICKER AND THE CASCADE. It used to be built
|
| 2217 |
# inline at the return; hoisted so the prune below reads the SAME vocabulary the payload
|
| 2218 |
# offers rather than paying a second store round-trip per database to ask again.
|
| 2219 |
-
|
|
|
|
|
|
|
| 2220 |
# ⭐⭐ W46-T34 / OWNER INSTRUCTION 6 — THE METRIC COUNT, STATED PER DATABASE. The rows are
|
| 2221 |
# already inside `fields_by_module` (they are `metric: true` members of the hide vocabulary);
|
| 2222 |
# what could not be read off them is the DIFFERENCE between a database with no measures and
|
|
@@ -2242,7 +2274,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2242 |
# `stored` test alone would very nearly cover it (an un-migrated record carries no perms
|
| 2243 |
# block), so the marker is asked explicitly rather than inferred from an empty dict.
|
| 2244 |
if stored and perm_scope.is_migrated(rec):
|
| 2245 |
-
_pruned, _gone = _prune_stale_filters(stored, session=session
|
|
|
|
| 2246 |
if _gone:
|
| 2247 |
users.set_access(uname, perms=_pruned)
|
| 2248 |
# Re-read rather than trust the write: this route's whole job is to report what is
|
|
@@ -2346,33 +2379,51 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
|
|
| 2346 |
|
| 2347 |
|
| 2348 |
def _sync_database_shares(session, uname, before, after):
|
| 2349 |
-
"""
|
| 2350 |
|
| 2351 |
Two doors answer two questions: the access rule decides WHICH rows, a `core.shares` `database`
|
| 2352 |
-
grant decides IF the member sees
|
| 2353 |
-
|
| 2354 |
-
|
| 2355 |
"""
|
| 2356 |
-
from core import shares
|
| 2357 |
-
|
| 2358 |
-
def is_on(block, key):
|
| 2359 |
-
entry = (block or {}).get(key)
|
| 2360 |
-
return isinstance(entry, dict) and entry.get("access") is True
|
| 2361 |
|
| 2362 |
-
for key in sorted(
|
| 2363 |
-
|
| 2364 |
-
|
| 2365 |
-
|
| 2366 |
-
|
| 2367 |
-
|
| 2368 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2369 |
held = [e for e in record["entries"] if e.get("user") == uname]
|
| 2370 |
-
if now and held:
|
|
|
|
|
|
|
| 2371 |
continue
|
| 2372 |
entries = [e for e in record["entries"] if e.get("user") != uname]
|
| 2373 |
if now:
|
| 2374 |
-
entries.append({"user": uname, "role": "
|
| 2375 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2376 |
|
| 2377 |
|
| 2378 |
@router.put("/admin/users/{username}/perms")
|
|
@@ -2479,7 +2530,9 @@ def settings(session: Session = Depends(require_session)):
|
|
| 2479 |
would answer a question about our customer list.
|
| 2480 |
"""
|
| 2481 |
modules = perms.allowed_modules(session.user)
|
|
|
|
| 2482 |
return {
|
|
|
|
| 2483 |
# The build this Space is running. Behind the session on purpose — see `main.py::health`,
|
| 2484 |
# which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
|
| 2485 |
# Space secret at every deploy; absent means "somebody started this container by hand".
|
|
|
|
| 72 |
#: ⚠ DELIBERATELY PERMISSIVE, because the store key is the thing being constrained and not the
|
| 73 |
#: deliverability of the address: one `@`, a dot-bearing domain, and the same character class the
|
| 74 |
#: slug already trusts. A stricter grammar here would refuse addresses that exist.
|
| 75 |
+
_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$", re.IGNORECASE | re.ASCII)
|
| 76 |
#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`.
|
| 77 |
_UNAME_MAX = 254
|
| 78 |
#: Password policy lives in core.users so this route and self-service rotation cannot disagree.
|
|
|
|
| 115 |
resolved once for the whole request; the single-record routes pass neither, because their
|
| 116 |
caller fetches the editor payload separately and `GET /perms` is the authority there.
|
| 117 |
"""
|
| 118 |
+
return {"username": rec.get("username_original") or uname,
|
| 119 |
"name": rec.get("name") or uname,
|
| 120 |
"role": rec.get("role", "user"),
|
| 121 |
"bus": rec.get("bus", "all"),
|
|
|
|
| 208 |
accounts above are administrable today despite being unconstructable. Arming this validator on
|
| 209 |
those routes would look like hardening and would break five live accounts.
|
| 210 |
"""
|
| 211 |
+
uname = str(v or "").strip()
|
| 212 |
+
is_email = bool(_UNAME_EMAIL_RE.fullmatch(uname))
|
| 213 |
+
if len(uname) > _UNAME_MAX or not (_UNAME_RE.fullmatch(uname.lower()) or is_email):
|
| 214 |
raise err(400, "bad_username",
|
| 215 |
"a username is either 2 to 32 characters of lowercase letters, digits, dot, "
|
| 216 |
"dash or underscore, or a full email address of up to 254 characters")
|
| 217 |
+
# D-385: validation preserves an email's spelling. The account registry still owns
|
| 218 |
+
# canonical login/workspace keys; changing that storage contract requires core.users.
|
| 219 |
+
return uname if is_email else uname.lower()
|
| 220 |
|
| 221 |
|
| 222 |
def _clean_password(v, username=None, name=None, email=None, record=None,
|
|
|
|
| 337 |
"""
|
| 338 |
body = body or {}
|
| 339 |
_require_store()
|
| 340 |
+
requested_username = _clean_username(body.get("username"))
|
| 341 |
+
# core.users.create_user and every account lookup below use a canonical key. Keep
|
| 342 |
+
# collision checks and readback on that same key; _view exposes original spelling.
|
| 343 |
+
uname = requested_username.lower()
|
| 344 |
name = str(body.get("name") or "").strip() or uname
|
| 345 |
email = str(body.get("email") or "").strip() or None
|
| 346 |
pw = _clean_password(body.get("password"), username=uname, name=name, email=email)
|
|
|
|
| 381 |
# Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's —
|
| 382 |
# the tenant is stamped from the session, not taken from the body.
|
| 383 |
try:
|
| 384 |
+
users.create_user(requested_username, pw, name, role=role, bus=bus, modules=modules,
|
| 385 |
agent=agent, email=email, tenant=_tenant_of(session.user))
|
| 386 |
except users.PasswordPolicyError as exc:
|
| 387 |
raise err(400, "weak_password", "That password does not meet the policy: " + str(exc))
|
|
|
|
| 591 |
return module["label"]
|
| 592 |
try:
|
| 593 |
from core import user_tables
|
| 594 |
+
label = next((row.get("label") for row in user_tables.nav_entries(
|
| 595 |
+
viewer=session.uname, is_admin=session.admin, st=session.runtime)
|
| 596 |
+
if row.get("key") == key), None)
|
| 597 |
except Exception: # noqa: BLE001
|
| 598 |
label = None
|
| 599 |
return label.strip() if isinstance(label, str) and label.strip() else "This database"
|
|
|
|
| 742 |
**{_FILTER_REASON_KEY: _ADMIN_ONLY_FILTER_REASON.format(owner=owner)})
|
| 743 |
|
| 744 |
|
| 745 |
+
def _admin_usernames(roster=None):
|
| 746 |
"""Every account holding the `admin` role, lowercased — or `None` when that set could not be
|
| 747 |
established.
|
| 748 |
|
|
|
|
| 761 |
answer.
|
| 762 |
"""
|
| 763 |
try:
|
| 764 |
+
reg = (users.registry() if roster is None else roster) or {}
|
| 765 |
except Exception: # noqa: BLE001
|
| 766 |
return None
|
| 767 |
if not isinstance(reg, dict) or not reg:
|
|
|
|
| 816 |
classes = {}
|
| 817 |
admins = None
|
| 818 |
if session is not None:
|
| 819 |
+
admins = _admin_usernames(getattr(session, "_permission_roster", None))
|
| 820 |
try:
|
| 821 |
# ⛔ `grant_topic` IS THE MODULE KEY, NOT THE STORE BUCKET. `shares.field_oid` namespaces
|
| 822 |
# a grant by the topic the write doors already use — `"customer_data"` /
|
|
|
|
| 848 |
return out
|
| 849 |
|
| 850 |
|
| 851 |
+
def _module_fields(key, session=None, wall_verdicts=None, vocabulary_memo=None):
|
| 852 |
"""The field schema the editor builds its pickers from — the SAME vocabulary the grid uses,
|
| 853 |
so an admin filters on exactly what the user sees.
|
| 854 |
|
|
|
|
| 909 |
f"{key!r} is not a database in this workspace")
|
| 910 |
src = defn.get("fields") or []
|
| 911 |
else:
|
| 912 |
+
providers = _static_field_providers()
|
| 913 |
+
provider = providers.get(key)
|
| 914 |
if provider is None:
|
| 915 |
# Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative
|
| 916 |
# (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is
|
|
|
|
| 949 |
if isinstance(_uf, dict) and _uf.get("key") and _uf["key"] not in _have:
|
| 950 |
_have.add(_uf["key"])
|
| 951 |
src.append(_uf)
|
| 952 |
+
# Share even an incomplete vocabulary result across this request's databases. Failed
|
| 953 |
+
# sources stay omitted and retry on the next request; only successes enter process caches.
|
| 954 |
+
if vocabulary_memo is not None and "result" in vocabulary_memo:
|
| 955 |
+
known = vocabulary_memo["result"]
|
| 956 |
+
else:
|
| 957 |
+
known = _server_side_vocabularies(session=session)
|
| 958 |
+
if vocabulary_memo is not None:
|
| 959 |
+
vocabulary_memo["result"] = known
|
| 960 |
out = []
|
| 961 |
for f in src:
|
| 962 |
if not isinstance(f, dict) or not f.get("key"):
|
|
|
|
| 1607 |
- {str(h) for h in (hidden or ()) if str(h).startswith(_mpre)})
|
| 1608 |
|
| 1609 |
|
| 1610 |
+
def _prune_stale_filters(block, session=None, fields_by_module=None):
|
| 1611 |
"""`(block, {module: [dropped keys]})` — owner I16's cascade over a whole stored perms block.
|
| 1612 |
|
| 1613 |
⭐⭐ *"If the field is deleted, its permission filter goes with it."* Run at the ADMIN DOOR,
|
|
|
|
| 1626 |
for key, entry in (block or {}).items():
|
| 1627 |
if not isinstance(entry, dict) or not entry.get("filter"):
|
| 1628 |
continue
|
| 1629 |
+
vocab = _prunable_vocabulary(key, session=session,
|
| 1630 |
+
fields=(fields_by_module or {}).get(key))
|
| 1631 |
if vocab is None:
|
| 1632 |
continue
|
| 1633 |
tree, gone = perm_scope.prune_filter_to_fields(entry["filter"], vocab)
|
|
|
|
| 2212 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 2213 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 2214 |
import core.perm_scope as perm_scope
|
| 2215 |
+
from copy import copy
|
| 2216 |
+
from core import user_tables
|
| 2217 |
+
from routes_customers import _AssemblyStore
|
| 2218 |
+
|
| 2219 |
+
# One read-only request snapshot: definitions, grants and provenance are shared by the
|
| 2220 |
+
# picker and cascade. It never enters a write door or outlives this request. Global account
|
| 2221 |
+
# writes and their required post-write readback below continue through the original store.
|
| 2222 |
+
session = copy(session)
|
| 2223 |
+
session._permission_roster = reg
|
| 2224 |
+
try:
|
| 2225 |
+
session.runtime = user_tables.lend_defs(_AssemblyStore(session.runtime, schema_only=True))
|
| 2226 |
+
except Exception:
|
| 2227 |
+
raise err(503, "schema_unavailable", "The permission field definitions are unavailable. Try again.")
|
| 2228 |
|
| 2229 |
rec = reg[uname]
|
| 2230 |
stored = rec.get("perms") or {}
|
|
|
|
| 2246 |
# ⭐ ONE `_module_fields` PASS, SHARED BY THE PICKER AND THE CASCADE. It used to be built
|
| 2247 |
# inline at the return; hoisted so the prune below reads the SAME vocabulary the payload
|
| 2248 |
# offers rather than paying a second store round-trip per database to ask again.
|
| 2249 |
+
vocabulary_memo = {}
|
| 2250 |
+
fields_by_module = {k: _module_fields(k, session=session, vocabulary_memo=vocabulary_memo)
|
| 2251 |
+
for k in governed}
|
| 2252 |
# ⭐⭐ W46-T34 / OWNER INSTRUCTION 6 — THE METRIC COUNT, STATED PER DATABASE. The rows are
|
| 2253 |
# already inside `fields_by_module` (they are `metric: true` members of the hide vocabulary);
|
| 2254 |
# what could not be read off them is the DIFFERENCE between a database with no measures and
|
|
|
|
| 2274 |
# `stored` test alone would very nearly cover it (an un-migrated record carries no perms
|
| 2275 |
# block), so the marker is asked explicitly rather than inferred from an empty dict.
|
| 2276 |
if stored and perm_scope.is_migrated(rec):
|
| 2277 |
+
_pruned, _gone = _prune_stale_filters(stored, session=session,
|
| 2278 |
+
fields_by_module=fields_by_module)
|
| 2279 |
if _gone:
|
| 2280 |
users.set_access(uname, perms=_pruned)
|
| 2281 |
# Re-read rather than trust the write: this route's whole job is to report what is
|
|
|
|
| 2379 |
|
| 2380 |
|
| 2381 |
def _sync_database_shares(session, uname, before, after):
|
| 2382 |
+
"""R15/C5: explicit ut_* access grants editor authority; explicit off removes the share.
|
| 2383 |
|
| 2384 |
Two doors answer two questions: the access rule decides WHICH rows, a `core.shares` `database`
|
| 2385 |
+
grant decides IF the member sees and edits it, so the editor's toggle writes both. Re-saving
|
| 2386 |
+
an enabled entry repairs old view-only grants. Other principals, owners and provenance stay
|
| 2387 |
+
intact; an omitted entry does not revoke an independently held share.
|
| 2388 |
"""
|
| 2389 |
+
from core import shares, user_tables
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2390 |
|
| 2391 |
+
changes = [(key, entry["access"]) for key, entry in sorted((after or {}).items())
|
| 2392 |
+
if str(key).startswith("ut_") and isinstance(entry, dict)
|
| 2393 |
+
and type(entry.get("access")) is bool]
|
| 2394 |
+
if not changes:
|
| 2395 |
+
return
|
| 2396 |
+
try:
|
| 2397 |
+
bucket = session.runtime.get(shares.SHARES_KEY, fresh=True)
|
| 2398 |
+
except Exception as exc:
|
| 2399 |
+
raise err(503, "store_unavailable", "Database access could not be saved.") from exc
|
| 2400 |
+
if bucket is None:
|
| 2401 |
+
bucket = {}
|
| 2402 |
+
database_grants = bucket.get("database", {}) if isinstance(bucket, dict) else None
|
| 2403 |
+
if not isinstance(database_grants, dict):
|
| 2404 |
+
raise err(503, "store_unavailable", "Database access could not be saved.")
|
| 2405 |
+
lent = user_tables.lend(session.runtime, **{shares.SHARES_KEY: bucket})
|
| 2406 |
+
|
| 2407 |
+
for key, now in changes:
|
| 2408 |
+
raw = database_grants.get(key, {})
|
| 2409 |
+
if not isinstance(raw, dict) or not isinstance(raw.get("entries", []), list):
|
| 2410 |
+
raise err(503, "store_unavailable", "Database access could not be saved.")
|
| 2411 |
+
record = shares.grants("database", key, st=lent)
|
| 2412 |
+
if len(record["entries"]) != len(raw.get("entries", [])):
|
| 2413 |
+
raise err(503, "store_unavailable", "Database access could not be saved.")
|
| 2414 |
held = [e for e in record["entries"] if e.get("user") == uname]
|
| 2415 |
+
if now and (record.get("owner") == uname or any(e.get("role") == "edit" for e in held)):
|
| 2416 |
+
continue
|
| 2417 |
+
if not now and not held:
|
| 2418 |
continue
|
| 2419 |
entries = [e for e in record["entries"] if e.get("user") != uname]
|
| 2420 |
if now:
|
| 2421 |
+
entries.append({**(held[0] if held else {}), "user": uname, "role": "edit"})
|
| 2422 |
+
try:
|
| 2423 |
+
shares.set_grants("database", key, entries, st=session.runtime,
|
| 2424 |
+
granter=session.uname, expect=record["entries"])
|
| 2425 |
+
except shares.GrantsChanged as exc:
|
| 2426 |
+
raise err(409, "shares_changed", "Database sharing changed. Save access again.") from exc
|
| 2427 |
|
| 2428 |
|
| 2429 |
@router.put("/admin/users/{username}/perms")
|
|
|
|
| 2530 |
would answer a question about our customer list.
|
| 2531 |
"""
|
| 2532 |
modules = perms.allowed_modules(session.user)
|
| 2533 |
+
from core import freeze_notice
|
| 2534 |
return {
|
| 2535 |
+
"freezeNotice": freeze_notice.notice(),
|
| 2536 |
# The build this Space is running. Behind the session on purpose — see `main.py::health`,
|
| 2537 |
# which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a
|
| 2538 |
# Space secret at every deploy; absent means "somebody started this container by hand".
|
api/routes_alerts.py
CHANGED
|
@@ -646,7 +646,7 @@ def run_alert(alert_id: str, session: Session = Depends(require_session)):
|
|
| 646 |
|
| 647 |
@router.get("/notifications")
|
| 648 |
def notifications(session: Session = Depends(require_session)):
|
| 649 |
-
"""The inbox
|
| 650 |
|
| 651 |
⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
|
| 652 |
plan was a push hook: the automation engine calls `after_write` when it lands rows. But
|
|
@@ -673,12 +673,14 @@ def notifications(session: Session = Depends(require_session)):
|
|
| 673 |
feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
|
| 674 |
this does: every alert is still evaluated, against the same rows, in the same order.
|
| 675 |
"""
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
|
|
|
|
|
|
| 682 |
# ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
|
| 683 |
# this wave carries a `target` too. See `notification_view`'s header for why it is derived.
|
| 684 |
box = inbox_view(alerts.inbox(session.uname, st=session.runtime))
|
|
|
|
| 646 |
|
| 647 |
@router.get("/notifications")
|
| 648 |
def notifications(session: Session = Depends(require_session)):
|
| 649 |
+
"""The inbox re-evaluates on read unless maintenance freezes writes.
|
| 650 |
|
| 651 |
⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
|
| 652 |
plan was a push hook: the automation engine calls `after_write` when it lands rows. But
|
|
|
|
| 673 |
feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
|
| 674 |
this does: every alert is still evaluated, against the same rows, in the same order.
|
| 675 |
"""
|
| 676 |
+
from core import freeze_notice
|
| 677 |
+
if not freeze_notice.enabled():
|
| 678 |
+
assemblies = {}
|
| 679 |
+
for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
|
| 680 |
+
try:
|
| 681 |
+
_evaluate(session, rec, assemblies=assemblies)
|
| 682 |
+
except Exception: # noqa: BLE001
|
| 683 |
+
continue # one bad alert must not empty the pane
|
| 684 |
# ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
|
| 685 |
# this wave carries a `target` too. See `notification_view`'s header for why it is derived.
|
| 686 |
box = inbox_view(alerts.inbox(session.uname, st=session.runtime))
|
api/routes_assets.py
CHANGED
|
@@ -36,6 +36,7 @@ from fastapi import APIRouter
|
|
| 36 |
from fastapi.responses import FileResponse
|
| 37 |
|
| 38 |
from deps import Session, err, module_gate, require_session
|
|
|
|
| 39 |
from routes_admin import admin_gate
|
| 40 |
import application_registry as applications
|
| 41 |
import semantic_registry as semantic_rt
|
|
@@ -420,8 +421,14 @@ def editorial_asset(slug: str, q: str = "web",
|
|
| 420 |
def editorial_upload(body: dict = Body(default=None),
|
| 421 |
session: Session = Depends(admin_gate)):
|
| 422 |
"""Store one editorial image: `{slug, data}` with `data` base64 (JSON keeps the API free
|
| 423 |
-
of a multipart dependency;
|
| 424 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
body = body or {}
|
| 426 |
slug = str(body.get("slug") or "").strip().lower()
|
| 427 |
if not _SLUG_RE.match(slug):
|
|
@@ -479,8 +486,8 @@ def editorial_upload(body: dict = Body(default=None),
|
|
| 479 |
# ⛔ THREE WALLS, and each exists because of a specific way this could go wrong:
|
| 480 |
#
|
| 481 |
# 1. SESSION-GATED, NOT ADMIN-GATED. R7 is explicit — a non-admin attaches a picture to their
|
| 482 |
-
# own record. The editorial upload above IS admin-only because it publishes into
|
| 483 |
-
# shared catalogue library; this writes a per-record attachment, which is the same class of
|
| 484 |
# act as typing a note.
|
| 485 |
# 2. TENANT FROM THE SESSION, NEVER FROM THE STRING. The stored address is `<tenant>/<id>` and
|
| 486 |
# both halves of it are composed server-side. A `rec:` reference pasted from another
|
|
|
|
| 36 |
from fastapi.responses import FileResponse
|
| 37 |
|
| 38 |
from deps import Session, err, module_gate, require_session
|
| 39 |
+
from core import platform_admin
|
| 40 |
from routes_admin import admin_gate
|
| 41 |
import application_registry as applications
|
| 42 |
import semantic_registry as semantic_rt
|
|
|
|
| 421 |
def editorial_upload(body: dict = Body(default=None),
|
| 422 |
session: Session = Depends(admin_gate)):
|
| 423 |
"""Store one editorial image: `{slug, data}` with `data` base64 (JSON keeps the API free
|
| 424 |
+
of a multipart dependency; a platform-admin upload is rare enough to wear the +33%).
|
| 425 |
+
This library is global, so only a Loopable platform operator may publish into it.
|
| 426 |
+
The bytes must OPEN as an image — a corrupt file is refused here, not discovered on a
|
| 427 |
+
printed page."""
|
| 428 |
+
if (not platform_admin.is_platform_admin(session.user)
|
| 429 |
+
or str(session.tenant or "").strip().lower() != platform_admin.PLATFORM_TENANT):
|
| 430 |
+
raise err(403, "platform_admin_only",
|
| 431 |
+
"editorial uploads require a Loopable platform administrator")
|
| 432 |
body = body or {}
|
| 433 |
slug = str(body.get("slug") or "").strip().lower()
|
| 434 |
if not _SLUG_RE.match(slug):
|
|
|
|
| 486 |
# ⛔ THREE WALLS, and each exists because of a specific way this could go wrong:
|
| 487 |
#
|
| 488 |
# 1. SESSION-GATED, NOT ADMIN-GATED. R7 is explicit — a non-admin attaches a picture to their
|
| 489 |
+
# own record. The editorial upload above IS platform-admin-only because it publishes into
|
| 490 |
+
# the shared catalogue library; this writes a per-record attachment, which is the same class of
|
| 491 |
# act as typing a note.
|
| 492 |
# 2. TENANT FROM THE SESSION, NEVER FROM THE STRING. The stored address is `<tenant>/<id>` and
|
| 493 |
# both halves of it are composed server-side. A `rec:` reference pasted from another
|
api/routes_auth.py
CHANGED
|
@@ -13,7 +13,7 @@ import re as _re
|
|
| 13 |
from fastapi import APIRouter, Body, Depends, Request, Response
|
| 14 |
|
| 15 |
import aios_session
|
| 16 |
-
from core import audit
|
| 17 |
from deps import Session, err, require_session, users, perms
|
| 18 |
|
| 19 |
router = APIRouter(prefix="/api/v1/auth")
|
|
@@ -310,7 +310,7 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 310 |
_clear_failures(throttle_key)
|
| 311 |
session_epoch = int(user.get("epoch") or 0)
|
| 312 |
epoch_bumped = False
|
| 313 |
-
if not user.get("qa_runner"):
|
| 314 |
try:
|
| 315 |
bumped = users.bump_epoch(user["username"])
|
| 316 |
if bumped is not None:
|
|
|
|
| 13 |
from fastapi import APIRouter, Body, Depends, Request, Response
|
| 14 |
|
| 15 |
import aios_session
|
| 16 |
+
from core import audit, freeze_notice
|
| 17 |
from deps import Session, err, require_session, users, perms
|
| 18 |
|
| 19 |
router = APIRouter(prefix="/api/v1/auth")
|
|
|
|
| 310 |
_clear_failures(throttle_key)
|
| 311 |
session_epoch = int(user.get("epoch") or 0)
|
| 312 |
epoch_bumped = False
|
| 313 |
+
if not user.get("qa_runner") and not freeze_notice.enabled():
|
| 314 |
try:
|
| 315 |
bumped = users.bump_epoch(user["username"])
|
| 316 |
if bumped is not None:
|
api/routes_connectors.py
CHANGED
|
@@ -22,6 +22,8 @@ from __future__ import annotations
|
|
| 22 |
|
| 23 |
import os
|
| 24 |
import copy
|
|
|
|
|
|
|
| 25 |
import ssl
|
| 26 |
import time
|
| 27 |
import threading
|
|
@@ -174,7 +176,9 @@ _DATABASE_SAFE_ERRORS = {
|
|
| 174 |
"freshness_unconfirmed": "Freshness could not be confirmed for this database.",
|
| 175 |
"sync_status_unavailable": "Sync status is unavailable for this database.",
|
| 176 |
"sync_stalled": "This database sync is stalled.",
|
|
|
|
| 177 |
"sync_state_unreadable": "This database's saved sync state could not be read.",
|
|
|
|
| 178 |
"maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
|
| 179 |
"scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
|
| 180 |
"maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.",
|
|
@@ -326,11 +330,8 @@ TOKEN_CONNECTORS = [
|
|
| 326 |
"hint": "Create a Production app client in the Solution Provider Portal, self authorize "
|
| 327 |
"it, then paste client_id, client_secret, refresh_token, seller_id and "
|
| 328 |
"marketplace_id.",
|
| 329 |
-
#
|
| 330 |
-
|
| 331 |
-
"note": "Choose Test in the Keychain tab to verify the credential live against your "
|
| 332 |
-
"marketplace participations. Bulk reads use the Reports API, so a backfill is one "
|
| 333 |
-
"document rather than a page loop."},
|
| 334 |
]
|
| 335 |
|
| 336 |
#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
|
|
@@ -851,6 +852,110 @@ def _amazon_rotation_expired(deadline):
|
|
| 851 |
return False
|
| 852 |
|
| 853 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 854 |
def directory(rt, uname, is_admin=False, incomplete=None):
|
| 855 |
"""The composed row set. Pure over the session, so the gate drives it without a request.
|
| 856 |
|
|
@@ -899,10 +1004,19 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 899 |
# would have flipped a real connection to "available" — the silent case W31-T44 exists to
|
| 900 |
# prevent, re-introduced by the fix for its sibling.
|
| 901 |
def _odoo():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 902 |
rows.append({
|
| 903 |
"key": "odoo", "label": "Odoo", "kind": "builtin",
|
| 904 |
"desc": "Sales, invoices, products and customers from your Odoo ERP.",
|
| 905 |
-
"state": ("
|
|
|
|
| 906 |
else "available"),
|
| 907 |
# W32-T11 / C1 — always business: Odoo is what the whole workspace's databases are
|
| 908 |
# read through, which is why `routes_keychain.TENANT_WIDE_TYPES` refuses to make it
|
|
@@ -998,13 +1112,9 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 998 |
# note IS the generic sentence, so theirs is unchanged).
|
| 999 |
"note": t["note"],
|
| 1000 |
})
|
| 1001 |
-
#
|
| 1002 |
-
# but no passing Test is on record for it" (`keychain.entry_verified`), measured on
|
| 1003 |
-
# staging's admin view with the data still syncing. The bare word read as a broken
|
| 1004 |
-
# connection with no way to tell what to do.
|
| 1005 |
if rows[-1]["state"] == "reconnect" and t["entry"] == "amazon_sp":
|
| 1006 |
-
rows[-1]["needs"] =
|
| 1007 |
-
"Choose Test in the Keychain tab, or replace the key.")
|
| 1008 |
|
| 1009 |
contribute("token", _tokens,
|
| 1010 |
"the token-connector registry could not be composed",
|
|
@@ -1036,6 +1146,7 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 1036 |
"key": "webhooks", "label": "Webhooks", "kind": "builtin",
|
| 1037 |
"desc": "Let another system start an automation by calling a URL.",
|
| 1038 |
"state": "connected", "manage": "automation",
|
|
|
|
| 1039 |
})
|
| 1040 |
|
| 1041 |
contribute("builtin", _builtin, "a built-in connector row could not be composed",
|
|
@@ -1109,7 +1220,14 @@ def directory(rt, uname, is_admin=False, incomplete=None):
|
|
| 1109 |
for row in rows:
|
| 1110 |
row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key"))))
|
| 1111 |
_safe_connector_status(row.get("key"), row)
|
| 1112 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1113 |
|
| 1114 |
|
| 1115 |
def _oauth_row(slug, meta, row_st):
|
|
@@ -1274,7 +1392,8 @@ def connectors_directory(session: Session = Depends(require_session)):
|
|
| 1274 |
# `connectionKey` with `graphRefusal(selectedKey, ...)`, so a card with no envelope on the
|
| 1275 |
# wire renders exactly as it did before. The list comprehension holds the SAME dicts, so the
|
| 1276 |
# in-place attachment still lands on the native rows.
|
| 1277 |
-
attach_relationship_graphs([r for r in rows if r.get("provider") != "monid"
|
|
|
|
| 1278 |
session.user, session.runtime, session.uname,
|
| 1279 |
bool(session.admin))
|
| 1280 |
# ⛔⛔ W49-T18 / C4 — THE ONE PLACE THE SERVER ROW BECOMES THE CLIENT DTO. `provider`
|
|
@@ -1323,6 +1442,8 @@ def connectors_directory(session: Session = Depends(require_session)):
|
|
| 1323 |
_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
|
| 1324 |
_ODOO_SYNC_STATUS = "odoo_sync_status"
|
| 1325 |
_ODOO_HEALTH_STATUS = "odoo_health_status_v1"
|
|
|
|
|
|
|
| 1326 |
_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness")
|
| 1327 |
_SYNC_REQUEST_CONNECTIONS = {}
|
| 1328 |
_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock()
|
|
@@ -1624,6 +1745,8 @@ def odoo_health_receipt(rt, connection_id=None, *, now=None):
|
|
| 1624 |
except Exception: # noqa: BLE001
|
| 1625 |
document = {}
|
| 1626 |
stored = dict(((document.get("connections") or {}).get(cid)) or {})
|
|
|
|
|
|
|
| 1627 |
stages = _empty_odoo_stages()
|
| 1628 |
stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items()
|
| 1629 |
if key in stages and isinstance(value, dict)})
|
|
@@ -1717,12 +1840,12 @@ def odoo_health_receipt(rt, connection_id=None, *, now=None):
|
|
| 1717 |
last_success = max(successes, default=stored.get("lastSuccess"))
|
| 1718 |
row_counts = [row.get("rowCount") for row in databases.values()
|
| 1719 |
if row.get("rowCount") is not None]
|
| 1720 |
-
failure = (_odoo_stage_failure(stages) or stored.get("failure") or next(
|
| 1721 |
(row.get("failure") for row in databases.values() if row.get("failure")), None))
|
| 1722 |
return {"connectionId": cid, "cadenceSeconds": cadence,
|
| 1723 |
"lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now),
|
| 1724 |
"rowCount": sum(row_counts) if row_counts else None,
|
| 1725 |
-
"failure": failure, "stages": stages,
|
| 1726 |
"databases": [databases[key] for key in sorted(databases)]}
|
| 1727 |
|
| 1728 |
|
|
@@ -1762,8 +1885,9 @@ def _sync_payload(connector_key, connection_id, row=None):
|
|
| 1762 |
|
| 1763 |
|
| 1764 |
def _odoo_result_database_failures(rt, connection_id, result):
|
| 1765 |
-
"""Map provider
|
| 1766 |
import odoo_relational as rel
|
|
|
|
| 1767 |
try:
|
| 1768 |
choices = rel.grid_choices(rt, connection_id)
|
| 1769 |
except Exception: # noqa: BLE001
|
|
@@ -1807,7 +1931,7 @@ def _odoo_result_database_failures(rt, connection_id, result):
|
|
| 1807 |
failures[key] = _odoo_failure(raw, "materialization")
|
| 1808 |
elif limits:
|
| 1809 |
unmatched.append(str(limits))
|
| 1810 |
-
return failures, unmatched
|
| 1811 |
|
| 1812 |
|
| 1813 |
def _odoo_sync_runner(session, connection_id):
|
|
@@ -2074,15 +2198,16 @@ def _run_database_job(rt, candidate):
|
|
| 2074 |
username="automation", table_keys={key},
|
| 2075 |
on_progress=lambda rows, total: connector_sync.progress(
|
| 2076 |
fenced, job, rows=rows, total=total))}
|
| 2077 |
-
|
|
|
|
| 2078 |
published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
|
| 2079 |
-
receipt = (result.get("databases") or {})
|
|
|
|
| 2080 |
receipt = receipt if isinstance(receipt, dict) else None
|
| 2081 |
-
database_errors =
|
| 2082 |
-
success = (key in published and not
|
| 2083 |
-
and not
|
| 2084 |
-
and key not in database_errors
|
| 2085 |
-
and (receipt is None or receipt.get("state") == "succeeded"))
|
| 2086 |
counts = published.get(key) if isinstance(published.get(key), dict) else {}
|
| 2087 |
nrows = (receipt.get("servingCount") if receipt is not None
|
| 2088 |
else counts.get("rows") if isinstance(counts, dict) else None)
|
|
@@ -2229,9 +2354,25 @@ def _dispatch_database_jobs(rt):
|
|
| 2229 |
continue
|
| 2230 |
|
| 2231 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2232 |
@router.on_event("startup")
|
| 2233 |
def _start_database_jobs():
|
| 2234 |
"""Discover durable work at boot, including tenants that have no active browser session."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2235 |
global _materialization_thread
|
| 2236 |
def run():
|
| 2237 |
global _materialization_thread
|
|
@@ -2249,12 +2390,33 @@ def _start_database_jobs():
|
|
| 2249 |
for tenant in tenants:
|
| 2250 |
if _materialization_stop.is_set():
|
| 2251 |
break
|
|
|
|
| 2252 |
try:
|
| 2253 |
-
|
| 2254 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2255 |
continue # Durable queued/leased work is retried by the next scan.
|
| 2256 |
-
|
| 2257 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2258 |
_materialization_wake.wait(15)
|
| 2259 |
with _materialization_lifecycle:
|
| 2260 |
_materialization_stop.clear()
|
|
@@ -2373,6 +2535,11 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
|
|
| 2373 |
lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection)
|
| 2374 |
durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
|
| 2375 |
receipt = odoo_health_receipt(session.runtime, resolved_connection)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2376 |
if durable:
|
| 2377 |
return {"schedule": cfg.get("syncEvery"),
|
| 2378 |
"health": "paused" if cfg.get("frozen") else durable["state"],
|
|
@@ -2391,6 +2558,10 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
|
|
| 2391 |
or "queued")
|
| 2392 |
errors_total = ((lifecycle or {}).get("errorsTotal", 0) if named or lifecycle
|
| 2393 |
else len(errors))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2394 |
return {"schedule": cfg.get("syncEvery"),
|
| 2395 |
"health": "paused" if cfg.get("frozen") else
|
| 2396 |
shown_phase if named else "error" if shown_errors else shown_phase,
|
|
@@ -2954,10 +3125,14 @@ _AMAZON_FIELD_LABELS = {"client_id": "Client ID", "client_secret": "Client secre
|
|
| 2954 |
"refresh_token": "Refresh token", "seller_id": "Seller ID",
|
| 2955 |
"marketplace_id": "Marketplace ID"}
|
| 2956 |
_CONNECT_FIELDS = {
|
| 2957 |
-
"odoo": [{"key": "url", "label": "Server URL
|
| 2958 |
-
|
| 2959 |
-
{"key": "
|
| 2960 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2961 |
# ⭐ THE KEYS AND THEIR ORDER ARE DERIVED, NEVER TYPED. `amazon_client.CREDENTIAL_FIELDS` is
|
| 2962 |
# the one roster, and lane G's `verify_amazon` asserts this sequence against it, so a field
|
| 2963 |
# added there appears here rather than drifting. Labels are the one thing NOT derived:
|
|
@@ -2967,15 +3142,86 @@ _CONNECT_FIELDS = {
|
|
| 2967 |
{"key": key,
|
| 2968 |
"label": _AMAZON_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()),
|
| 2969 |
"secret": key in ("client_secret", "refresh_token"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2970 |
**({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
|
| 2971 |
for key in amazon_client.CREDENTIAL_FIELDS
|
| 2972 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2973 |
}
|
| 2974 |
#: Everything else is one pasted token. The field name is `api_key` because that is the first of
|
| 2975 |
#: `core.keychain._PREVIEW_FIELDS`, so the stored entry gets a preview rather than a blank cell —
|
| 2976 |
#: the wave-31 defect that comment records, avoided rather than repeated.
|
| 2977 |
_CONNECT_TOKEN_FIELD = [{"key": "api_key", "label": "API key", "secret": True}]
|
| 2978 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2979 |
|
| 2980 |
def _safe_connector_status(connector_key, payload):
|
| 2981 |
"""Apply the provider's disclosure guard at the status boundary, outside fallback catches."""
|
|
@@ -3059,7 +3305,7 @@ def _odoo_database_sync_health(raw, enabled=True, *, now=None):
|
|
| 3059 |
row_count=row.get("rowCount"), error=failure, now=now)
|
| 3060 |
|
| 3061 |
|
| 3062 |
-
def _amazon_database_sync_health(raw, enabled=True, *, now=None):
|
| 3063 |
if not enabled:
|
| 3064 |
return _database_sync_health(enabled=False)
|
| 3065 |
if not isinstance(raw, dict):
|
|
@@ -3067,7 +3313,15 @@ def _amazon_database_sync_health(raw, enabled=True, *, now=None):
|
|
| 3067 |
state="failed", error={"code": "sync_status_unavailable",
|
| 3068 |
"message": "Sync status is unavailable for this database."})
|
| 3069 |
success = raw.get("lastSuccess")
|
| 3070 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3071 |
state = "partial" if success else "failed"
|
| 3072 |
error = {"code": "sync_stalled",
|
| 3073 |
"message": "This database sync is stalled."}
|
|
@@ -3108,8 +3362,18 @@ def _detail_connect(entry_type, is_admin):
|
|
| 3108 |
return None
|
| 3109 |
fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
|
| 3110 |
tenant_wide = entry_type in TENANT_WIDE_TYPES
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3111 |
return {"type": entry_type,
|
|
|
|
| 3112 |
"fields": [dict(field) for field in fields],
|
|
|
|
|
|
|
| 3113 |
# A member may always store a personal one; business-wide is an administrator's to
|
| 3114 |
# give (R4), and a tenant-wide type is business-wide by construction.
|
| 3115 |
"scopes": (["business"] if tenant_wide
|
|
@@ -3756,6 +4020,9 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None,
|
|
| 3756 |
"databaseKey": key, "state": "unknown", "rowCount": None,
|
| 3757 |
"lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None,
|
| 3758 |
"failure": None, "stages": _empty_odoo_stages()}
|
|
|
|
|
|
|
|
|
|
| 3759 |
out.append({"key": key,
|
| 3760 |
"identity": connector_sync.database_identity(
|
| 3761 |
"odoo", resolved_connection_id, key, model),
|
|
@@ -3881,6 +4148,42 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 3881 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 3882 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 3883 |
raw_sync = (states or {}).get(name) if is_admin else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3884 |
is_enabled = enabled.get(key) is True
|
| 3885 |
out.append({"key": key,
|
| 3886 |
"label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
|
|
@@ -3892,7 +4195,7 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 3892 |
# definition untouched: only this provider config document changes.
|
| 3893 |
"enabled": is_enabled,
|
| 3894 |
**({"sync": raw_sync} if is_admin else {}),
|
| 3895 |
-
"syncHealth": _amazon_database_sync_health(raw_sync, is_enabled),
|
| 3896 |
"fields": fields,
|
| 3897 |
"excluded": excluded,
|
| 3898 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
|
|
|
| 22 |
|
| 23 |
import os
|
| 24 |
import copy
|
| 25 |
+
import logging
|
| 26 |
+
import re
|
| 27 |
import ssl
|
| 28 |
import time
|
| 29 |
import threading
|
|
|
|
| 176 |
"freshness_unconfirmed": "Freshness could not be confirmed for this database.",
|
| 177 |
"sync_status_unavailable": "Sync status is unavailable for this database.",
|
| 178 |
"sync_stalled": "This database sync is stalled.",
|
| 179 |
+
"amazon_report_refused": "Amazon refused this report. Check seller eligibility and connection access, then retry sync.",
|
| 180 |
"sync_state_unreadable": "This database's saved sync state could not be read.",
|
| 181 |
+
"database_dispatch_failed": "Database sync could not start.",
|
| 182 |
"maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
|
| 183 |
"scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
|
| 184 |
"maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.",
|
|
|
|
| 330 |
"hint": "Create a Production app client in the Solution Provider Portal, self authorize "
|
| 331 |
"it, then paste client_id, client_secret, refresh_token, seller_id and "
|
| 332 |
"marketplace_id.",
|
| 333 |
+
# R22: status comes from the stored verification state, without mechanism copy.
|
| 334 |
+
"note": ""},
|
|
|
|
|
|
|
|
|
|
| 335 |
]
|
| 336 |
|
| 337 |
#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
|
|
|
|
| 852 |
return False
|
| 853 |
|
| 854 |
|
| 855 |
+
AI_CONTACT_CLIENTS = (
|
| 856 |
+
("claude", "Claude.ai", ("claude",)),
|
| 857 |
+
("chatgpt", "ChatGPT", ("chatgpt", "chat gpt")),
|
| 858 |
+
("claude-code", "Claude Code", ("claude code", "claude-code", "claude_code")),
|
| 859 |
+
("codex", "Codex", ("codex",)),
|
| 860 |
+
("cursor", "Cursor", ("cursor",)),
|
| 861 |
+
)
|
| 862 |
+
|
| 863 |
+
|
| 864 |
+
def _external_client_id(connection):
|
| 865 |
+
"""Match registered client metadata, with specific names ahead of broad brand names."""
|
| 866 |
+
declared = connection.get("clientId") or connection.get("client_id")
|
| 867 |
+
if declared in {client[0] for client in AI_CONTACT_CLIENTS}:
|
| 868 |
+
return declared
|
| 869 |
+
label = str(connection.get("label") or "").lower()
|
| 870 |
+
matches = [(len(alias), key) for key, _label, aliases in AI_CONTACT_CLIENTS
|
| 871 |
+
for alias in aliases if alias in label]
|
| 872 |
+
return max(matches, default=(0, ""))[1]
|
| 873 |
+
|
| 874 |
+
|
| 875 |
+
def _external_active(record):
|
| 876 |
+
if record.get("revoked_at") or record.get("disconnected_at"):
|
| 877 |
+
return False
|
| 878 |
+
expiry = record.get("expires_at")
|
| 879 |
+
if not expiry:
|
| 880 |
+
return True
|
| 881 |
+
try:
|
| 882 |
+
stamp = datetime.fromisoformat(str(expiry).replace("Z", "+00:00"))
|
| 883 |
+
return stamp.tzinfo is not None and stamp > datetime.now(timezone.utc)
|
| 884 |
+
except (TypeError, ValueError):
|
| 885 |
+
return False
|
| 886 |
+
|
| 887 |
+
|
| 888 |
+
def _contact_rows(rt, uname, is_admin, incomplete):
|
| 889 |
+
"""Read existing channel and external metadata through their tenant/user walls. No writes."""
|
| 890 |
+
rows = [
|
| 891 |
+
{"key": "whatsapp", "label": "WhatsApp", "kind": "channel", "manage": "channel"},
|
| 892 |
+
{"key": "imessage", "label": "iMessage", "kind": "channel", "manage": ""},
|
| 893 |
+
{"key": "slack", "label": "Slack", "kind": "channel", "manage": "channel"},
|
| 894 |
+
*[{"key": key, "label": label, "kind": "mcp", "manage": "external",
|
| 895 |
+
"clientId": key} for key, label, _aliases in AI_CONTACT_CLIENTS],
|
| 896 |
+
{"key": "loopable-api", "label": "Loopable API", "kind": "api", "manage": "external"},
|
| 897 |
+
]
|
| 898 |
+
for row in rows:
|
| 899 |
+
row.update(section="tools" if row["kind"] == "api" else "contact",
|
| 900 |
+
state="available", action="connect", desc="", scope="personal")
|
| 901 |
+
by_key = {row["key"]: row for row in rows}
|
| 902 |
+
by_key["imessage"].update(state="soon", action="soon")
|
| 903 |
+
|
| 904 |
+
def read(source, keys, fn):
|
| 905 |
+
try:
|
| 906 |
+
fn()
|
| 907 |
+
except Exception: # noqa: BLE001
|
| 908 |
+
for key in keys:
|
| 909 |
+
by_key[key]["state"] = "unavailable"
|
| 910 |
+
incomplete.append({"source": source, "cause": "Connection status unavailable.",
|
| 911 |
+
"recommendation": "Try again."})
|
| 912 |
+
|
| 913 |
+
def whatsapp():
|
| 914 |
+
import whatsapp_identity
|
| 915 |
+
if whatsapp_identity.status(rt.key, uname).get("state") == "connected":
|
| 916 |
+
by_key["whatsapp"]["state"] = "connected"
|
| 917 |
+
|
| 918 |
+
def slack():
|
| 919 |
+
from routes_slack import _slack_entries, _slack_links, _team_links
|
| 920 |
+
installs = _slack_entries(rt)
|
| 921 |
+
links = _slack_links(rt)
|
| 922 |
+
if any(is_admin or uname in _team_links(links, fields["team_id"]).values()
|
| 923 |
+
for _row, fields in installs):
|
| 924 |
+
by_key["slack"]["state"] = "connected"
|
| 925 |
+
|
| 926 |
+
def mcp():
|
| 927 |
+
from core import external_oauth
|
| 928 |
+
for grant in external_oauth.list_connections(rt, uname, rt.key):
|
| 929 |
+
key = _external_client_id(grant)
|
| 930 |
+
if key and _external_active(grant):
|
| 931 |
+
by_key[key]["state"] = "connected"
|
| 932 |
+
|
| 933 |
+
def api():
|
| 934 |
+
from core import external_access
|
| 935 |
+
if any(_external_active(key) for key in external_access.list_for_user(rt, uname, rt.key)):
|
| 936 |
+
by_key["loopable-api"]["state"] = "connected"
|
| 937 |
+
|
| 938 |
+
read("whatsapp", ["whatsapp"], whatsapp)
|
| 939 |
+
read("slack", ["slack"], slack)
|
| 940 |
+
read("mcp", [client[0] for client in AI_CONTACT_CLIENTS], mcp)
|
| 941 |
+
read("loopable-api", ["loopable-api"], api)
|
| 942 |
+
for row in rows:
|
| 943 |
+
if row["state"] == "connected":
|
| 944 |
+
row["action"] = "settings"
|
| 945 |
+
return rows
|
| 946 |
+
|
| 947 |
+
|
| 948 |
+
def _directory_card_action(row, is_admin):
|
| 949 |
+
"""C2: one card action; paid operations remain separate and behind the settings door."""
|
| 950 |
+
if row.get("action") not in (None, "connect", "settings", "soon"):
|
| 951 |
+
row["operation"] = row["action"]
|
| 952 |
+
held = row.get("state") in ("connected", "reconnect", "expired")
|
| 953 |
+
row["section"] = "tools"
|
| 954 |
+
row["action"] = "settings" if held else "connect"
|
| 955 |
+
row["actionDisabled"] = not (held or row.get("manage")
|
| 956 |
+
or (row.get("operation") and is_admin))
|
| 957 |
+
|
| 958 |
+
|
| 959 |
def directory(rt, uname, is_admin=False, incomplete=None):
|
| 960 |
"""The composed row set. Pure over the session, so the gate drives it without a request.
|
| 961 |
|
|
|
|
| 1004 |
# would have flipped a real connection to "available" — the silent case W31-T44 exists to
|
| 1005 |
# prevent, re-introduced by the fix for its sibling.
|
| 1006 |
def _odoo():
|
| 1007 |
+
# S70-T37 (R4/D-202): a DISCONNECT freezes the tenant's mirror durably, and a frozen tenant
|
| 1008 |
+
# reads "disconnected" even while Royal's environment credential is still present. The read
|
| 1009 |
+
# is fresh and tenant-bound, so one tenant's freeze never shows on another tenant's card.
|
| 1010 |
+
try:
|
| 1011 |
+
cfg = rt.get("odoo_connector_config", fresh=True)
|
| 1012 |
+
frozen = isinstance(cfg, dict) and bool(cfg.get("frozen"))
|
| 1013 |
+
except Exception: # noqa: BLE001
|
| 1014 |
+
frozen = False
|
| 1015 |
rows.append({
|
| 1016 |
"key": "odoo", "label": "Odoo", "kind": "builtin",
|
| 1017 |
"desc": "Sales, invoices, products and customers from your Odoo ERP.",
|
| 1018 |
+
"state": ("disconnected" if frozen
|
| 1019 |
+
else "connected" if ("odoo" in have or env_odoo_available(rt))
|
| 1020 |
else "available"),
|
| 1021 |
# W32-T11 / C1 — always business: Odoo is what the whole workspace's databases are
|
| 1022 |
# read through, which is why `routes_keychain.TENANT_WIDE_TYPES` refuses to make it
|
|
|
|
| 1112 |
# note IS the generic sentence, so theirs is unchanged).
|
| 1113 |
"note": t["note"],
|
| 1114 |
})
|
| 1115 |
+
# R22: a stored Amazon key without a passing test carries a short status only.
|
|
|
|
|
|
|
|
|
|
| 1116 |
if rows[-1]["state"] == "reconnect" and t["entry"] == "amazon_sp":
|
| 1117 |
+
rows[-1]["needs"] = "Needs a test"
|
|
|
|
| 1118 |
|
| 1119 |
contribute("token", _tokens,
|
| 1120 |
"the token-connector registry could not be composed",
|
|
|
|
| 1146 |
"key": "webhooks", "label": "Webhooks", "kind": "builtin",
|
| 1147 |
"desc": "Let another system start an automation by calling a URL.",
|
| 1148 |
"state": "connected", "manage": "automation",
|
| 1149 |
+
"tutorial": _CONNECT_TUTORIALS["webhooks"],
|
| 1150 |
})
|
| 1151 |
|
| 1152 |
contribute("builtin", _builtin, "a built-in connector row could not be composed",
|
|
|
|
| 1220 |
for row in rows:
|
| 1221 |
row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key"))))
|
| 1222 |
_safe_connector_status(row.get("key"), row)
|
| 1223 |
+
_directory_card_action(row, is_admin)
|
| 1224 |
+
contact = _contact_rows(rt, uname, is_admin, short)
|
| 1225 |
+
for row in contact:
|
| 1226 |
+
# Eligibility rides every card, Contact's included, or the toggle's filter has nothing to read.
|
| 1227 |
+
row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key"))))
|
| 1228 |
+
reserved = {row["key"] for row in contact}
|
| 1229 |
+
# Slack's old sign-in-provider listing must not become a second card for the same channel.
|
| 1230 |
+
return contact + [row for row in rows if row.get("key") not in reserved]
|
| 1231 |
|
| 1232 |
|
| 1233 |
def _oauth_row(slug, meta, row_st):
|
|
|
|
| 1392 |
# `connectionKey` with `graphRefusal(selectedKey, ...)`, so a card with no envelope on the
|
| 1393 |
# wire renders exactly as it did before. The list comprehension holds the SAME dicts, so the
|
| 1394 |
# in-place attachment still lands on the native rows.
|
| 1395 |
+
attach_relationship_graphs([r for r in rows if r.get("provider") != "monid"
|
| 1396 |
+
and r.get("kind") not in ("channel", "mcp", "api")],
|
| 1397 |
session.user, session.runtime, session.uname,
|
| 1398 |
bool(session.admin))
|
| 1399 |
# ⛔⛔ W49-T18 / C4 — THE ONE PLACE THE SERVER ROW BECOMES THE CLIENT DTO. `provider`
|
|
|
|
| 1442 |
_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
|
| 1443 |
_ODOO_SYNC_STATUS = "odoo_sync_status"
|
| 1444 |
_ODOO_HEALTH_STATUS = "odoo_health_status_v1"
|
| 1445 |
+
_DATABASE_DISPATCH_STATUS = "connector_database_dispatch_status_v1"
|
| 1446 |
+
_DATABASE_JOBS_LOG = logging.getLogger(__name__)
|
| 1447 |
_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness")
|
| 1448 |
_SYNC_REQUEST_CONNECTIONS = {}
|
| 1449 |
_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock()
|
|
|
|
| 1745 |
except Exception: # noqa: BLE001
|
| 1746 |
document = {}
|
| 1747 |
stored = dict(((document.get("connections") or {}).get(cid)) or {})
|
| 1748 |
+
dispatch = rt.get(_DATABASE_DISPATCH_STATUS) or {}
|
| 1749 |
+
dispatch_failure = dispatch.get("failure") if dispatch.get("state") == "failed" else None
|
| 1750 |
stages = _empty_odoo_stages()
|
| 1751 |
stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items()
|
| 1752 |
if key in stages and isinstance(value, dict)})
|
|
|
|
| 1840 |
last_success = max(successes, default=stored.get("lastSuccess"))
|
| 1841 |
row_counts = [row.get("rowCount") for row in databases.values()
|
| 1842 |
if row.get("rowCount") is not None]
|
| 1843 |
+
failure = (dispatch_failure or _odoo_stage_failure(stages) or stored.get("failure") or next(
|
| 1844 |
(row.get("failure") for row in databases.values() if row.get("failure")), None))
|
| 1845 |
return {"connectionId": cid, "cadenceSeconds": cadence,
|
| 1846 |
"lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now),
|
| 1847 |
"rowCount": sum(row_counts) if row_counts else None,
|
| 1848 |
+
"failure": failure, "stages": stages, "dispatch": dispatch or None,
|
| 1849 |
"databases": [databases[key] for key in sorted(databases)]}
|
| 1850 |
|
| 1851 |
|
|
|
|
| 1885 |
|
| 1886 |
|
| 1887 |
def _odoo_result_database_failures(rt, connection_id, result):
|
| 1888 |
+
"""Map blocking provider failures to durable database keys without changing the report."""
|
| 1889 |
import odoo_relational as rel
|
| 1890 |
+
result = rel.health_view(result)
|
| 1891 |
try:
|
| 1892 |
choices = rel.grid_choices(rt, connection_id)
|
| 1893 |
except Exception: # noqa: BLE001
|
|
|
|
| 1931 |
failures[key] = _odoo_failure(raw, "materialization")
|
| 1932 |
elif limits:
|
| 1933 |
unmatched.append(str(limits))
|
| 1934 |
+
return rel.blocking_failures(failures, result), unmatched
|
| 1935 |
|
| 1936 |
|
| 1937 |
def _odoo_sync_runner(session, connection_id):
|
|
|
|
| 2198 |
username="automation", table_keys={key},
|
| 2199 |
on_progress=lambda rows, total: connector_sync.progress(
|
| 2200 |
fenced, job, rows=rows, total=total))}
|
| 2201 |
+
health = rel.health_view(result)
|
| 2202 |
+
discovered = health.get("discovered") or {}
|
| 2203 |
published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
|
| 2204 |
+
receipt = {**(result.get("databases") or {}),
|
| 2205 |
+
**(result.get("canonicalDatabases") or {})}.get(key)
|
| 2206 |
receipt = receipt if isinstance(receipt, dict) else None
|
| 2207 |
+
database_errors, unmatched_errors = _odoo_result_database_failures(fenced, source_id, health)
|
| 2208 |
+
success = (key in published and not health.get("failed") and not health.get("frozen")
|
| 2209 |
+
and not health.get("limits") and not discovered.get("errors")
|
| 2210 |
+
and key not in database_errors and not unmatched_errors)
|
|
|
|
| 2211 |
counts = published.get(key) if isinstance(published.get(key), dict) else {}
|
| 2212 |
nrows = (receipt.get("servingCount") if receipt is not None
|
| 2213 |
else counts.get("rows") if isinstance(counts, dict) else None)
|
|
|
|
| 2354 |
continue
|
| 2355 |
|
| 2356 |
|
| 2357 |
+
def _record_database_dispatch(rt, exception_type=None):
|
| 2358 |
+
"""Keep a tenant-local dispatch receipt without changing queued jobs or source freshness."""
|
| 2359 |
+
if exception_type is None and (rt.get(_DATABASE_DISPATCH_STATUS) or {}).get("state") != "failed":
|
| 2360 |
+
return # Healthy scans do not create a write every fifteen seconds.
|
| 2361 |
+
failure = ({"code": "database_dispatch_failed", "type": exception_type,
|
| 2362 |
+
"message": _DATABASE_SAFE_ERRORS["database_dispatch_failed"]}
|
| 2363 |
+
if exception_type is not None else None)
|
| 2364 |
+
receipt = {"state": "failed" if failure else "ready",
|
| 2365 |
+
"at": datetime.now(timezone.utc).isoformat(), "failure": failure}
|
| 2366 |
+
rt.update(_DATABASE_DISPATCH_STATUS, lambda _current: receipt, flush="sync")
|
| 2367 |
+
|
| 2368 |
+
|
| 2369 |
@router.on_event("startup")
|
| 2370 |
def _start_database_jobs():
|
| 2371 |
"""Discover durable work at boot, including tenants that have no active browser session."""
|
| 2372 |
+
from core import freeze_notice
|
| 2373 |
+
if freeze_notice.enabled():
|
| 2374 |
+
print("[aios-api] maintenance freeze holds connector database jobs")
|
| 2375 |
+
return
|
| 2376 |
global _materialization_thread
|
| 2377 |
def run():
|
| 2378 |
global _materialization_thread
|
|
|
|
| 2390 |
for tenant in tenants:
|
| 2391 |
if _materialization_stop.is_set():
|
| 2392 |
break
|
| 2393 |
+
rt = None
|
| 2394 |
try:
|
| 2395 |
+
rt = runtime.get_runtime(tenant)
|
| 2396 |
+
_dispatch_database_jobs(rt)
|
| 2397 |
+
except Exception as exc:
|
| 2398 |
+
exception_type = type(exc).__name__
|
| 2399 |
+
_DATABASE_JOBS_LOG.error(
|
| 2400 |
+
"connector database dispatch failed: tenant=%s type=%s",
|
| 2401 |
+
tenant, exception_type)
|
| 2402 |
+
if rt is not None:
|
| 2403 |
+
try:
|
| 2404 |
+
_record_database_dispatch(rt, exception_type)
|
| 2405 |
+
except Exception as receipt_exc:
|
| 2406 |
+
_DATABASE_JOBS_LOG.error(
|
| 2407 |
+
"connector database dispatch receipt failed: tenant=%s type=%s",
|
| 2408 |
+
tenant, type(receipt_exc).__name__)
|
| 2409 |
continue # Durable queued/leased work is retried by the next scan.
|
| 2410 |
+
else:
|
| 2411 |
+
try:
|
| 2412 |
+
_record_database_dispatch(rt)
|
| 2413 |
+
except Exception as receipt_exc:
|
| 2414 |
+
_DATABASE_JOBS_LOG.error(
|
| 2415 |
+
"connector database dispatch recovery receipt failed: tenant=%s type=%s",
|
| 2416 |
+
tenant, type(receipt_exc).__name__)
|
| 2417 |
+
except Exception as exc:
|
| 2418 |
+
_DATABASE_JOBS_LOG.error(
|
| 2419 |
+
"connector database tenant scan failed: type=%s", type(exc).__name__)
|
| 2420 |
_materialization_wake.wait(15)
|
| 2421 |
with _materialization_lifecycle:
|
| 2422 |
_materialization_stop.clear()
|
|
|
|
| 2535 |
lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection)
|
| 2536 |
durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
|
| 2537 |
receipt = odoo_health_receipt(session.runtime, resolved_connection)
|
| 2538 |
+
dispatch_failure = (receipt.get("dispatch") or {}).get("failure")
|
| 2539 |
+
if dispatch_failure and durable:
|
| 2540 |
+
durable = {**durable, "state": "error",
|
| 2541 |
+
"errors": [dispatch_failure["message"], *(durable.get("errors") or [])][:20],
|
| 2542 |
+
"errorsTotal": durable.get("errorsTotal", 0) + 1}
|
| 2543 |
if durable:
|
| 2544 |
return {"schedule": cfg.get("syncEvery"),
|
| 2545 |
"health": "paused" if cfg.get("frozen") else durable["state"],
|
|
|
|
| 2558 |
or "queued")
|
| 2559 |
errors_total = ((lifecycle or {}).get("errorsTotal", 0) if named or lifecycle
|
| 2560 |
else len(errors))
|
| 2561 |
+
if dispatch_failure:
|
| 2562 |
+
shown_errors = [dispatch_failure["message"], *shown_errors]
|
| 2563 |
+
shown_phase, state = "error", "error"
|
| 2564 |
+
errors_total += 1
|
| 2565 |
return {"schedule": cfg.get("syncEvery"),
|
| 2566 |
"health": "paused" if cfg.get("frozen") else
|
| 2567 |
shown_phase if named else "error" if shown_errors else shown_phase,
|
|
|
|
| 3125 |
"refresh_token": "Refresh token", "seller_id": "Seller ID",
|
| 3126 |
"marketplace_id": "Marketplace ID"}
|
| 3127 |
_CONNECT_FIELDS = {
|
| 3128 |
+
"odoo": [{"key": "url", "label": "Server URL", "secret": False,
|
| 3129 |
+
"hint": "Your Odoo address, starting with https://."},
|
| 3130 |
+
{"key": "db", "label": "Database", "secret": False,
|
| 3131 |
+
"hint": "The database name from your Odoo administrator."},
|
| 3132 |
+
{"key": "user", "label": "API user", "secret": False,
|
| 3133 |
+
"hint": "The login of the user who created the API key."},
|
| 3134 |
+
{"key": "api_key", "label": "API key", "secret": True,
|
| 3135 |
+
"hint": "The key from My profile, Account security, API keys."}],
|
| 3136 |
# ⭐ THE KEYS AND THEIR ORDER ARE DERIVED, NEVER TYPED. `amazon_client.CREDENTIAL_FIELDS` is
|
| 3137 |
# the one roster, and lane G's `verify_amazon` asserts this sequence against it, so a field
|
| 3138 |
# added there appears here rather than drifting. Labels are the one thing NOT derived:
|
|
|
|
| 3142 |
{"key": key,
|
| 3143 |
"label": _AMAZON_FIELD_LABELS.get(key, key.replace("_", " ").capitalize()),
|
| 3144 |
"secret": key in ("client_secret", "refresh_token"),
|
| 3145 |
+
"hint": {"client_id": "The Production app's Login with Amazon client ID.",
|
| 3146 |
+
"client_secret": "The same app's Login with Amazon client secret.",
|
| 3147 |
+
"refresh_token": "The refresh token returned when you self-authorize the app.",
|
| 3148 |
+
"seller_id": "Your selling partner ID from Seller Central.",
|
| 3149 |
+
"marketplace_id": "The marketplace this seller account can access."}.get(key, ""),
|
| 3150 |
**({"default": amazon_client.US_MARKETPLACE_ID} if key == "marketplace_id" else {})}
|
| 3151 |
for key in amazon_client.CREDENTIAL_FIELDS
|
| 3152 |
],
|
| 3153 |
+
"stripe": [{"key": "api_key", "label": "API key", "secret": True,
|
| 3154 |
+
"hint": "A restricted key with read access."}],
|
| 3155 |
+
"shopify": [{"key": "api_key", "label": "Admin API token", "secret": True,
|
| 3156 |
+
"hint": "The installed custom app's Admin API access token."}],
|
| 3157 |
+
"meta_ads": [{"key": "api_key", "label": "Access token", "secret": True,
|
| 3158 |
+
"hint": "A Marketing API token with ads_read for your ad account."}],
|
| 3159 |
}
|
| 3160 |
#: Everything else is one pasted token. The field name is `api_key` because that is the first of
|
| 3161 |
#: `core.keychain._PREVIEW_FIELDS`, so the stored entry gets a preview rather than a blank cell —
|
| 3162 |
#: the wave-31 defect that comment records, avoided rather than repeated.
|
| 3163 |
_CONNECT_TOKEN_FIELD = [{"key": "api_key", "label": "API key", "secret": True}]
|
| 3164 |
|
| 3165 |
+
# S70-T09: instructions accompany the canonical field descriptor. No credentials, probes or
|
| 3166 |
+
# writers live here. Test descriptions reflect what core.keychain.test_entry actually checks.
|
| 3167 |
+
_CONNECT_TUTORIALS = {
|
| 3168 |
+
"odoo": {
|
| 3169 |
+
"need": "An Odoo URL, database, user and API key.",
|
| 3170 |
+
"steps": ["Copy your Odoo URL and ask your administrator for the database name.",
|
| 3171 |
+
"Sign in as the API user. Open My profile, then Account security and API keys.",
|
| 3172 |
+
"Create a key and copy it now. Keep this user's login with the key."],
|
| 3173 |
+
"failureAction": "Open the server URL in a browser and confirm it is reachable. "
|
| 3174 |
+
"Then check the database name and that the API key belongs to the API user.",
|
| 3175 |
+
"testNote": "Checks server reachability only. It does not verify the user or API key.",
|
| 3176 |
+
},
|
| 3177 |
+
"amazon_sp": {
|
| 3178 |
+
"need": "A Production app client and access to your Amazon seller account.",
|
| 3179 |
+
"steps": ["Open Solution Provider Portal, then Apps. Create or open your Production app.",
|
| 3180 |
+
"Open its Login with Amazon credentials. Copy the client ID and client secret.",
|
| 3181 |
+
"Self-authorize the app for your seller account and copy the refresh token.",
|
| 3182 |
+
"In Seller Central, open Settings, Account info and copy your selling partner ID. "
|
| 3183 |
+
"Choose the marketplace that account sells in."],
|
| 3184 |
+
"failureAction": "In Solution Provider Portal, confirm the client ID and secret belong to "
|
| 3185 |
+
"the same Production app. Self-authorize that app again for this seller "
|
| 3186 |
+
"and check its marketplace access. Correct the saved key in Settings.",
|
| 3187 |
+
"testNote": "Checks marketplace access. Saves the key before testing.",
|
| 3188 |
+
},
|
| 3189 |
+
"stripe": {
|
| 3190 |
+
"need": "Access to your Stripe dashboard's API keys.",
|
| 3191 |
+
"steps": ["Open the Stripe dashboard, then Developers and API keys.",
|
| 3192 |
+
"Create a restricted key with read access and copy it."],
|
| 3193 |
+
"failureAction": "Open Stripe's API keys and copy a current restricted key with read access.",
|
| 3194 |
+
"testNote": "Checks that a key is stored. Stripe access is not verified.",
|
| 3195 |
+
},
|
| 3196 |
+
"shopify": {
|
| 3197 |
+
"need": "Permission to create a custom app in your Shopify store.",
|
| 3198 |
+
"steps": ["In your store admin, open Settings, Apps and sales channels, then Develop apps.",
|
| 3199 |
+
"Create a custom app, configure its read permissions and install it.",
|
| 3200 |
+
"Open API credentials and copy its Admin API access token."],
|
| 3201 |
+
"failureAction": "Open the installed custom app's API credentials and copy its Admin API token.",
|
| 3202 |
+
"testNote": "Checks that a token is stored. Shopify access is not verified.",
|
| 3203 |
+
},
|
| 3204 |
+
"meta_ads": {
|
| 3205 |
+
"need": "A Meta app and access to the ad account you want to read.",
|
| 3206 |
+
"steps": ["Open Meta for Developers and select your app.",
|
| 3207 |
+
"Use Graph API Explorer to get a Marketing API token with ads_read.",
|
| 3208 |
+
"Confirm the token's user can access your ad account, then copy the token."],
|
| 3209 |
+
"failureAction": "Generate a current Marketing API token with ads_read and confirm its "
|
| 3210 |
+
"user has access to the ad account.",
|
| 3211 |
+
"testNote": "Checks access to your ad accounts. Saves the token before testing.",
|
| 3212 |
+
},
|
| 3213 |
+
"webhooks": {
|
| 3214 |
+
"need": "An automation and a system that can send an HTTP POST.",
|
| 3215 |
+
"steps": ["Go to Automations and create or open an automation.",
|
| 3216 |
+
"Choose the Webhooks connector trigger. Saving it creates its token.",
|
| 3217 |
+
"Use your Loopable host followed by /api/v1/automations/{automation_id}/hook/{token}. "
|
| 3218 |
+
"Use this automation's ID and trigger token; keep the token private.",
|
| 3219 |
+
"Paste that URL into your sending system."],
|
| 3220 |
+
"failureAction": "Check the automation ID and token in the URL, then confirm the trigger is enabled.",
|
| 3221 |
+
"testNote": "Sending to the hook runs the automation. Check its run history after an intentional send.",
|
| 3222 |
+
},
|
| 3223 |
+
}
|
| 3224 |
+
|
| 3225 |
|
| 3226 |
def _safe_connector_status(connector_key, payload):
|
| 3227 |
"""Apply the provider's disclosure guard at the status boundary, outside fallback catches."""
|
|
|
|
| 3305 |
row_count=row.get("rowCount"), error=failure, now=now)
|
| 3306 |
|
| 3307 |
|
| 3308 |
+
def _amazon_database_sync_health(raw, enabled=True, *, now=None, database_key=""):
|
| 3309 |
if not enabled:
|
| 3310 |
return _database_sync_health(enabled=False)
|
| 3311 |
if not isinstance(raw, dict):
|
|
|
|
| 3313 |
state="failed", error={"code": "sync_status_unavailable",
|
| 3314 |
"message": "Sync status is unavailable for this database."})
|
| 3315 |
success = raw.get("lastSuccess")
|
| 3316 |
+
table = str(database_key).removeprefix("ut_amzn_")
|
| 3317 |
+
from harness.amazon_store import is_refusal_reason
|
| 3318 |
+
refused = (raw.get("refused") is True or is_refusal_reason(raw.get("unknownCause")) or any(item.get("table") == table for item in
|
| 3319 |
+
((raw.get("firstLoad") or {}).get("refused") or ()) if isinstance(item, dict))
|
| 3320 |
+
)
|
| 3321 |
+
if refused:
|
| 3322 |
+
state = "partial" if success else "failed"
|
| 3323 |
+
error = {"code": "amazon_report_refused"}
|
| 3324 |
+
elif raw.get("stuck") is True:
|
| 3325 |
state = "partial" if success else "failed"
|
| 3326 |
error = {"code": "sync_stalled",
|
| 3327 |
"message": "This database sync is stalled."}
|
|
|
|
| 3362 |
return None
|
| 3363 |
fields = list(_CONNECT_FIELDS.get(entry_type) or _CONNECT_TOKEN_FIELD)
|
| 3364 |
tenant_wide = entry_type in TENANT_WIDE_TYPES
|
| 3365 |
+
hint = ""
|
| 3366 |
+
try:
|
| 3367 |
+
hint = next((str(token.get("hint") or "") for token in TOKEN_CONNECTORS
|
| 3368 |
+
if token.get("entry") == entry_type), "")
|
| 3369 |
+
except Exception: # A failed token registry must not take away Odoo's setup descriptor.
|
| 3370 |
+
pass
|
| 3371 |
+
from connector_onboarding import choices
|
| 3372 |
return {"type": entry_type,
|
| 3373 |
+
"initialDatabases": choices(entry_type),
|
| 3374 |
"fields": [dict(field) for field in fields],
|
| 3375 |
+
"hint": hint,
|
| 3376 |
+
"tutorial": _CONNECT_TUTORIALS.get(entry_type),
|
| 3377 |
# A member may always store a personal one; business-wide is an administrator's to
|
| 3378 |
# give (R4), and a tenant-wide type is business-wide by construction.
|
| 3379 |
"scopes": (["business"] if tenant_wide
|
|
|
|
| 4020 |
"databaseKey": key, "state": "unknown", "rowCount": None,
|
| 4021 |
"lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None,
|
| 4022 |
"failure": None, "stages": _empty_odoo_stages()}
|
| 4023 |
+
dispatch_failure = (health.get("dispatch") or {}).get("failure")
|
| 4024 |
+
if choice.get("enabled") and dispatch_failure:
|
| 4025 |
+
database_health = {**database_health, "failure": dispatch_failure}
|
| 4026 |
out.append({"key": key,
|
| 4027 |
"identity": connector_sync.database_identity(
|
| 4028 |
"odoo", resolved_connection_id, key, model),
|
|
|
|
| 4148 |
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
|
| 4149 |
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
|
| 4150 |
raw_sync = (states or {}).get(name) if is_admin else None
|
| 4151 |
+
if isinstance(raw_sync, dict):
|
| 4152 |
+
from harness.amazon_store import plain_refusal_reason, is_refusal_reason
|
| 4153 |
+
raw_sync = dict(raw_sync)
|
| 4154 |
+
raw_sync["refused"] = (raw_sync.get("refused") is True
|
| 4155 |
+
or is_refusal_reason(raw_sync.get("unknownCause")))
|
| 4156 |
+
load = raw_sync.get("firstLoad")
|
| 4157 |
+
diagnostics = {}
|
| 4158 |
+
if isinstance(load, dict) and isinstance(load.get("refused"), list):
|
| 4159 |
+
reports = []
|
| 4160 |
+
for item in load["refused"]:
|
| 4161 |
+
if not isinstance(item, dict):
|
| 4162 |
+
continue
|
| 4163 |
+
reason = str(item.get("reason") or "")
|
| 4164 |
+
reports.append({**item, "reason": plain_refusal_reason(
|
| 4165 |
+
reason, status=item.get("amazonStatus"), table=item.get("table", "")),
|
| 4166 |
+
"diagnostic": item.get("diagnostic") or reason})
|
| 4167 |
+
if item.get("table") == name:
|
| 4168 |
+
raw_sync["refused"] = True
|
| 4169 |
+
diagnostics["refusal"] = item.get("diagnostic") or reason
|
| 4170 |
+
previous_cause = str(raw_sync.get("unknownCause") or "").strip()
|
| 4171 |
+
if (previous_cause and reason.strip()
|
| 4172 |
+
and (previous_cause.startswith(reason.strip())
|
| 4173 |
+
or reason.strip().startswith(previous_cause))):
|
| 4174 |
+
diagnostics["unknownCause"] = previous_cause
|
| 4175 |
+
raw_sync["unknownCause"] = reports[-1]["reason"]
|
| 4176 |
+
raw_sync["firstLoad"] = {**load, "refused": reports}
|
| 4177 |
+
cause = str(raw_sync.get("unknownCause") or "").strip()
|
| 4178 |
+
own = next((item for item in ((raw_sync.get("firstLoad") or {}).get("refused") or ())
|
| 4179 |
+
if item.get("table") == name), {})
|
| 4180 |
+
if cause and ((raw_sync["refused"] and not own) or is_refusal_reason(cause) or re.search(
|
| 4181 |
+
r"REFUSED:|FATAL|\b\w*(?:Error|Exception)\b|GET_[A-Z_]+|[{}]|SQLSTATE|SELECT\s|report[ _-]?id\b", cause, re.I)):
|
| 4182 |
+
diagnostics["unknownCause"] = cause
|
| 4183 |
+
raw_sync["unknownCause"] = plain_refusal_reason(
|
| 4184 |
+
cause, status=own.get("amazonStatus"), table=name)
|
| 4185 |
+
if diagnostics:
|
| 4186 |
+
raw_sync["diagnostics"] = diagnostics
|
| 4187 |
is_enabled = enabled.get(key) is True
|
| 4188 |
out.append({"key": key,
|
| 4189 |
"label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
|
|
|
|
| 4195 |
# definition untouched: only this provider config document changes.
|
| 4196 |
"enabled": is_enabled,
|
| 4197 |
**({"sync": raw_sync} if is_admin else {}),
|
| 4198 |
+
"syncHealth": _amazon_database_sync_health(raw_sync, is_enabled, database_key=key),
|
| 4199 |
"fields": fields,
|
| 4200 |
"excluded": excluded,
|
| 4201 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
api/routes_customers.py
CHANGED
|
@@ -22,7 +22,7 @@ owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtim
|
|
| 22 |
proof is single-process and would report green on exactly the thing that is still broken.
|
| 23 |
"""
|
| 24 |
import datetime as _dt
|
| 25 |
-
from copy import deepcopy
|
| 26 |
from dataclasses import is_dataclass, replace
|
| 27 |
import hashlib
|
| 28 |
import json
|
|
@@ -70,16 +70,42 @@ def _assembly_pg_store(st):
|
|
| 70 |
return handle if isinstance(handle, store_pg.PgStore) else None
|
| 71 |
|
| 72 |
|
| 73 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
"""Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
|
| 75 |
from core import store, store_pg
|
| 76 |
names = tuple(sorted(set(names)))
|
| 77 |
handle = _assembly_pg_store(runtime)
|
| 78 |
users_stamp = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
if handle is None:
|
| 80 |
tokens = {name: _assembly_revision(runtime, name) for name in names}
|
| 81 |
if uses_roster:
|
| 82 |
-
users_stamp = _assembly_revision(store, "users")
|
| 83 |
return tokens, users_stamp
|
| 84 |
|
| 85 |
key = getattr(runtime, "store_key", str)
|
|
@@ -111,8 +137,8 @@ def _assembly_revisions(runtime, names, uses_roster=False):
|
|
| 111 |
absent = {"rev": 0, "updated_at": None, "token": "pg:0"}
|
| 112 |
tokens = {name: dict(found.get(mapped, absent)) for name, mapped in keyed.items()}
|
| 113 |
if uses_roster:
|
| 114 |
-
users_stamp = (dict(found.get(roster_key, absent)) if same_roster
|
| 115 |
-
|
| 116 |
if isinstance(runtime, _AssemblyStore):
|
| 117 |
# The page may read these documents after its memo hit. Reuse this request's pre-read
|
| 118 |
# tokens without retaining any document bytes across requests.
|
|
@@ -269,13 +295,17 @@ def shared_fields(st=None):
|
|
| 269 |
"""
|
| 270 |
from core import field_permissions, shared_overlay
|
| 271 |
try:
|
|
|
|
| 272 |
field_permissions.migrate_legacy_fields(
|
| 273 |
-
_shared_key(), st=st, grant_topic="customer_data", shared_key=_shared_key()
|
| 274 |
-
|
|
|
|
| 275 |
# Normalized Routes use an invisible field-shaped bridge so the established Share dialog
|
| 276 |
# and grant registry can be reused. It is object metadata, never a customer column.
|
| 277 |
return {key: value for key, value in fields.items()
|
| 278 |
if not _is_route_share_bridge(value)}
|
|
|
|
|
|
|
| 279 |
except Exception: # noqa: BLE001
|
| 280 |
# Lenient like every other display read: an unreachable store degrades to "nothing is
|
| 281 |
# shared yet", never to a 500 on a grid that would otherwise render. The WALL does not
|
|
@@ -385,8 +415,9 @@ def _mark_offerability(fields):
|
|
| 385 |
|
| 386 |
class _AssemblyStore:
|
| 387 |
"""One build's borrowed reads and their pre-read revisions, never a second cache."""
|
| 388 |
-
def __init__(self, runtime):
|
| 389 |
self._runtime = runtime
|
|
|
|
| 390 |
self.docs, self.tokens = {}, {}
|
| 391 |
self._validated_tokens = {}
|
| 392 |
# A lent/test reader without its own coherent revision door can share a flight only.
|
|
@@ -408,12 +439,20 @@ class _AssemblyStore:
|
|
| 408 |
except AttributeError:
|
| 409 |
self.cacheable = False
|
| 410 |
key = (name, shape)
|
| 411 |
-
if fresh or not self.cacheable or key not in self.docs:
|
| 412 |
self.docs[key] = read()
|
| 413 |
# TableStore readers can mutate nested projections; lend an independent read each time.
|
| 414 |
return deepcopy(self.docs[key])
|
| 415 |
|
| 416 |
def get(self, name, fresh=False):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 417 |
return self._read(name, "whole", lambda: (
|
| 418 |
self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)), fresh=fresh)
|
| 419 |
|
|
@@ -422,10 +461,17 @@ class _AssemblyStore:
|
|
| 422 |
lambda: self._runtime.get_projection(name, drop=drop))
|
| 423 |
|
| 424 |
def get_entry(self, name, entry):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
return self._read(name, ("entry", str(entry)),
|
| 426 |
lambda: self._runtime.get_entry(name, entry))
|
| 427 |
|
| 428 |
def _read_strict(self, name):
|
|
|
|
|
|
|
| 429 |
reader = getattr(self._runtime, "_read_strict", self._runtime.get)
|
| 430 |
return self._read(name, "strict", lambda: reader(name))
|
| 431 |
|
|
@@ -438,11 +484,17 @@ class _AssemblyStore:
|
|
| 438 |
return token
|
| 439 |
|
| 440 |
def update(self, name, *args, **kwargs):
|
|
|
|
|
|
|
|
|
|
| 441 |
self.cacheable = False
|
| 442 |
self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
|
| 443 |
return self._runtime.update(name, *args, **kwargs)
|
| 444 |
|
| 445 |
def put(self, name, *args, **kwargs):
|
|
|
|
|
|
|
|
|
|
| 446 |
self.cacheable = False
|
| 447 |
self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
|
| 448 |
return self._runtime.put(name, *args, **kwargs)
|
|
@@ -461,26 +513,33 @@ def _wall_stamp(session, table_key):
|
|
| 461 |
|
| 462 |
|
| 463 |
def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 464 |
-
consume_corrections: bool = False, *, build=None, pool_stamp=None
|
|
|
|
| 465 |
"""C2: one wall- and revision-scoped memo for customer, product and stored envelopes."""
|
| 466 |
global grid_assembly_shared
|
| 467 |
rt = session.runtime
|
|
|
|
| 468 |
if build is None:
|
| 469 |
-
build = lambda borrowed: _compute_grid_assembly(
|
|
|
|
| 470 |
if pool_stamp is None:
|
| 471 |
team_id, agent = _team_agent(session)
|
| 472 |
-
pool_stamp = lambda: _pool_stamp(rt, team_id, agent)
|
| 473 |
table_key = scope if scope.startswith("ut_") else (
|
| 474 |
"product_data" if scope == "product" else MODULE)
|
|
|
|
|
|
|
| 475 |
|
| 476 |
def base_key(revisions=()):
|
| 477 |
# _field_usage reads users.registry() from this global account store, including on pg.
|
| 478 |
-
uses_roster = scope != "product"
|
| 479 |
prefix = ("workspace_assembly", getattr(session, "tenant", None), scope,
|
| 480 |
getattr(session, "uname", None), getattr(session, "admin", False),
|
| 481 |
json.dumps(getattr(session, "user", {}), sort_keys=True,
|
| 482 |
separators=(",", ":"), default=str),
|
| 483 |
-
_wall_stamp(session, table_key) if memoable else None, pool_stamp()
|
|
|
|
|
|
|
| 484 |
if not memoable:
|
| 485 |
result = ((*prefix, json.dumps(None)), {})
|
| 486 |
return result
|
|
@@ -489,7 +548,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 489 |
for memo_key in list(rt.pool_cache):
|
| 490 |
if memo_key[:-2] == prefix:
|
| 491 |
names.update(name for name, _token in memo_key[-1])
|
| 492 |
-
tokens, users_stamp = _assembly_revisions(rt, names, uses_roster
|
|
|
|
| 493 |
result = ((*prefix, json.dumps(users_stamp, sort_keys=True, default=str)), tokens)
|
| 494 |
return result
|
| 495 |
|
|
@@ -505,7 +565,10 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 505 |
if not memoable:
|
| 506 |
# Partial sessions and revision-less readers keep their original build semantics,
|
| 507 |
# but concurrent callers can still borrow this flight's result.
|
| 508 |
-
|
|
|
|
|
|
|
|
|
|
| 509 |
return out
|
| 510 |
lent = assembly_rt
|
| 511 |
result = build(replace(session, runtime=lent))
|
|
@@ -520,11 +583,15 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 520 |
|
| 521 |
# Without a pool cache there is no runtime-scoped flight identity.
|
| 522 |
if not hasattr(rt, "pool_cache"):
|
| 523 |
-
|
|
|
|
|
|
|
|
|
|
| 524 |
return out
|
| 525 |
memoable = (is_dataclass(session)
|
| 526 |
and all(hasattr(session, name) for name in ("tenant", "uname", "admin", "user")))
|
| 527 |
-
assembly_rt = (rt if isinstance(rt, _AssemblyStore)
|
|
|
|
| 528 |
memoable = memoable and assembly_rt.cacheable
|
| 529 |
key, checked_tokens = base_key()
|
| 530 |
|
|
@@ -620,7 +687,7 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 620 |
|
| 621 |
|
| 622 |
def _compute_grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 623 |
-
consume_corrections: bool = True):
|
| 624 |
"""ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
|
| 625 |
events route (2026-07-31 — the standalone measure gap, owner item 1).
|
| 626 |
|
|
@@ -653,7 +720,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 653 |
|
| 654 |
rt = session.runtime
|
| 655 |
team_id, agent = _team_agent(session)
|
| 656 |
-
rows_src = _pool_for(rt, team_id, agent)
|
| 657 |
# ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that
|
| 658 |
# frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so
|
| 659 |
# scoping here means a row this account may not see never enters ANY of them, rather than
|
|
@@ -676,8 +743,9 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 676 |
# ⛔ THE SAME HANDLE GOES TO `allowed_pids` BELOW, AND THE PAIR IS NOT OPTIONAL. That is the
|
| 677 |
# WRITE wall to this one's READ wall; lending it here alone would make a user-generated rule
|
| 678 |
# narrow what an account SEES while leaving what it may PATCH untouched.
|
| 679 |
-
|
| 680 |
-
|
|
|
|
| 681 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
| 682 |
# ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because
|
| 683 |
# `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this
|
|
@@ -701,7 +769,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 701 |
# number, so a per-user leftover under the same key is stale by construction. It is also what
|
| 702 |
# makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not
|
| 703 |
# promoted.
|
| 704 |
-
_shared = shared_cells(pids, st=rt)
|
| 705 |
if _shared:
|
| 706 |
_ov = dict(ws.get("overlays") or {})
|
| 707 |
for _pid, _cells in _shared.items():
|
|
@@ -839,7 +907,7 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 839 |
|
| 840 |
today = time.strftime("%Y-%m-%d")
|
| 841 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
| 842 |
-
stamp = _pool_stamp(rt, team_id, agent)
|
| 843 |
# ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE
|
| 844 |
# OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their
|
| 845 |
# condition answers through helpers that short-circuit on an empty `offer`, so emptying the
|
|
@@ -853,10 +921,10 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 853 |
measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else []
|
| 854 |
measure_sets = measure_resolve.condition_sets(
|
| 855 |
[v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp,
|
| 856 |
-
rt.mset_memo, on_error=_measure_err) if may_metrics else {}
|
| 857 |
# The derived channel: cohort membership cells + measure column values, ONE dict — the
|
| 858 |
# same read-only channel the embed host hands to rows_from_pool.
|
| 859 |
-
derived = aios_grid.cohort_cells(lists)
|
| 860 |
# ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried
|
| 861 |
# NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id`
|
| 862 |
# as its join key — so the merge would have lost the one value every Odoo document joins on.
|
|
@@ -876,9 +944,11 @@ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_ke
|
|
| 876 |
# grains already do under an empty offer. Deleting the column instead would be a second,
|
| 877 |
# louder behaviour for the same fact on one surface out of three, and it would destroy a
|
| 878 |
# definition the admin can restore with one tick.
|
|
|
|
| 879 |
for pid, cells in (measure_resolve.column_values(
|
| 880 |
-
|
| 881 |
-
on_error=_measure_err) if may_metrics
|
|
|
|
| 882 |
derived.setdefault(pid, {}).update(cells)
|
| 883 |
|
| 884 |
# Route stops and Zone membership are normalized system relations. They project into two
|
|
|
|
| 22 |
proof is single-process and would report green on exactly the thing that is still broken.
|
| 23 |
"""
|
| 24 |
import datetime as _dt
|
| 25 |
+
from copy import copy, deepcopy
|
| 26 |
from dataclasses import is_dataclass, replace
|
| 27 |
import hashlib
|
| 28 |
import json
|
|
|
|
| 70 |
return handle if isinstance(handle, store_pg.PgStore) else None
|
| 71 |
|
| 72 |
|
| 73 |
+
def _assembly_roster_stamp(tenant):
|
| 74 |
+
"""Only permission and visible identity changes invalidate workspace metadata."""
|
| 75 |
+
from core import users
|
| 76 |
+
from routes_admin import _tenant_of
|
| 77 |
+
roster = users.registry()
|
| 78 |
+
if not isinstance(roster, dict) or not roster:
|
| 79 |
+
# A failed roster read must never validate a previously permitted answer.
|
| 80 |
+
return time.time_ns()
|
| 81 |
+
keys = ("tenant", "name", "email", "avatar", "active", "role", "bus", "modules",
|
| 82 |
+
"agent", "perms", "perms_v", "platform_admin")
|
| 83 |
+
visible = {str(name): {key: record.get(key) for key in keys}
|
| 84 |
+
for name, record in roster.items()
|
| 85 |
+
if isinstance(record, dict) and _tenant_of(record) == tenant}
|
| 86 |
+
return hashlib.sha256(json.dumps(visible, sort_keys=True, separators=(",", ":"),
|
| 87 |
+
default=str).encode("utf-8")).hexdigest()
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _assembly_revisions(runtime, names, uses_roster=False, roster_tenant=None, roster_memo=None):
|
| 91 |
"""Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
|
| 92 |
from core import store, store_pg
|
| 93 |
names = tuple(sorted(set(names)))
|
| 94 |
handle = _assembly_pg_store(runtime)
|
| 95 |
users_stamp = None
|
| 96 |
+
|
| 97 |
+
def roster_stamp(revision):
|
| 98 |
+
token = json.dumps(revision, sort_keys=True, default=str)
|
| 99 |
+
if roster_memo is None:
|
| 100 |
+
return _assembly_roster_stamp(roster_tenant)
|
| 101 |
+
if token not in roster_memo:
|
| 102 |
+
roster_memo[token] = _assembly_roster_stamp(roster_tenant)
|
| 103 |
+
return roster_memo[token]
|
| 104 |
+
|
| 105 |
if handle is None:
|
| 106 |
tokens = {name: _assembly_revision(runtime, name) for name in names}
|
| 107 |
if uses_roster:
|
| 108 |
+
users_stamp = roster_stamp(_assembly_revision(store, "users"))
|
| 109 |
return tokens, users_stamp
|
| 110 |
|
| 111 |
key = getattr(runtime, "store_key", str)
|
|
|
|
| 137 |
absent = {"rev": 0, "updated_at": None, "token": "pg:0"}
|
| 138 |
tokens = {name: dict(found.get(mapped, absent)) for name, mapped in keyed.items()}
|
| 139 |
if uses_roster:
|
| 140 |
+
users_stamp = roster_stamp(dict(found.get(roster_key, absent)) if same_roster
|
| 141 |
+
else _assembly_revision(store, "users"))
|
| 142 |
if isinstance(runtime, _AssemblyStore):
|
| 143 |
# The page may read these documents after its memo hit. Reuse this request's pre-read
|
| 144 |
# tokens without retaining any document bytes across requests.
|
|
|
|
| 295 |
"""
|
| 296 |
from core import field_permissions, shared_overlay
|
| 297 |
try:
|
| 298 |
+
document = shared_overlay.document(_shared_key(), st=st) if getattr(st, "schema_only", False) else None
|
| 299 |
field_permissions.migrate_legacy_fields(
|
| 300 |
+
_shared_key(), st=st, grant_topic="customer_data", shared_key=_shared_key(),
|
| 301 |
+
shared_document=document)
|
| 302 |
+
fields = shared_overlay.fields(_shared_key(), st=st, document=document)
|
| 303 |
# Normalized Routes use an invisible field-shaped bridge so the established Share dialog
|
| 304 |
# and grant registry can be reused. It is object metadata, never a customer column.
|
| 305 |
return {key: value for key, value in fields.items()
|
| 306 |
if not _is_route_share_bridge(value)}
|
| 307 |
+
except shared_overlay.SchemaUnavailable:
|
| 308 |
+
raise
|
| 309 |
except Exception: # noqa: BLE001
|
| 310 |
# Lenient like every other display read: an unreachable store degrades to "nothing is
|
| 311 |
# shared yet", never to a 500 on a grid that would otherwise render. The WALL does not
|
|
|
|
| 415 |
|
| 416 |
class _AssemblyStore:
|
| 417 |
"""One build's borrowed reads and their pre-read revisions, never a second cache."""
|
| 418 |
+
def __init__(self, runtime, *, schema_only=False):
|
| 419 |
self._runtime = runtime
|
| 420 |
+
self.schema_only = bool(schema_only)
|
| 421 |
self.docs, self.tokens = {}, {}
|
| 422 |
self._validated_tokens = {}
|
| 423 |
# A lent/test reader without its own coherent revision door can share a flight only.
|
|
|
|
| 439 |
except AttributeError:
|
| 440 |
self.cacheable = False
|
| 441 |
key = (name, shape)
|
| 442 |
+
if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs:
|
| 443 |
self.docs[key] = read()
|
| 444 |
# TableStore readers can mutate nested projections; lend an independent read each time.
|
| 445 |
return deepcopy(self.docs[key])
|
| 446 |
|
| 447 |
def get(self, name, fresh=False):
|
| 448 |
+
if self.schema_only and fresh:
|
| 449 |
+
self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
|
| 450 |
+
if self.schema_only and name.endswith("_table_workspace"):
|
| 451 |
+
return self._read(name, ("projection", ("overlays",)),
|
| 452 |
+
lambda: self._runtime.get_projection(name, drop=("overlays",)), fresh=fresh)
|
| 453 |
+
if self.schema_only and name.endswith("__shared"):
|
| 454 |
+
from core import shared_overlay
|
| 455 |
+
return shared_overlay.document(name[:-len("__shared")], st=self)
|
| 456 |
return self._read(name, "whole", lambda: (
|
| 457 |
self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)), fresh=fresh)
|
| 458 |
|
|
|
|
| 461 |
lambda: self._runtime.get_projection(name, drop=drop))
|
| 462 |
|
| 463 |
def get_entry(self, name, entry):
|
| 464 |
+
if self.schema_only:
|
| 465 |
+
backend = self._runtime._store() if callable(getattr(self._runtime, "_store", None)) else self._runtime
|
| 466 |
+
if not callable(getattr(backend, "get_entry", None)):
|
| 467 |
+
from core.shared_overlay import SchemaUnavailable
|
| 468 |
+
raise SchemaUnavailable("field definitions cannot be read without loading row cells")
|
| 469 |
return self._read(name, ("entry", str(entry)),
|
| 470 |
lambda: self._runtime.get_entry(name, entry))
|
| 471 |
|
| 472 |
def _read_strict(self, name):
|
| 473 |
+
if self.schema_only and name.endswith("_table_workspace"):
|
| 474 |
+
return self.get_projection(name, drop=("overlays",))
|
| 475 |
reader = getattr(self._runtime, "_read_strict", self._runtime.get)
|
| 476 |
return self._read(name, "strict", lambda: reader(name))
|
| 477 |
|
|
|
|
| 484 |
return token
|
| 485 |
|
| 486 |
def update(self, name, *args, **kwargs):
|
| 487 |
+
if self.schema_only:
|
| 488 |
+
from core.grid_events import StoreUnavailable
|
| 489 |
+
raise StoreUnavailable("field definitions need a full workspace refresh before they can be read")
|
| 490 |
self.cacheable = False
|
| 491 |
self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
|
| 492 |
return self._runtime.update(name, *args, **kwargs)
|
| 493 |
|
| 494 |
def put(self, name, *args, **kwargs):
|
| 495 |
+
if self.schema_only:
|
| 496 |
+
from core.grid_events import StoreUnavailable
|
| 497 |
+
raise StoreUnavailable("field definitions need a full workspace refresh before they can be read")
|
| 498 |
self.cacheable = False
|
| 499 |
self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
|
| 500 |
return self._runtime.put(name, *args, **kwargs)
|
|
|
|
| 513 |
|
| 514 |
|
| 515 |
def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 516 |
+
consume_corrections: bool = False, *, build=None, pool_stamp=None,
|
| 517 |
+
with_rows: bool = True, needed=None):
|
| 518 |
"""C2: one wall- and revision-scoped memo for customer, product and stored envelopes."""
|
| 519 |
global grid_assembly_shared
|
| 520 |
rt = session.runtime
|
| 521 |
+
needed = None if needed is None else frozenset(str(key) for key in needed)
|
| 522 |
if build is None:
|
| 523 |
+
build = lambda borrowed: _compute_grid_assembly(
|
| 524 |
+
borrowed, scope, storage_key, False, with_rows=with_rows, needed=needed)
|
| 525 |
if pool_stamp is None:
|
| 526 |
team_id, agent = _team_agent(session)
|
| 527 |
+
pool_stamp = lambda: _pool_stamp(rt, team_id, agent) if with_rows else 0
|
| 528 |
table_key = scope if scope.startswith("ut_") else (
|
| 529 |
"product_data" if scope == "product" else MODULE)
|
| 530 |
+
# Pre/post-build validation shares one roster read only while its revision is unchanged.
|
| 531 |
+
roster_memo = {}
|
| 532 |
|
| 533 |
def base_key(revisions=()):
|
| 534 |
# _field_usage reads users.registry() from this global account store, including on pg.
|
| 535 |
+
uses_roster = scope != "product" and not getattr(session, "user", {}).get("qa_runner")
|
| 536 |
prefix = ("workspace_assembly", getattr(session, "tenant", None), scope,
|
| 537 |
getattr(session, "uname", None), getattr(session, "admin", False),
|
| 538 |
json.dumps(getattr(session, "user", {}), sort_keys=True,
|
| 539 |
separators=(",", ":"), default=str),
|
| 540 |
+
_wall_stamp(session, table_key) if memoable else None, pool_stamp(),
|
| 541 |
+
("full" if needed is None else ("columns", tuple(sorted(needed))))
|
| 542 |
+
if with_rows else "fields")
|
| 543 |
if not memoable:
|
| 544 |
result = ((*prefix, json.dumps(None)), {})
|
| 545 |
return result
|
|
|
|
| 548 |
for memo_key in list(rt.pool_cache):
|
| 549 |
if memo_key[:-2] == prefix:
|
| 550 |
names.update(name for name, _token in memo_key[-1])
|
| 551 |
+
tokens, users_stamp = _assembly_revisions(rt, names, uses_roster,
|
| 552 |
+
getattr(session, "tenant", None), roster_memo)
|
| 553 |
result = ((*prefix, json.dumps(users_stamp, sort_keys=True, default=str)), tokens)
|
| 554 |
return result
|
| 555 |
|
|
|
|
| 565 |
if not memoable:
|
| 566 |
# Partial sessions and revision-less readers keep their original build semantics,
|
| 567 |
# but concurrent callers can still borrow this flight's result.
|
| 568 |
+
borrowed = copy(session)
|
| 569 |
+
if not with_rows:
|
| 570 |
+
borrowed.runtime = _AssemblyStore(rt, schema_only=True)
|
| 571 |
+
out = (build(borrowed), (), False)
|
| 572 |
return out
|
| 573 |
lent = assembly_rt
|
| 574 |
result = build(replace(session, runtime=lent))
|
|
|
|
| 583 |
|
| 584 |
# Without a pool cache there is no runtime-scoped flight identity.
|
| 585 |
if not hasattr(rt, "pool_cache"):
|
| 586 |
+
borrowed = copy(session)
|
| 587 |
+
if not with_rows:
|
| 588 |
+
borrowed.runtime = _AssemblyStore(rt, schema_only=True)
|
| 589 |
+
out = own_copy(build(borrowed))
|
| 590 |
return out
|
| 591 |
memoable = (is_dataclass(session)
|
| 592 |
and all(hasattr(session, name) for name in ("tenant", "uname", "admin", "user")))
|
| 593 |
+
assembly_rt = (rt if isinstance(rt, _AssemblyStore) and rt.schema_only == (not with_rows)
|
| 594 |
+
else _AssemblyStore(rt, schema_only=not with_rows)) if memoable else None
|
| 595 |
memoable = memoable and assembly_rt.cacheable
|
| 596 |
key, checked_tokens = base_key()
|
| 597 |
|
|
|
|
| 687 |
|
| 688 |
|
| 689 |
def _compute_grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
|
| 690 |
+
consume_corrections: bool = True, *, with_rows: bool = True, needed=None):
|
| 691 |
"""ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
|
| 692 |
events route (2026-07-31 — the standalone measure gap, owner item 1).
|
| 693 |
|
|
|
|
| 720 |
|
| 721 |
rt = session.runtime
|
| 722 |
team_id, agent = _team_agent(session)
|
| 723 |
+
rows_src = _pool_for(rt, team_id, agent) if with_rows else []
|
| 724 |
# ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that
|
| 725 |
# frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so
|
| 726 |
# scoping here means a row this account may not see never enters ANY of them, rather than
|
|
|
|
| 743 |
# ⛔ THE SAME HANDLE GOES TO `allowed_pids` BELOW, AND THE PAIR IS NOT OPTIONAL. That is the
|
| 744 |
# WRITE wall to this one's READ wall; lending it here alone would make a user-generated rule
|
| 745 |
# narrow what an account SEES while leaving what it may PATCH untouched.
|
| 746 |
+
if with_rows:
|
| 747 |
+
rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS,
|
| 748 |
+
st=rt)
|
| 749 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
| 750 |
# ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because
|
| 751 |
# `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this
|
|
|
|
| 769 |
# number, so a per-user leftover under the same key is stale by construction. It is also what
|
| 770 |
# makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not
|
| 771 |
# promoted.
|
| 772 |
+
_shared = shared_cells(pids, st=rt) if with_rows else {}
|
| 773 |
if _shared:
|
| 774 |
_ov = dict(ws.get("overlays") or {})
|
| 775 |
for _pid, _cells in _shared.items():
|
|
|
|
| 907 |
|
| 908 |
today = time.strftime("%Y-%m-%d")
|
| 909 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
| 910 |
+
stamp = _pool_stamp(rt, team_id, agent) if with_rows else 0
|
| 911 |
# ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE
|
| 912 |
# OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their
|
| 913 |
# condition answers through helpers that short-circuit on an empty `offer`, so emptying the
|
|
|
|
| 921 |
measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else []
|
| 922 |
measure_sets = measure_resolve.condition_sets(
|
| 923 |
[v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp,
|
| 924 |
+
rt.mset_memo, on_error=_measure_err) if may_metrics and with_rows and needed is None else {}
|
| 925 |
# The derived channel: cohort membership cells + measure column values, ONE dict — the
|
| 926 |
# same read-only channel the embed host hands to rows_from_pool.
|
| 927 |
+
derived = aios_grid.cohort_cells(lists) if with_rows else {}
|
| 928 |
# ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried
|
| 929 |
# NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id`
|
| 930 |
# as its join key — so the merge would have lost the one value every Odoo document joins on.
|
|
|
|
| 944 |
# grains already do under an empty offer. Deleting the column instead would be a second,
|
| 945 |
# louder behaviour for the same fact on one surface out of three, and it would destroy a
|
| 946 |
# definition the admin can restore with one tick.
|
| 947 |
+
materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed]
|
| 948 |
for pid, cells in (measure_resolve.column_values(
|
| 949 |
+
materialized_fields, team_id, pids, today, stamp, rt.measure_memo,
|
| 950 |
+
on_error=_measure_err) if may_metrics and with_rows and
|
| 951 |
+
(needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items():
|
| 952 |
derived.setdefault(pid, {}).update(cells)
|
| 953 |
|
| 954 |
# Route stops and Zone membership are normalized system relations. They project into two
|
api/routes_external.py
CHANGED
|
@@ -8,6 +8,8 @@ import hmac
|
|
| 8 |
import importlib
|
| 9 |
import inspect
|
| 10 |
import json
|
|
|
|
|
|
|
| 11 |
import time
|
| 12 |
import uuid
|
| 13 |
from urllib.parse import parse_qs, quote, unquote, urlencode, urlsplit, urlunsplit
|
|
@@ -1370,28 +1372,43 @@ def _maybe_session(request: Request, response: Response):
|
|
| 1370 |
|
| 1371 |
|
| 1372 |
_PAGE_STYLE = """
|
| 1373 |
-
body{margin:0;background:
|
| 1374 |
-
|
| 1375 |
-
|
| 1376 |
-
|
| 1377 |
-
|
| 1378 |
-
|
| 1379 |
-
|
| 1380 |
-
|
| 1381 |
-
|
| 1382 |
-
|
| 1383 |
-
|
| 1384 |
-
.
|
|
|
|
|
|
|
|
|
|
| 1385 |
"""
|
| 1386 |
|
| 1387 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1388 |
def _page(title: str, body: str, status: int = 200):
|
| 1389 |
from fastapi.responses import HTMLResponse
|
| 1390 |
return HTMLResponse(
|
| 1391 |
f"<!doctype html><html lang=en><head><meta charset=utf-8>"
|
| 1392 |
f"<meta name=viewport content='width=device-width,initial-scale=1'>"
|
| 1393 |
-
f"<title>{_html.escape(title)}</title><style>{_PAGE_STYLE}</style></head>"
|
| 1394 |
-
f"<body><main class=card>{body}</main></body></html>", status_code=status,
|
| 1395 |
headers={"Cache-Control": "no-store", "X-Frame-Options": "DENY",
|
| 1396 |
"Content-Security-Policy": "frame-ancestors 'none'"})
|
| 1397 |
|
|
@@ -1447,35 +1464,18 @@ def _granted_scopes(requested, access: str) -> str:
|
|
| 1447 |
def _consent_page(request: Request, session: Session, meta: dict, fields: dict):
|
| 1448 |
name = _html.escape(meta["n"])
|
| 1449 |
host = _html.escape(urlsplit(fields["redirect_uri"]).netloc)
|
| 1450 |
-
try:
|
| 1451 |
-
rows = [row for row in _resolve_external_scope(session) if row.get("permitted")]
|
| 1452 |
-
except Exception:
|
| 1453 |
-
rows = []
|
| 1454 |
-
dbs = "".join(
|
| 1455 |
-
f"<label><input type=checkbox name=database value='{_html.escape(str(row['database']))}'>"
|
| 1456 |
-
f" {_html.escape(str(row.get('label') or row['database']))}</label>" for row in rows)
|
| 1457 |
token = _html.escape(_consent_token(session, fields))
|
| 1458 |
action = _html.escape(str(request.url.path))
|
| 1459 |
return _page(f"Connect {meta['n']} to Loopable", (
|
| 1460 |
f"<h1>Connect {name} to Loopable</h1>"
|
| 1461 |
-
f"<p>{name}
|
| 1462 |
-
f"<strong>{host}</strong>.</p>"
|
| 1463 |
f"<form method=post action='{action}'>"
|
| 1464 |
f"<input type=hidden name=consent value='{token}'>"
|
| 1465 |
-
"<
|
| 1466 |
-
"<label><input type=radio name=access value=read checked> Read only"
|
| 1467 |
-
"<small>It can read the databases you choose.</small></label>"
|
| 1468 |
-
"<label><input type=radio name=access value=readwrite> Read and write"
|
| 1469 |
-
"<small>It can also make the changes you could make yourself.</small></label>"
|
| 1470 |
-
"</fieldset><fieldset><legend>Databases</legend>"
|
| 1471 |
-
"<label><input type=radio name=scope_kind value=all checked> All databases you can see"
|
| 1472 |
-
"</label><label><input type=radio name=scope_kind value=some> Only these databases"
|
| 1473 |
-
f"</label><div class=dbs>{dbs or '<small>No databases are available to you.</small>'}"
|
| 1474 |
-
"</div></fieldset><div class=row>"
|
| 1475 |
"<button type=submit name=decision value=deny>Deny</button>"
|
| 1476 |
"<button type=submit name=decision value=allow class=primary>Allow</button></div></form>"
|
| 1477 |
-
f"<p class=who>Signed in as {_html.escape(session.uname)}.
|
| 1478 |
-
"time under AI API.</p>"))
|
| 1479 |
|
| 1480 |
|
| 1481 |
@router.get("/oauth/callback")
|
|
@@ -1557,16 +1557,9 @@ async def oauth_consent(request: Request, session: Session = Depends(require_ses
|
|
| 1557 |
if (form.get("decision") or [""])[-1] != "allow":
|
| 1558 |
return _oauth_redirect(fields["redirect_uri"], error="access_denied",
|
| 1559 |
state=fields["state"])
|
| 1560 |
-
|
| 1561 |
-
|
| 1562 |
-
|
| 1563 |
-
if (form.get("scope_kind") or ["all"])[-1] == "some":
|
| 1564 |
-
if not chosen:
|
| 1565 |
-
return _refusal_page("Choose at least one database, or allow all of them. Go back "
|
| 1566 |
-
"and try again.")
|
| 1567 |
-
databases = chosen
|
| 1568 |
-
else:
|
| 1569 |
-
databases = external_access.ALL_DATABASES
|
| 1570 |
reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
|
| 1571 |
if reason:
|
| 1572 |
return _refusal_page(reason, 403)
|
|
|
|
| 8 |
import importlib
|
| 9 |
import inspect
|
| 10 |
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import re
|
| 13 |
import time
|
| 14 |
import uuid
|
| 15 |
from urllib.parse import parse_qs, quote, unquote, urlencode, urlsplit, urlunsplit
|
|
|
|
| 1372 |
|
| 1373 |
|
| 1374 |
_PAGE_STYLE = """
|
| 1375 |
+
body.oauth-page{margin:0;background:var(--lp-surface-2);
|
| 1376 |
+
font-family:Inter,system-ui,-apple-system,Segoe UI,sans-serif;font-size:var(--lp-fs-sm);
|
| 1377 |
+
color:var(--lp-ink);display:flex;justify-content:center;padding:48px 16px}
|
| 1378 |
+
.oauth-card{background:var(--lp-surface);border:1px solid var(--lp-line);
|
| 1379 |
+
border-radius:var(--lp-r-lg);max-width:460px;width:100%;padding:24px}
|
| 1380 |
+
.oauth-card h1{font-size:var(--lp-fs-lg);font-weight:600;margin:0 0 8px}
|
| 1381 |
+
.oauth-card p{line-height:var(--lp-lh);margin:0 0 16px;color:var(--lp-muted)}
|
| 1382 |
+
.oauth-card .row{display:flex;gap:8px;justify-content:flex-end;margin-top:8px}
|
| 1383 |
+
.oauth-card button{border:0;background:transparent;font:inherit;color:inherit;
|
| 1384 |
+
border-radius:var(--lp-r-md);padding:8px 16px;cursor:pointer}
|
| 1385 |
+
.oauth-card button:hover{background:var(--lp-surface-2)}
|
| 1386 |
+
.oauth-card button.primary{background:var(--lp-primary);color:var(--lp-surface);font-weight:600}
|
| 1387 |
+
.oauth-card button.primary:hover{background:var(--lp-primary-hover)}
|
| 1388 |
+
.oauth-card button:focus-visible{outline:2px solid var(--lp-blue-deep);outline-offset:2px}
|
| 1389 |
+
.oauth-card .who{font-size:var(--lp-fs-xs);color:var(--lp-muted);margin:16px 0 0}
|
| 1390 |
"""
|
| 1391 |
|
| 1392 |
|
| 1393 |
+
def _page_stylesheets() -> str:
|
| 1394 |
+
"""Borrow the emitted app stylesheet, including its canonical tokens and Inter font."""
|
| 1395 |
+
index = Path(__file__).resolve().parents[1] / "web" / "dist" / "index.html"
|
| 1396 |
+
try:
|
| 1397 |
+
document = index.read_text(encoding="utf-8")
|
| 1398 |
+
except OSError:
|
| 1399 |
+
return ""
|
| 1400 |
+
paths = re.findall(r'<link\b[^>]*\bhref=[\"\'](/assets/[^\"\']+\.css)[\"\']', document)
|
| 1401 |
+
return "".join(f'<link rel="stylesheet" href="{_html.escape(path, quote=True)}">'
|
| 1402 |
+
for path in dict.fromkeys(paths))
|
| 1403 |
+
|
| 1404 |
+
|
| 1405 |
def _page(title: str, body: str, status: int = 200):
|
| 1406 |
from fastapi.responses import HTMLResponse
|
| 1407 |
return HTMLResponse(
|
| 1408 |
f"<!doctype html><html lang=en><head><meta charset=utf-8>"
|
| 1409 |
f"<meta name=viewport content='width=device-width,initial-scale=1'>"
|
| 1410 |
+
f"<title>{_html.escape(title)}</title>{_page_stylesheets()}<style>{_PAGE_STYLE}</style></head>"
|
| 1411 |
+
f"<body class=oauth-page><main class=oauth-card>{body}</main></body></html>", status_code=status,
|
| 1412 |
headers={"Cache-Control": "no-store", "X-Frame-Options": "DENY",
|
| 1413 |
"Content-Security-Policy": "frame-ancestors 'none'"})
|
| 1414 |
|
|
|
|
| 1464 |
def _consent_page(request: Request, session: Session, meta: dict, fields: dict):
|
| 1465 |
name = _html.escape(meta["n"])
|
| 1466 |
host = _html.escape(urlsplit(fields["redirect_uri"]).netloc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1467 |
token = _html.escape(_consent_token(session, fields))
|
| 1468 |
action = _html.escape(str(request.url.path))
|
| 1469 |
return _page(f"Connect {meta['n']} to Loopable", (
|
| 1470 |
f"<h1>Connect {name} to Loopable</h1>"
|
| 1471 |
+
f"<p>Allow {name} to connect?</p>"
|
| 1472 |
+
f"<p>Return to <strong>{host}</strong>.</p>"
|
| 1473 |
f"<form method=post action='{action}'>"
|
| 1474 |
f"<input type=hidden name=consent value='{token}'>"
|
| 1475 |
+
"<div class=row>"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1476 |
"<button type=submit name=decision value=deny>Deny</button>"
|
| 1477 |
"<button type=submit name=decision value=allow class=primary>Allow</button></div></form>"
|
| 1478 |
+
f"<p class=who>Signed in as {_html.escape(session.uname)}.</p>"))
|
|
|
|
| 1479 |
|
| 1480 |
|
| 1481 |
@router.get("/oauth/callback")
|
|
|
|
| 1557 |
if (form.get("decision") or [""])[-1] != "allow":
|
| 1558 |
return _oauth_redirect(fields["redirect_uri"], error="access_denied",
|
| 1559 |
state=fields["state"])
|
| 1560 |
+
# R15/C5: Allow approves the connection. Old/tampered picker fields cannot set a cap;
|
| 1561 |
+
# every bearer call resolves the account's current permission record instead.
|
| 1562 |
+
access, databases = external_access.ACCESS_READWRITE, external_access.ALL_DATABASES
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1563 |
reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
|
| 1564 |
if reason:
|
| 1565 |
return _refusal_page(reason, 403)
|
api/routes_forms.py
CHANGED
|
@@ -187,6 +187,9 @@ def _tokens(rt) -> dict:
|
|
| 187 |
def _resolve(token: str):
|
| 188 |
"""`(runtime, tenant_slug, table_key, form_spec)` for a token, or None.
|
| 189 |
|
|
|
|
|
|
|
|
|
|
| 190 |
⚠ THE LOOP DOES NOT SHORT-CIRCUIT ON THE TENANT, only on the match. A `break` on the first
|
| 191 |
tenant whose store answered would make "the third tenant's form" depend on the first two
|
| 192 |
being healthy, and a form that stops working because somebody else's store blipped is the
|
|
@@ -207,9 +210,24 @@ def _resolve(token: str):
|
|
| 207 |
rt = _rt.get_runtime(slug)
|
| 208 |
except Exception: # noqa: BLE001
|
| 209 |
continue
|
| 210 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
if not _same(str(stored), token) or not isinstance(where, dict):
|
| 212 |
continue
|
|
|
|
|
|
|
| 213 |
table_key = str(where.get("table") or "")
|
| 214 |
view, _owner = _find_view(rt, table_key, str(where.get("view") or ""))
|
| 215 |
spec = _spec_of(view)
|
|
@@ -403,6 +421,162 @@ def _link_of(token: str) -> str:
|
|
| 403 |
return f"{_public_base()}/#/form/{token}"
|
| 404 |
|
| 405 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 406 |
def _may_administer_view(session: Session, table_key: str, view_id: str):
|
| 407 |
"""`(view, spec)` when this caller may publish THIS view, else raises.
|
| 408 |
|
|
@@ -513,6 +687,8 @@ def get_form(token: str, request: Request):
|
|
| 513 |
if not found:
|
| 514 |
raise _refuse()
|
| 515 |
rt, _slug, table_key, spec = found
|
|
|
|
|
|
|
| 516 |
return _public_form(rt, table_key, spec)
|
| 517 |
|
| 518 |
|
|
@@ -562,6 +738,8 @@ async def submit_form(token: str, request: Request):
|
|
| 562 |
if not found:
|
| 563 |
raise _refuse()
|
| 564 |
rt, _slug, table_key, spec = found
|
|
|
|
|
|
|
| 565 |
|
| 566 |
values = (body or {}).get("values")
|
| 567 |
if not isinstance(values, dict):
|
|
|
|
| 187 |
def _resolve(token: str):
|
| 188 |
"""`(runtime, tenant_slug, table_key, form_spec)` for a token, or None.
|
| 189 |
|
| 190 |
+
A hashed setup capability returns `table_key=None` and server-owned setup metadata;
|
| 191 |
+
ordinary forms retain their persistent view pointer and row-append path.
|
| 192 |
+
|
| 193 |
⚠ THE LOOP DOES NOT SHORT-CIRCUIT ON THE TENANT, only on the match. A `break` on the first
|
| 194 |
tenant whose store answered would make "the third tenant's form" depend on the first two
|
| 195 |
being healthy, and a form that stops working because somebody else's store blipped is the
|
|
|
|
| 210 |
rt = _rt.get_runtime(slug)
|
| 211 |
except Exception: # noqa: BLE001
|
| 212 |
continue
|
| 213 |
+
index = _tokens(rt)
|
| 214 |
+
from core import setup_links
|
| 215 |
+
hashed = setup_links.digest(token)
|
| 216 |
+
if hashed and hashed in index:
|
| 217 |
+
try:
|
| 218 |
+
_setup_store(rt, slug)
|
| 219 |
+
record = setup_links.read_record(index, token, slug)
|
| 220 |
+
if record is None:
|
| 221 |
+
return None
|
| 222 |
+
_setup_owner(rt, slug, record["owner"], record["scope"])
|
| 223 |
+
return rt, slug, None, record
|
| 224 |
+
except Exception:
|
| 225 |
+
return None
|
| 226 |
+
for stored, where in index.items():
|
| 227 |
if not _same(str(stored), token) or not isinstance(where, dict):
|
| 228 |
continue
|
| 229 |
+
if where.get("kind") == setup_links.KIND:
|
| 230 |
+
return None
|
| 231 |
table_key = str(where.get("table") or "")
|
| 232 |
view, _owner = _find_view(rt, table_key, str(where.get("view") or ""))
|
| 233 |
spec = _spec_of(view)
|
|
|
|
| 421 |
return f"{_public_base()}/#/form/{token}"
|
| 422 |
|
| 423 |
|
| 424 |
+
def _setup_store(rt, tenant):
|
| 425 |
+
if not tenant or getattr(rt, "key", None) != tenant or not rt.available():
|
| 426 |
+
raise err(503, "setup_unavailable", "Unavailable")
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def _setup_owner(rt, tenant, owner, scope):
|
| 430 |
+
"""Re-read the account registry; a minted capability does not freeze permissions."""
|
| 431 |
+
from core import perms, users
|
| 432 |
+
|
| 433 |
+
_setup_store(rt, tenant)
|
| 434 |
+
try:
|
| 435 |
+
registry = users.registry()
|
| 436 |
+
record = registry.get(owner) if isinstance(registry, dict) else None
|
| 437 |
+
if (not isinstance(record, dict) or not record.get("active", True)
|
| 438 |
+
or users._public(owner, record).get("tenant") != tenant):
|
| 439 |
+
raise _refuse()
|
| 440 |
+
if scope == "business" and not perms.is_admin(record):
|
| 441 |
+
raise err(403, "setup_admin_required", "Ask an admin")
|
| 442 |
+
return Session(tenant=tenant, user=users._public(owner, record), claims={}, runtime=rt)
|
| 443 |
+
except Exception as exc:
|
| 444 |
+
# Preserve our refusals, but never return a registry exception or its content.
|
| 445 |
+
if getattr(exc, "status_code", None) is not None:
|
| 446 |
+
raise
|
| 447 |
+
raise _refuse() from None
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _setup_fields(connector):
|
| 451 |
+
from core import keychain
|
| 452 |
+
from routes_connectors import _CONNECT_FIELDS, _CONNECT_TOKEN_FIELD
|
| 453 |
+
|
| 454 |
+
if connector not in keychain.ENTRY_TYPES:
|
| 455 |
+
raise _refuse()
|
| 456 |
+
return [dict(field) for field in (_CONNECT_FIELDS.get(connector) or _CONNECT_TOKEN_FIELD)]
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
def _public_setup(record):
|
| 460 |
+
from core import keychain, setup_links
|
| 461 |
+
|
| 462 |
+
fields = []
|
| 463 |
+
for field in _setup_fields(record["connector"]):
|
| 464 |
+
fields.append({"key": field["key"], "label": field["label"],
|
| 465 |
+
"type": ("password" if field.get("secret") else
|
| 466 |
+
"url" if field["key"] == "url" else "string"),
|
| 467 |
+
"secret": bool(field.get("secret")),
|
| 468 |
+
"required": bool(field.get("required", True)),
|
| 469 |
+
**({"hint": field["hint"]} if field.get("hint") else {})})
|
| 470 |
+
return {"kind": setup_links.KIND, "title": keychain.kind_label(record["connector"]),
|
| 471 |
+
"desc": "", "submitLabel": "Connect", "fields": fields,
|
| 472 |
+
"honeypot": HONEYPOT_FIELD, "singleUse": True}
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
@router.post("/connector-setup-link")
|
| 476 |
+
def mint_setup_link(body: dict, session: Session = Depends(require_session)):
|
| 477 |
+
"""Mint a credential-free setup capability for the authenticated asking user only."""
|
| 478 |
+
from core import keychain, setup_links
|
| 479 |
+
|
| 480 |
+
if not isinstance(body, dict) or set(body) - {"connector", "scope"}:
|
| 481 |
+
raise err(400, "bad_setup_request", "Invalid request")
|
| 482 |
+
connector = body.get("connector")
|
| 483 |
+
if not isinstance(connector, str) or connector not in keychain.ENTRY_TYPES:
|
| 484 |
+
raise err(400, "bad_connector", "Unavailable")
|
| 485 |
+
scope = body.get("scope", "business" if connector in keychain.TENANT_WIDE_TYPES else "personal")
|
| 486 |
+
if scope not in ("personal", "business"):
|
| 487 |
+
raise err(400, "bad_scope", "Invalid scope")
|
| 488 |
+
current = _setup_owner(session.runtime, session.tenant, session.uname, scope)
|
| 489 |
+
if scope == "personal" and connector in keychain.TENANT_WIDE_TYPES:
|
| 490 |
+
raise err(403, "setup_admin_required", "Ask an admin")
|
| 491 |
+
if not _setup_fields(connector):
|
| 492 |
+
raise err(400, "bad_connector", "Unavailable")
|
| 493 |
+
try:
|
| 494 |
+
return setup_links.mint(session.runtime, tokens_key=TOKENS_KEY, tenant=current.tenant,
|
| 495 |
+
owner=current.uname, connector=connector, scope=scope,
|
| 496 |
+
origin=_public_base())
|
| 497 |
+
except setup_links.InvalidSetupLink:
|
| 498 |
+
raise err(503, "setup_unavailable", "Unavailable") from None
|
| 499 |
+
except Exception:
|
| 500 |
+
raise err(503, "setup_unavailable", "Unavailable") from None
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
def _starter_selection(rt, provider):
|
| 504 |
+
"""S70-T10 through the S70-T11 door: a first connection syncs the starter set, never everything.
|
| 505 |
+
|
| 506 |
+
The link form has no database step, so it takes the wizard's preselected rows. An existing
|
| 507 |
+
connection or an earlier choice document is left alone (Settings owns a change)."""
|
| 508 |
+
from connector_onboarding import choices, install_selection
|
| 509 |
+
from core import keychain
|
| 510 |
+
import routes_keychain as keys
|
| 511 |
+
|
| 512 |
+
starter = [row["key"] for row in choices(provider) if row.get("selected")]
|
| 513 |
+
if not starter:
|
| 514 |
+
return False
|
| 515 |
+
if any(entry.get("type") == provider for entry in keychain.list_entries(rt)) \
|
| 516 |
+
or (provider == "odoo" and keys.env_odoo_available(rt)):
|
| 517 |
+
return False
|
| 518 |
+
try:
|
| 519 |
+
install_selection(rt, provider, starter, existing=False)
|
| 520 |
+
except ValueError:
|
| 521 |
+
return False
|
| 522 |
+
return True
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
def _write_setup_credentials(rt, record, fields):
|
| 526 |
+
"""Keep API-owned scoping and reconnect effects out of the lower core module."""
|
| 527 |
+
from core import keychain
|
| 528 |
+
import routes_keychain as keys
|
| 529 |
+
|
| 530 |
+
current = _setup_owner(rt, record["tenant"], record["owner"], record["scope"])
|
| 531 |
+
# Before the credential, as the wizard does, so no runner sees a key with all-on defaults.
|
| 532 |
+
selected = _starter_selection(rt, record["connector"])
|
| 533 |
+
keychain.add_entry(rt, None, record["connector"], fields, current.uname,
|
| 534 |
+
scope=record["scope"], write_scope=keys._write_scope)
|
| 535 |
+
if selected:
|
| 536 |
+
from connector_onboarding import seal_selection
|
| 537 |
+
seal_selection(rt, record["connector"])
|
| 538 |
+
if record["connector"] == "odoo":
|
| 539 |
+
try:
|
| 540 |
+
keys._rel_reconnect(rt)
|
| 541 |
+
except Exception:
|
| 542 |
+
pass
|
| 543 |
+
elif record["connector"] == "amazon_sp":
|
| 544 |
+
try:
|
| 545 |
+
from harness import amazon_store
|
| 546 |
+
amazon_store.cancel_deletion_trigger(rt.key, rt=rt, principal=current.uname)
|
| 547 |
+
except Exception:
|
| 548 |
+
pass
|
| 549 |
+
try:
|
| 550 |
+
from core import connector_sync
|
| 551 |
+
connector_sync.request_sync("amazon_sp", rt.key, requested_by=current.uname)
|
| 552 |
+
except Exception:
|
| 553 |
+
pass
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
def _submit_setup(rt, token, record, body):
|
| 557 |
+
from core import setup_links
|
| 558 |
+
|
| 559 |
+
_setup_owner(rt, record["tenant"], record["owner"], record["scope"])
|
| 560 |
+
if set(body) - {"values"} or not isinstance(body.get("values"), dict):
|
| 561 |
+
raise err(400, "invalid_submission", "Invalid fields")
|
| 562 |
+
values = dict(body["values"])
|
| 563 |
+
if values.pop(HONEYPOT_FIELD, ""):
|
| 564 |
+
raise err(400, "invalid_submission", "Invalid fields")
|
| 565 |
+
try:
|
| 566 |
+
return setup_links.submit(rt, token, record, values, tokens_key=TOKENS_KEY,
|
| 567 |
+
field_specs=_setup_fields(record["connector"]),
|
| 568 |
+
write_credentials=lambda meta, clean:
|
| 569 |
+
_write_setup_credentials(rt, meta, clean))
|
| 570 |
+
except setup_links.InvalidSetupLink:
|
| 571 |
+
raise _refuse() from None
|
| 572 |
+
except setup_links.InvalidSetupValues:
|
| 573 |
+
raise err(400, "invalid_submission", "Invalid fields") from None
|
| 574 |
+
except setup_links.SetupWriteFailed:
|
| 575 |
+
raise err(503, "setup_write_failed", "Save failed") from None
|
| 576 |
+
finally:
|
| 577 |
+
values.clear()
|
| 578 |
+
|
| 579 |
+
|
| 580 |
def _may_administer_view(session: Session, table_key: str, view_id: str):
|
| 581 |
"""`(view, spec)` when this caller may publish THIS view, else raises.
|
| 582 |
|
|
|
|
| 687 |
if not found:
|
| 688 |
raise _refuse()
|
| 689 |
rt, _slug, table_key, spec = found
|
| 690 |
+
if table_key is None:
|
| 691 |
+
return _public_setup(spec)
|
| 692 |
return _public_form(rt, table_key, spec)
|
| 693 |
|
| 694 |
|
|
|
|
| 738 |
if not found:
|
| 739 |
raise _refuse()
|
| 740 |
rt, _slug, table_key, spec = found
|
| 741 |
+
if table_key is None:
|
| 742 |
+
return _submit_setup(rt, token, spec, body)
|
| 743 |
|
| 744 |
values = (body or {}).get("values")
|
| 745 |
if not isinstance(values, dict):
|
api/routes_grid.py
CHANGED
|
@@ -319,9 +319,34 @@ def _map_setup(scope, fields, runtime):
|
|
| 319 |
return database_setup.map_for(_database_key_for_scope(scope), fields, st=runtime)
|
| 320 |
|
| 321 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 322 |
@router.get("/workspace")
|
| 323 |
def workspace(scope: str = "customer",
|
| 324 |
-
session: Session = Depends(require_session)):
|
| 325 |
"""The durable table workspace for this session: views, fields, overlays, folders.
|
| 326 |
|
| 327 |
⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope
|
|
@@ -343,7 +368,7 @@ def workspace(scope: str = "customer",
|
|
| 343 |
wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a
|
| 344 |
full-access user.
|
| 345 |
"""
|
| 346 |
-
from core import grid_events
|
| 347 |
from routes_customers import grid_assembly
|
| 348 |
|
| 349 |
scope = _scope_or_400(scope)
|
|
@@ -366,13 +391,18 @@ def workspace(scope: str = "customer",
|
|
| 366 |
# bought nothing and cost everything — it is why `?scope=ut_odoo_gl_lines` answered
|
| 367 |
# `409 window_required` six times out of six on the live deploy while the ROWS door for
|
| 368 |
# the same table served a page in ~150 ms, i.e. a grid that could not be opened at all.
|
| 369 |
-
g = ut_assembly(session, scope, storage_key=storage_key, with_rows=False
|
| 370 |
-
|
| 371 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 372 |
except grid_events.StoreUnavailable:
|
| 373 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 374 |
workspace = g["workspace"]
|
| 375 |
-
workspace["overlays"] =
|
|
|
|
| 376 |
# The field contract rides the cheap re-read on EVERY topic — see the customer branch
|
| 377 |
# below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)`
|
| 378 |
# is scope-parameterized, so `ut_` surfaces have their own sets).
|
|
@@ -385,7 +415,7 @@ def workspace(scope: str = "customer",
|
|
| 385 |
# them away, so the Locked-views column arrived on the cheap re-read with permanently
|
| 386 |
# empty values. A column that exists and can never have one is worse than an absent
|
| 387 |
# column: it reads as "this record is in no locked view", which is a claim.
|
| 388 |
-
workspace["derived"] =
|
| 389 |
workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
|
| 390 |
try:
|
| 391 |
from core import users as _users
|
|
@@ -399,7 +429,8 @@ def workspace(scope: str = "customer",
|
|
| 399 |
workspace["permissionUserOptions"] = []
|
| 400 |
workspace["userAvatars"] = {}
|
| 401 |
workspace["scopeKey"] = scope
|
| 402 |
-
workspace["auditTimestamps"] =
|
|
|
|
| 403 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 404 |
if map_setup:
|
| 405 |
workspace["mapSetup"] = map_setup
|
|
@@ -409,6 +440,8 @@ def workspace(scope: str = "customer",
|
|
| 409 |
# `missing` on a shortened cohort). An unannounced empty scope is the silent limit R6
|
| 410 |
# forbids; this is the announcement, and W31-T22 is the ticket that renders it.
|
| 411 |
workspace["limits"] = g.get("limits") or []
|
|
|
|
|
|
|
| 412 |
return {"workspace": workspace}
|
| 413 |
|
| 414 |
# ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field
|
|
@@ -424,13 +457,18 @@ def workspace(scope: str = "customer",
|
|
| 424 |
session.require(PRODUCT_MODULE)
|
| 425 |
storage_key = f"{session.tenant}:product-list:{session.uname}:all:all"
|
| 426 |
try:
|
| 427 |
-
g = product_assembly(session, scope=scope, storage_key=storage_key
|
| 428 |
-
|
| 429 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 430 |
except grid_events.StoreUnavailable:
|
| 431 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 432 |
workspace = g["workspace"]
|
| 433 |
-
workspace["overlays"] =
|
|
|
|
| 434 |
# R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own
|
| 435 |
# first-cohort contract change. Same reason as the customer branch below.
|
| 436 |
workspace["fields"] = g["fields"]
|
|
@@ -442,7 +480,7 @@ def workspace(scope: str = "customer",
|
|
| 442 |
# reason: `product_assembly` builds this topic's cohort cells and this route discarded
|
| 443 |
# them. The measure half stays empty on this topic by descope, which is a different
|
| 444 |
# statement and one the offer already makes.
|
| 445 |
-
workspace["derived"] =
|
| 446 |
workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
|
| 447 |
try:
|
| 448 |
from core import users as _users
|
|
@@ -456,10 +494,13 @@ def workspace(scope: str = "customer",
|
|
| 456 |
workspace["permissionUserOptions"] = []
|
| 457 |
workspace["userAvatars"] = {}
|
| 458 |
workspace["scopeKey"] = scope
|
| 459 |
-
workspace["auditTimestamps"] =
|
|
|
|
| 460 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 461 |
if map_setup:
|
| 462 |
workspace["mapSetup"] = map_setup
|
|
|
|
|
|
|
| 463 |
return {"workspace": workspace}
|
| 464 |
|
| 465 |
# The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid`
|
|
@@ -483,9 +524,12 @@ def workspace(scope: str = "customer",
|
|
| 483 |
# never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer
|
| 484 |
# surface. One projection for both servers is the fix that cannot drift.
|
| 485 |
try:
|
| 486 |
-
g = grid_assembly(session, scope=scope, storage_key=storage_key)
|
| 487 |
-
|
| 488 |
-
|
|
|
|
|
|
|
|
|
|
| 489 |
except grid_events.StoreUnavailable:
|
| 490 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 491 |
workspace = g["workspace"]
|
|
@@ -497,7 +541,8 @@ def workspace(scope: str = "customer",
|
|
| 497 |
# /customers rows, but this route is the cheap read-back a write can be verified against
|
| 498 |
# (verify_api's overlay probe) without paying the pool call. Per-user by construction —
|
| 499 |
# `table_workspace` is this session's workspace.
|
| 500 |
-
workspace["overlays"] =
|
|
|
|
| 501 |
# ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration.
|
| 502 |
#
|
| 503 |
# `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked
|
|
@@ -522,7 +567,7 @@ def workspace(scope: str = "customer",
|
|
| 522 |
workspace["measureSets"] = g["measure_sets"]
|
| 523 |
# Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are
|
| 524 |
# strings; the client indexes with String(pid).
|
| 525 |
-
workspace["derived"] =
|
| 526 |
# Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns —
|
| 527 |
# the two other host-only render args the shell was missing (fail-closed without them:
|
| 528 |
# restricted fields uneditable, assignee picker empty).
|
|
@@ -546,8 +591,12 @@ def workspace(scope: str = "customer",
|
|
| 546 |
# ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort
|
| 547 |
# list, and two switches for one behaviour would drift.
|
| 548 |
workspace["scopeKey"] = scope
|
| 549 |
-
|
| 550 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 551 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 552 |
if map_setup:
|
| 553 |
workspace["mapSetup"] = map_setup
|
|
|
|
| 319 |
return database_setup.map_for(_database_key_for_scope(scope), fields, st=runtime)
|
| 320 |
|
| 321 |
|
| 322 |
+
def _workspace_values(values, session, module, fields):
|
| 323 |
+
"""Project stored cell maps onto this request's served field contract.
|
| 324 |
+
|
| 325 |
+
Group containers may carry embedded rows/values. Only those containers recurse;
|
| 326 |
+
the JSON value of a visible field stays verbatim, even if it uses a hidden key.
|
| 327 |
+
"""
|
| 328 |
+
from core import perm_scope
|
| 329 |
+
|
| 330 |
+
visible = {field.get("key") for field in fields if isinstance(field, dict)}
|
| 331 |
+
values = perm_scope.visible_overlays(
|
| 332 |
+
values or {}, session.user, module, fields, st=session.runtime)
|
| 333 |
+
|
| 334 |
+
def cells(raw):
|
| 335 |
+
if isinstance(raw, list):
|
| 336 |
+
return [cells(row) for row in raw]
|
| 337 |
+
if not isinstance(raw, dict):
|
| 338 |
+
return raw
|
| 339 |
+
return {key: value if key in visible else cells(value)
|
| 340 |
+
for key, value in raw.items()
|
| 341 |
+
if key in visible or (key in {"values", "rows", "groups", "children"}
|
| 342 |
+
and isinstance(value, (dict, list)))}
|
| 343 |
+
|
| 344 |
+
return {str(pid): cells(raw) for pid, raw in values.items()}
|
| 345 |
+
|
| 346 |
+
|
| 347 |
@router.get("/workspace")
|
| 348 |
def workspace(scope: str = "customer",
|
| 349 |
+
session: Session = Depends(require_session), fields_only: bool = False):
|
| 350 |
"""The durable table workspace for this session: views, fields, overlays, folders.
|
| 351 |
|
| 352 |
⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope
|
|
|
|
| 368 |
wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a
|
| 369 |
full-access user.
|
| 370 |
"""
|
| 371 |
+
from core import grid_events, shared_overlay
|
| 372 |
from routes_customers import grid_assembly
|
| 373 |
|
| 374 |
scope = _scope_or_400(scope)
|
|
|
|
| 391 |
# bought nothing and cost everything — it is why `?scope=ut_odoo_gl_lines` answered
|
| 392 |
# `409 window_required` six times out of six on the live deploy while the ROWS door for
|
| 393 |
# the same table served a page in ~150 ms, i.e. a grid that could not be opened at all.
|
| 394 |
+
g = ut_assembly(session, scope, storage_key=storage_key, with_rows=False,
|
| 395 |
+
fields_only=fields_only)
|
| 396 |
+
if not fields_only:
|
| 397 |
+
grid_events.workspace_corrections(session.runtime, scope + '_table_workspace',
|
| 398 |
+
session.uname, g)
|
| 399 |
+
except shared_overlay.SchemaUnavailable as exc:
|
| 400 |
+
raise err(503, "schema_unavailable", str(exc))
|
| 401 |
except grid_events.StoreUnavailable:
|
| 402 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 403 |
workspace = g["workspace"]
|
| 404 |
+
workspace["overlays"] = _workspace_values(
|
| 405 |
+
g["ws"].get("overlays"), session, scope, g["fields"])
|
| 406 |
# The field contract rides the cheap re-read on EVERY topic — see the customer branch
|
| 407 |
# below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)`
|
| 408 |
# is scope-parameterized, so `ut_` surfaces have their own sets).
|
|
|
|
| 415 |
# them away, so the Locked-views column arrived on the cheap re-read with permanently
|
| 416 |
# empty values. A column that exists and can never have one is worse than an absent
|
| 417 |
# column: it reads as "this record is in no locked view", which is a claim.
|
| 418 |
+
workspace["derived"] = _workspace_values(g["derived"], session, scope, g["fields"])
|
| 419 |
workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
|
| 420 |
try:
|
| 421 |
from core import users as _users
|
|
|
|
| 429 |
workspace["permissionUserOptions"] = []
|
| 430 |
workspace["userAvatars"] = {}
|
| 431 |
workspace["scopeKey"] = scope
|
| 432 |
+
workspace["auditTimestamps"] = ({} if fields_only else
|
| 433 |
+
_audit_timestamps(scope, g["pids"], set(), session.runtime))
|
| 434 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 435 |
if map_setup:
|
| 436 |
workspace["mapSetup"] = map_setup
|
|
|
|
| 440 |
# `missing` on a shortened cohort). An unannounced empty scope is the silent limit R6
|
| 441 |
# forbids; this is the announcement, and W31-T22 is the ticket that renders it.
|
| 442 |
workspace["limits"] = g.get("limits") or []
|
| 443 |
+
if fields_only:
|
| 444 |
+
workspace["fieldsOnly"] = True
|
| 445 |
return {"workspace": workspace}
|
| 446 |
|
| 447 |
# ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field
|
|
|
|
| 457 |
session.require(PRODUCT_MODULE)
|
| 458 |
storage_key = f"{session.tenant}:product-list:{session.uname}:all:all"
|
| 459 |
try:
|
| 460 |
+
g = product_assembly(session, scope=scope, storage_key=storage_key,
|
| 461 |
+
with_rows=not fields_only)
|
| 462 |
+
if not fields_only:
|
| 463 |
+
grid_events.workspace_corrections(session.runtime, 'product_table_workspace',
|
| 464 |
+
session.uname, g)
|
| 465 |
+
except shared_overlay.SchemaUnavailable as exc:
|
| 466 |
+
raise err(503, "schema_unavailable", str(exc))
|
| 467 |
except grid_events.StoreUnavailable:
|
| 468 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 469 |
workspace = g["workspace"]
|
| 470 |
+
workspace["overlays"] = _workspace_values(
|
| 471 |
+
g["ws"].get("overlays"), session, PRODUCT_MODULE, g["fields"])
|
| 472 |
# R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own
|
| 473 |
# first-cohort contract change. Same reason as the customer branch below.
|
| 474 |
workspace["fields"] = g["fields"]
|
|
|
|
| 480 |
# reason: `product_assembly` builds this topic's cohort cells and this route discarded
|
| 481 |
# them. The measure half stays empty on this topic by descope, which is a different
|
| 482 |
# statement and one the offer already makes.
|
| 483 |
+
workspace["derived"] = _workspace_values(g["derived"], session, PRODUCT_MODULE, g["fields"])
|
| 484 |
workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
|
| 485 |
try:
|
| 486 |
from core import users as _users
|
|
|
|
| 494 |
workspace["permissionUserOptions"] = []
|
| 495 |
workspace["userAvatars"] = {}
|
| 496 |
workspace["scopeKey"] = scope
|
| 497 |
+
workspace["auditTimestamps"] = ({} if fields_only else
|
| 498 |
+
_audit_timestamps(scope, g["pids"], set(), session.runtime))
|
| 499 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 500 |
if map_setup:
|
| 501 |
workspace["mapSetup"] = map_setup
|
| 502 |
+
if fields_only:
|
| 503 |
+
workspace["fieldsOnly"] = True
|
| 504 |
return {"workspace": workspace}
|
| 505 |
|
| 506 |
# The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid`
|
|
|
|
| 524 |
# never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer
|
| 525 |
# surface. One projection for both servers is the fix that cannot drift.
|
| 526 |
try:
|
| 527 |
+
g = grid_assembly(session, scope=scope, storage_key=storage_key, with_rows=not fields_only)
|
| 528 |
+
if not fields_only:
|
| 529 |
+
grid_events.workspace_corrections(session.runtime, 'customer_table_workspace',
|
| 530 |
+
session.uname, g)
|
| 531 |
+
except shared_overlay.SchemaUnavailable as exc:
|
| 532 |
+
raise err(503, "schema_unavailable", str(exc))
|
| 533 |
except grid_events.StoreUnavailable:
|
| 534 |
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 535 |
workspace = g["workspace"]
|
|
|
|
| 541 |
# /customers rows, but this route is the cheap read-back a write can be verified against
|
| 542 |
# (verify_api's overlay probe) without paying the pool call. Per-user by construction —
|
| 543 |
# `table_workspace` is this session's workspace.
|
| 544 |
+
workspace["overlays"] = _workspace_values(
|
| 545 |
+
g["ws"].get("overlays"), session, MODULE, g["fields"])
|
| 546 |
# ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration.
|
| 547 |
#
|
| 548 |
# `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked
|
|
|
|
| 567 |
workspace["measureSets"] = g["measure_sets"]
|
| 568 |
# Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are
|
| 569 |
# strings; the client indexes with String(pid).
|
| 570 |
+
workspace["derived"] = _workspace_values(g["derived"], session, MODULE, g["fields"])
|
| 571 |
# Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns —
|
| 572 |
# the two other host-only render args the shell was missing (fail-closed without them:
|
| 573 |
# restricted fields uneditable, assignee picker empty).
|
|
|
|
| 591 |
# ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort
|
| 592 |
# list, and two switches for one behaviour would drift.
|
| 593 |
workspace["scopeKey"] = scope
|
| 594 |
+
if fields_only:
|
| 595 |
+
workspace["auditTimestamps"] = {}
|
| 596 |
+
workspace["fieldsOnly"] = True
|
| 597 |
+
else:
|
| 598 |
+
ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope)
|
| 599 |
+
workspace["auditTimestamps"] = _audit_timestamps(scope, g["pids"], ctx.hidden_keys, session.runtime)
|
| 600 |
map_setup = _map_setup(scope, workspace["fields"], session.runtime)
|
| 601 |
if map_setup:
|
| 602 |
workspace["mapSetup"] = map_setup
|
api/routes_keychain.py
CHANGED
|
@@ -939,6 +939,16 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 939 |
f"a {etype} connection is what this whole workspace's databases are read "
|
| 940 |
f"through, so it is always business-wide and cannot be a personal connection. "
|
| 941 |
f"An administrator can add it for everyone.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 942 |
try:
|
| 943 |
row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
|
| 944 |
body.get("fields"), session.uname)
|
|
@@ -949,6 +959,13 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 949 |
raise err(400, "bad_entry", str(e))
|
| 950 |
except Exception:
|
| 951 |
raise err(503, "store_unavailable", "the entry was not saved; try again")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 952 |
# ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode
|
| 953 |
# of a side bucket is to publish a credential, not to hide one. So the second write is not
|
| 954 |
# best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was
|
|
@@ -1577,11 +1594,11 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1577 |
return cur
|
| 1578 |
|
| 1579 |
# ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT
|
| 1580 |
-
# RATHER THAN A LIMIT. `rel.frozen`
|
| 1581 |
# materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`,
|
| 1582 |
# `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read
|
| 1583 |
# THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`,
|
| 1584 |
-
# which
|
| 1585 |
# workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's
|
| 1586 |
# own sentence promised *"nothing is being refreshed"*.
|
| 1587 |
#
|
|
@@ -1594,6 +1611,9 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1594 |
# · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to
|
| 1595 |
# serve; a failed capture leaves the connector live rather than paused-with-nothing.
|
| 1596 |
_, flag_key = _resolved_odoo_key(session.runtime)
|
|
|
|
|
|
|
|
|
|
| 1597 |
snapshots = 0
|
| 1598 |
if flag_key:
|
| 1599 |
try:
|
|
@@ -1602,14 +1622,16 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1602 |
# A snapshot is a nicety; the freeze is the promise. Reported, never fatal.
|
| 1603 |
snapshots = 0
|
| 1604 |
|
| 1605 |
-
# ⚠
|
| 1606 |
-
#
|
| 1607 |
# frozen with an unused key, which is recoverable from the Keychains pane. The reverse order
|
| 1608 |
# can leave a workspace with no key and a connector that still tries to sync.
|
| 1609 |
try:
|
| 1610 |
-
session.runtime.update(rel.CONFIG_KEY, _up, flush="sync")
|
|
|
|
|
|
|
| 1611 |
except Exception:
|
| 1612 |
-
raise err(503, "store_unavailable", "nothing was disconnected; try again")
|
| 1613 |
|
| 1614 |
paused_mirror = False
|
| 1615 |
if flag_key:
|
|
@@ -1618,10 +1640,15 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1618 |
cur[str(flag_key)] = {"paused": True}
|
| 1619 |
return cur
|
| 1620 |
try:
|
| 1621 |
-
session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync")
|
|
|
|
|
|
|
|
|
|
| 1622 |
paused_mirror = True
|
| 1623 |
except Exception: # noqa: BLE001
|
| 1624 |
-
|
|
|
|
|
|
|
| 1625 |
|
| 1626 |
removed = ""
|
| 1627 |
if entry is not None:
|
|
@@ -1641,10 +1668,6 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1641 |
# the limit that cannot be removed is REPORTED, with what to do instead.
|
| 1642 |
note = ("Your Odoo databases are frozen: every row and every column you had is still there and "
|
| 1643 |
"still readable, and nothing is being refreshed.")
|
| 1644 |
-
if not paused_mirror and flag_key:
|
| 1645 |
-
note += (" ⚠ The live mirror could not be paused, so the two read-through databases "
|
| 1646 |
-
"(order lines and GL lines) may keep advancing; pause the Odoo connector under "
|
| 1647 |
-
"Keychains to stop them.")
|
| 1648 |
note += (" Reconnecting adds the key back and resumes into the same databases." if entry
|
| 1649 |
else " This workspace's Odoo credential comes from the deployment environment, so "
|
| 1650 |
"there was no stored key to remove; the databases are frozen and Reconnect "
|
|
|
|
| 939 |
f"a {etype} connection is what this whole workspace's databases are read "
|
| 940 |
f"through, so it is always business-wide and cannot be a personal connection. "
|
| 941 |
f"An administrator can add it for everyone.")
|
| 942 |
+
if "initialDatabases" in body:
|
| 943 |
+
from connector_onboarding import install_selection
|
| 944 |
+
try:
|
| 945 |
+
existing = any(entry.get("type") == etype for entry in kc.list_entries(session.runtime))
|
| 946 |
+
existing = existing or (etype == "odoo" and env_odoo_available(session.runtime))
|
| 947 |
+
install_selection(session.runtime, etype, body["initialDatabases"], existing=existing)
|
| 948 |
+
except ValueError as exc:
|
| 949 |
+
raise err(400, "bad_initial_selection", str(exc))
|
| 950 |
+
except Exception:
|
| 951 |
+
raise err(503, "store_unavailable", "The database choices were not saved. Try again.")
|
| 952 |
try:
|
| 953 |
row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
|
| 954 |
body.get("fields"), session.uname)
|
|
|
|
| 959 |
raise err(400, "bad_entry", str(e))
|
| 960 |
except Exception:
|
| 961 |
raise err(503, "store_unavailable", "the entry was not saved; try again")
|
| 962 |
+
if "initialDatabases" in body:
|
| 963 |
+
from connector_onboarding import seal_selection
|
| 964 |
+
try:
|
| 965 |
+
seal_selection(session.runtime, etype)
|
| 966 |
+
except Exception:
|
| 967 |
+
raise err(503, "store_unavailable",
|
| 968 |
+
"The key was saved, but setup did not finish. Open Settings before trying again.")
|
| 969 |
# ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode
|
| 970 |
# of a side bucket is to publish a credential, not to hide one. So the second write is not
|
| 971 |
# best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was
|
|
|
|
| 1594 |
return cur
|
| 1595 |
|
| 1596 |
# ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT
|
| 1597 |
+
# RATHER THAN A LIMIT. `rel.frozen` originally had one consumer, `odoo_relational.refresh`, which
|
| 1598 |
# materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`,
|
| 1599 |
# `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read
|
| 1600 |
# THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`,
|
| 1601 |
+
# which originally gated only on the connector PAUSE flag. So a disconnected
|
| 1602 |
# workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's
|
| 1603 |
# own sentence promised *"nothing is being refreshed"*.
|
| 1604 |
#
|
|
|
|
| 1611 |
# · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to
|
| 1612 |
# serve; a failed capture leaves the connector live rather than paused-with-nothing.
|
| 1613 |
_, flag_key = _resolved_odoo_key(session.runtime)
|
| 1614 |
+
if not flag_key:
|
| 1615 |
+
raise err(503, "source_unavailable",
|
| 1616 |
+
"nothing was disconnected; the Odoo source could not be resolved for pausing")
|
| 1617 |
snapshots = 0
|
| 1618 |
if flag_key:
|
| 1619 |
try:
|
|
|
|
| 1622 |
# A snapshot is a nicety; the freeze is the promise. Reported, never fatal.
|
| 1623 |
snapshots = 0
|
| 1624 |
|
| 1625 |
+
# ⚠ BOTH FLAGS FIRST, THE CREDENTIAL SECOND. If either flag write fails, the key is kept;
|
| 1626 |
+
# if the delete failed AFTER the flags landed, the tenant is
|
| 1627 |
# frozen with an unused key, which is recoverable from the Keychains pane. The reverse order
|
| 1628 |
# can leave a workspace with no key and a connector that still tries to sync.
|
| 1629 |
try:
|
| 1630 |
+
frozen = session.runtime.update(rel.CONFIG_KEY, _up, flush="sync")
|
| 1631 |
+
if not isinstance(frozen, dict) or frozen.get("frozen") is not True:
|
| 1632 |
+
raise RuntimeError("the Odoo freeze was not confirmed")
|
| 1633 |
except Exception:
|
| 1634 |
+
raise err(503, "store_unavailable", "nothing was disconnected; try again") from None
|
| 1635 |
|
| 1636 |
paused_mirror = False
|
| 1637 |
if flag_key:
|
|
|
|
| 1640 |
cur[str(flag_key)] = {"paused": True}
|
| 1641 |
return cur
|
| 1642 |
try:
|
| 1643 |
+
flags = session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync")
|
| 1644 |
+
saved = flags.get(str(flag_key)) if isinstance(flags, dict) else None
|
| 1645 |
+
if not isinstance(saved, dict) or saved.get("paused") is not True:
|
| 1646 |
+
raise RuntimeError("the Odoo pause was not confirmed")
|
| 1647 |
paused_mirror = True
|
| 1648 |
except Exception: # noqa: BLE001
|
| 1649 |
+
raise err(503, "store_unavailable",
|
| 1650 |
+
"the databases are frozen but the Odoo pause could not be saved; "
|
| 1651 |
+
"no credential was removed; try again") from None
|
| 1652 |
|
| 1653 |
removed = ""
|
| 1654 |
if entry is not None:
|
|
|
|
| 1668 |
# the limit that cannot be removed is REPORTED, with what to do instead.
|
| 1669 |
note = ("Your Odoo databases are frozen: every row and every column you had is still there and "
|
| 1670 |
"still readable, and nothing is being refreshed.")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1671 |
note += (" Reconnecting adds the key back and resumes into the same databases." if entry
|
| 1672 |
else " This workspace's Odoo credential comes from the deployment environment, so "
|
| 1673 |
"there was no stored key to remove; the databases are frozen and Reconnect "
|
api/routes_nav.py
CHANGED
|
@@ -562,10 +562,14 @@ def _odoo_connection_folders(runtime):
|
|
| 562 |
return {}, None
|
| 563 |
try:
|
| 564 |
import core.keychain as keychain
|
|
|
|
| 565 |
import odoo_relational as odoo_rel
|
|
|
|
|
|
|
|
|
|
| 566 |
connections = 0
|
| 567 |
labels = {}
|
| 568 |
-
for row in keychain.list_entries(
|
| 569 |
if str(row.get("type") or "") != "odoo":
|
| 570 |
continue
|
| 571 |
connections += 1
|
|
@@ -574,8 +578,8 @@ def _odoo_connection_folders(runtime):
|
|
| 574 |
if cid and label and label.lower() not in _RESERVED_GROUP_LABELS:
|
| 575 |
labels[cid] = label
|
| 576 |
if connections < 2:
|
| 577 |
-
return {}, odoo_rel.default_connection_id(
|
| 578 |
-
return labels, odoo_rel.default_connection_id(
|
| 579 |
except Exception: # noqa: BLE001
|
| 580 |
return {}, None
|
| 581 |
|
|
@@ -891,6 +895,9 @@ def nav_opened(body: dict = Body(default=None),
|
|
| 891 |
key = str(body.get("key") or "").strip()[:60]
|
| 892 |
if not key:
|
| 893 |
raise err(400, "bad_request", "no page was named")
|
|
|
|
|
|
|
|
|
|
| 894 |
if not session.runtime.available():
|
| 895 |
raise err(503, "store_unavailable",
|
| 896 |
"the tenant store is unavailable. Nothing was recorded.")
|
|
|
|
| 562 |
return {}, None
|
| 563 |
try:
|
| 564 |
import core.keychain as keychain
|
| 565 |
+
import core.user_tables as user_tables
|
| 566 |
import odoo_relational as odoo_rel
|
| 567 |
+
# Both existing metadata resolvers see this one request's captured document. The
|
| 568 |
+
# keychain bucket is explicitly supplied; it never joins the automatic lend list.
|
| 569 |
+
borrowed = user_tables.lend(runtime, **{keychain.KEY: runtime.get(keychain.KEY) or {}})
|
| 570 |
connections = 0
|
| 571 |
labels = {}
|
| 572 |
+
for row in keychain.list_entries(borrowed):
|
| 573 |
if str(row.get("type") or "") != "odoo":
|
| 574 |
continue
|
| 575 |
connections += 1
|
|
|
|
| 578 |
if cid and label and label.lower() not in _RESERVED_GROUP_LABELS:
|
| 579 |
labels[cid] = label
|
| 580 |
if connections < 2:
|
| 581 |
+
return {}, odoo_rel.default_connection_id(borrowed)
|
| 582 |
+
return labels, odoo_rel.default_connection_id(borrowed)
|
| 583 |
except Exception: # noqa: BLE001
|
| 584 |
return {}, None
|
| 585 |
|
|
|
|
| 895 |
key = str(body.get("key") or "").strip()[:60]
|
| 896 |
if not key:
|
| 897 |
raise err(400, "bad_request", "no page was named")
|
| 898 |
+
from core import freeze_notice
|
| 899 |
+
if freeze_notice.enabled():
|
| 900 |
+
return {"key": key, "at": _now()}
|
| 901 |
if not session.runtime.available():
|
| 902 |
raise err(503, "store_unavailable",
|
| 903 |
"the tenant store is unavailable. Nothing was recorded.")
|
api/routes_odoo_tables.py
CHANGED
|
@@ -112,7 +112,10 @@ def _table_cursor(rt, table_key=None):
|
|
| 112 |
if rt is not None:
|
| 113 |
rt.assert_datastore_matches()
|
| 114 |
return datastore.ro_cursor()
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
@router.post("/odoo-tables/refresh")
|
|
@@ -192,10 +195,9 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 192 |
without anybody running a refresh to find out.
|
| 193 |
|
| 194 |
`brief=1` is the grid's one question at open time: which declared tables read through the
|
| 195 |
-
mirror?
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
a client-side key list.
|
| 199 |
"""
|
| 200 |
rel = _rel()
|
| 201 |
if brief:
|
|
@@ -208,6 +210,15 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 208 |
# serves is read-through; none is listed when this container's mirror holds no Amazon rows.
|
| 209 |
tables.update({key: {"readThrough": True} for key in sources
|
| 210 |
if str(key).startswith("ut_amzn_") and key not in tables})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
return {"ok": True, "tables": tables}
|
| 212 |
|
| 213 |
import core.user_tables as user_tables
|
|
@@ -725,25 +736,21 @@ def _sources():
|
|
| 725 |
return GRID_SOURCES
|
| 726 |
|
| 727 |
|
| 728 |
-
def _learn_amazon_sources():
|
| 729 |
-
"""Publish the
|
| 730 |
-
|
| 731 |
-
HOTFIX 2026-09-27 (moved here by S67-T28 so every door shares it): a registry built at boot
|
| 732 |
-
BEFORE the Amazon mirror restore holds no Amazon key, and only `refresh_state` invalidates
|
| 733 |
-
it. `_source_for` learned them on demand; `_materialised_table` did not, so a read-through
|
| 734 |
-
Amazon LINK TARGET (orders, 559k rows) was dispatched as a stored table and refused as more
|
| 735 |
-
than one window. A registry that already knows one Amazon key is not re-asked.
|
| 736 |
-
"""
|
| 737 |
_sources()
|
| 738 |
-
if
|
| 739 |
-
|
| 740 |
-
|
|
|
|
|
|
|
| 741 |
if not extra:
|
| 742 |
return
|
| 743 |
with _SOURCES_LOCK:
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
|
|
|
| 747 |
|
| 748 |
|
| 749 |
def _odoo_sources():
|
|
@@ -1092,6 +1099,8 @@ def _odoo_sources():
|
|
| 1092 |
"move_type": ("move_type", _s),
|
| 1093 |
},
|
| 1094 |
}
|
|
|
|
|
|
|
| 1095 |
return GRID_SOURCES
|
| 1096 |
|
| 1097 |
|
|
@@ -1160,7 +1169,7 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
|
|
| 1160 |
# `_sources()` skip its build, so the grid resolved while its link targets
|
| 1161 |
# (orders, read-through) were dispatched as stored tables: a first-page 500.
|
| 1162 |
if spec is None or spec.get("missing"):
|
| 1163 |
-
_learn_amazon_sources()
|
| 1164 |
with _SOURCES_LOCK:
|
| 1165 |
_sources()
|
| 1166 |
held = GRID_SOURCES.get(table_key)
|
|
@@ -1400,13 +1409,13 @@ def _source_for(cur, table_key):
|
|
| 1400 |
if module_cache_source(key) is not None:
|
| 1401 |
return None
|
| 1402 |
spec = _sources().get(key)
|
| 1403 |
-
if not spec and key.startswith("ut_amzn_"):
|
| 1404 |
# HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
|
| 1405 |
# Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
|
| 1406 |
# runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
|
| 1407 |
-
# read-through Amazon grid answered "no read-through binding".
|
| 1408 |
-
#
|
| 1409 |
-
_learn_amazon_sources()
|
| 1410 |
spec = GRID_SOURCES.get(key)
|
| 1411 |
if not spec:
|
| 1412 |
return None
|
|
@@ -1974,7 +1983,8 @@ def apply_filtered_bulk(table_key: str, body: dict = Body(default=None),
|
|
| 1974 |
definitions = shared_overlay.fields(table_key, st=session.runtime)
|
| 1975 |
if key not in definitions:
|
| 1976 |
raise err(400, "field_not_shared", "bulk updates on connected data require an existing shared field")
|
| 1977 |
-
if key in perm_scope.hidden_keys(session.user, table_key, list(definitions.values())
|
|
|
|
| 1978 |
raise err(403, "field_hidden", "you do not have access to that field")
|
| 1979 |
value = body.get("value")
|
| 1980 |
if isinstance(value, (dict, list, tuple, set)):
|
|
@@ -3203,7 +3213,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 3203 |
# two doors.
|
| 3204 |
import core.perm_scope as perm_scope
|
| 3205 |
wall_sql, wall_params = None, []
|
| 3206 |
-
_hidden = perm_scope.hidden_keys(session.user, table_key, fields)
|
| 3207 |
# `row_scope_applies` RATHER THAN AN ADMIN TEST SPELLED OUT HERE: it is this repo's ONE
|
| 3208 |
# statement of when the row wall bites (False for an admin, False for a record with no
|
| 3209 |
# declared filter), and a second spelling of it beside the first is how two of them come
|
|
@@ -3721,13 +3731,13 @@ def odoo_table_rows(table_key: str,
|
|
| 3721 |
# in an overlay column that reads a hidden mirror column was outside its reach - and a formula
|
| 3722 |
# carrying a hidden value is the leak `hidden_keys` exists to close, wearing a second door's
|
| 3723 |
# name. Same wall, asked once the field list is whole.
|
| 3724 |
-
_hidden = perm_scope.hidden_keys(session.user, table_key, fields)
|
| 3725 |
if _hidden:
|
| 3726 |
# BOTH WIRES, and the field list is narrowed BEFORE the rows are built so a hidden column
|
| 3727 |
# is never assembled rather than assembled and then removed. `strip_row` runs anyway on
|
| 3728 |
# the overlay-merged result, because a shared-overlay cell arrives by a different door
|
| 3729 |
# than the mirror window and one narrowing cannot speak for both.
|
| 3730 |
-
fields = perm_scope.visible_fields(fields, session.user, table_key)
|
| 3731 |
rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
|
| 3732 |
overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
|
| 3733 |
for pid, cells in overlays.items()}
|
|
@@ -3755,12 +3765,8 @@ def odoo_table_rows(table_key: str,
|
|
| 3755 |
# changes as you scroll, and nothing on screen says so.
|
| 3756 |
# ══════════════════════════════════════════════════════════
|
| 3757 |
|
| 3758 |
-
#:
|
| 3759 |
-
#:
|
| 3760 |
-
#: ⛔ STANDING RULE 1'S SECOND SENTENCE IS THE WHOLE OF THIS CONSTANT'S CONTRACT: over the cap
|
| 3761 |
-
#: the response carries the largest 1,000 by row count AND a `limits` entry naming the true
|
| 3762 |
-
#: distinct count, never a silent 1,000 that reads as the whole answer. A cap that does not report
|
| 3763 |
-
#: itself is the failure this number exists to avoid rather than to cause.
|
| 3764 |
MAX_GROUPS = 1_000
|
| 3765 |
|
| 3766 |
#: One request asks for at most this many summaries. Each is one more aggregate expression in a
|
|
@@ -3889,7 +3895,8 @@ def _agg_plan(cols, aggs, limits):
|
|
| 3889 |
return plan
|
| 3890 |
|
| 3891 |
|
| 3892 |
-
def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, limits=None
|
|
|
|
| 3893 |
"""The C1 summary itself: `{total_rows, aggregates, groups, limits}` over ONE compiled `where`.
|
| 3894 |
|
| 3895 |
⛔⛔ PURE OF THE SESSION ON PURPOSE. Everything session-shaped (both walls, the
|
|
@@ -3931,10 +3938,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 3931 |
gexpr = _fs().group_sql(gspec)
|
| 3932 |
|
| 3933 |
# ⛔⛔ THE OVERALL SUMMARY IS COMPUTED WHETHER OR NOT THIS REQUEST GROUPS, AND IT IS **NOT**
|
| 3934 |
-
# RECOVERED FROM THE GROUPS.
|
| 3935 |
-
#
|
| 3936 |
-
# buckets while calling it the total; and a `median` or a `percentile` does not compose from
|
| 3937 |
-
# per-group medians at all, at any cap. The first cut of this function returned an all-`null`
|
| 3938 |
# `aggregates` beside a populated `groups`, which would have put blanks in the summary bar of
|
| 3939 |
# every grouped view - caught by the gate leg that drives the same call ungrouped.
|
| 3940 |
# ⚠ AND `total_rows` COMES FROM **THIS** STATEMENT IN BOTH SHAPES. The grouped read below runs
|
|
@@ -3951,37 +3956,22 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 3951 |
return {"total_rows": total_rows, "aggregates": answers, "groups": None,
|
| 3952 |
"limits": limits}
|
| 3953 |
|
| 3954 |
-
#
|
| 3955 |
-
#
|
| 3956 |
-
|
| 3957 |
-
|
| 3958 |
-
|
| 3959 |
-
|
| 3960 |
-
|
| 3961 |
-
|
| 3962 |
-
|
| 3963 |
-
|
| 3964 |
-
|
| 3965 |
-
|
| 3966 |
-
|
| 3967 |
-
|
| 3968 |
-
|
| 3969 |
-
|
| 3970 |
-
# a numeric NULL alone (a missing amount is not a zero), so a numeric group column can
|
| 3971 |
-
# carry a blank group. Without this the sentence would understate by one, which is a
|
| 3972 |
-
# small lie in the one place whose entire job is to state how much is missing.
|
| 3973 |
-
distinct = _agg_number(datastore.window(
|
| 3974 |
-
select=(f"count(DISTINCT {gexpr}) + CASE WHEN count(*) - count({gexpr}) > 0 "
|
| 3975 |
-
f"THEN 1 ELSE 0 END"),
|
| 3976 |
-
where=where, params=tuple(params),
|
| 3977 |
-
order_by="1", limit=1, cur=cur, **frm)["rows"][0][0]) or 0
|
| 3978 |
-
limits.append({
|
| 3979 |
-
"subject": gcol, "effect": "groups_truncated",
|
| 3980 |
-
"cause": f"this column has {int(distinct):,} distinct values under the current "
|
| 3981 |
-
f"filter and one response carries at most {MAX_GROUPS:,} groups, so the "
|
| 3982 |
-
f"{MAX_GROUPS:,} with the most rows are here and the rest are not",
|
| 3983 |
-
"recommendation": "narrow the filter first, or group by a column with fewer distinct "
|
| 3984 |
-
"values"})
|
| 3985 |
groups = []
|
| 3986 |
for r in rows:
|
| 3987 |
vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
|
|
@@ -3989,7 +3979,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 3989 |
groups.append({"key": _group_key(r[0]), "label": _group_label(r[0]),
|
| 3990 |
"count": int(r[1] or 0), "aggregates": vals})
|
| 3991 |
return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
|
| 3992 |
-
"
|
|
|
|
| 3993 |
|
| 3994 |
|
| 3995 |
@router.post("/odoo-tables/{table_key}/aggregates")
|
|
@@ -3997,8 +3988,9 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 3997 |
session: Session = Depends(require_session)):
|
| 3998 |
"""SERVER-SIDE SUMMARIES over the WHOLE predicate a server-windowed grid is showing (C1).
|
| 3999 |
|
| 4000 |
-
Body `{filters, filterConj, search, related?, groupBy?,
|
| 4001 |
-
`{ok, total_rows, aggregates, groups?, limits}`
|
|
|
|
| 4002 |
|
| 4003 |
⚠ `related` IS A BODY KEY HERE AND A JSON QUERY STRING ON `/rows`. Same object either way
|
| 4004 |
(`{table, record, via}`), same compiler, two transports - because this door is a POST and that
|
|
@@ -4013,7 +4005,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 4013 |
⛔ Anything unanswerable is a `limits` sentence with a cause and a recommendation, in the
|
| 4014 |
SAME four-key shape `/rows` uses, and never a silent 0: an unknown column, a hidden one, a
|
| 4015 |
numeric summary over text, a percentile with no `p`, a second summary for one column, an
|
| 4016 |
-
ungroupable `groupBy`
|
| 4017 |
"""
|
| 4018 |
payload = body if isinstance(body, dict) else {}
|
| 4019 |
# ⚠ THE WIRE SHAPE IS VALIDATED BY THE SAME HELPER THE BULK DOORS USE, not re-checked by
|
|
@@ -4032,12 +4024,23 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 4032 |
group_by = payload.get("groupBy")
|
| 4033 |
if group_by is not None and not isinstance(group_by, str):
|
| 4034 |
raise err(400, "bad_selection", "groupBy must be a column id")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4035 |
|
| 4036 |
if _materialised_table(table_key):
|
| 4037 |
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
|
| 4038 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 4039 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 4040 |
-
limits=read["limits"]
|
|
|
|
| 4041 |
return {"ok": True, **out, "scope": {"table": table_key, "readThrough": True}}
|
| 4042 |
|
| 4043 |
# ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3).
|
|
@@ -4049,15 +4052,17 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 4049 |
# come from here) no longer re-reads every definition while `user_tables` is unchanged.
|
| 4050 |
read = _compiled_read(table_key, session, tree=query["filters"],
|
| 4051 |
filterConj=query["filterConj"], search=query["search"],
|
| 4052 |
-
related=payload.get("related"), st=_page_defs(session)[0])
|
| 4053 |
read = _restrict_row_ids(read, ids)
|
| 4054 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 4055 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 4056 |
-
limits=read["limits"]
|
|
|
|
| 4057 |
body_out = {"ok": True, "total_rows": out["total_rows"], "aggregates": out["aggregates"],
|
| 4058 |
"limits": out["limits"], "scope": {"table": table_key, "readThrough": True}}
|
| 4059 |
if out["groups"] is not None:
|
| 4060 |
body_out["groups"] = out["groups"]
|
|
|
|
| 4061 |
return body_out
|
| 4062 |
|
| 4063 |
|
|
|
|
| 112 |
if rt is not None:
|
| 113 |
rt.assert_datastore_matches()
|
| 114 |
return datastore.ro_cursor()
|
| 115 |
+
if rt is not None:
|
| 116 |
+
from connector_onboarding import mirror_cursor
|
| 117 |
+
return mirror_cursor(rt, table_key)
|
| 118 |
+
return datastore.ro_con()
|
| 119 |
|
| 120 |
|
| 121 |
@router.post("/odoo-tables/refresh")
|
|
|
|
| 195 |
without anybody running a refresh to find out.
|
| 196 |
|
| 197 |
`brief=1` is the grid's one question at open time: which declared tables read through the
|
| 198 |
+
mirror? It reads only definitions, never stored rows or mirror counts. Persisted Amazon
|
| 199 |
+
stubs retain their routing when the local mirror has not loaded; the rows door then names
|
| 200 |
+
that availability failure rather than painting an empty stored table.
|
|
|
|
| 201 |
"""
|
| 202 |
rel = _rel()
|
| 203 |
if brief:
|
|
|
|
| 210 |
# serves is read-through; none is listed when this container's mirror holds no Amazon rows.
|
| 211 |
tables.update({key: {"readThrough": True} for key in sources
|
| 212 |
if str(key).startswith("ut_amzn_") and key not in tables})
|
| 213 |
+
ut = _ut()
|
| 214 |
+
from core import perm_scope
|
| 215 |
+
definitions = ut.all_defs(st=session.runtime)
|
| 216 |
+
lent = ut.lend(session.runtime, **{ut.STORE_KEY: definitions})
|
| 217 |
+
for key, definition in definitions.items():
|
| 218 |
+
if (str(key).startswith("ut_amzn_") and isinstance(definition, dict)
|
| 219 |
+
and definition.get("readThrough") is True
|
| 220 |
+
and perm_scope.may_read(session.user, key, st=lent)):
|
| 221 |
+
tables[key] = {"readThrough": True}
|
| 222 |
return {"ok": True, "tables": tables}
|
| 223 |
|
| 224 |
import core.user_tables as user_tables
|
|
|
|
| 736 |
return GRID_SOURCES
|
| 737 |
|
| 738 |
|
| 739 |
+
def _learn_amazon_sources(cur=None):
|
| 740 |
+
"""Publish the complete current Amazon binding set, including restored missing entries."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 741 |
_sources()
|
| 742 |
+
if cur is None:
|
| 743 |
+
extra = _amazon_sources()
|
| 744 |
+
else:
|
| 745 |
+
import amazon_relational
|
| 746 |
+
extra = amazon_relational.grid_sources(cur)
|
| 747 |
if not extra:
|
| 748 |
return
|
| 749 |
with _SOURCES_LOCK:
|
| 750 |
+
# One known key cannot certify every history table. Replace old missing bindings as
|
| 751 |
+
# one publication, so the first rows AND aggregates request sees the restored schema.
|
| 752 |
+
GRID_SOURCES.update(extra)
|
| 753 |
+
_ut().register_connected(*extra)
|
| 754 |
|
| 755 |
|
| 756 |
def _odoo_sources():
|
|
|
|
| 1099 |
"move_type": ("move_type", _s),
|
| 1100 |
},
|
| 1101 |
}
|
| 1102 |
+
GRID_SOURCES.update(rel.stock_source_specs())
|
| 1103 |
+
GRID_SOURCES.update(rel.purchase_source_specs())
|
| 1104 |
return GRID_SOURCES
|
| 1105 |
|
| 1106 |
|
|
|
|
| 1169 |
# `_sources()` skip its build, so the grid resolved while its link targets
|
| 1170 |
# (orders, read-through) were dispatched as stored tables: a first-page 500.
|
| 1171 |
if spec is None or spec.get("missing"):
|
| 1172 |
+
_learn_amazon_sources(cur)
|
| 1173 |
with _SOURCES_LOCK:
|
| 1174 |
_sources()
|
| 1175 |
held = GRID_SOURCES.get(table_key)
|
|
|
|
| 1409 |
if module_cache_source(key) is not None:
|
| 1410 |
return None
|
| 1411 |
spec = _sources().get(key)
|
| 1412 |
+
if (not spec or spec.get("missing")) and key.startswith("ut_amzn_"):
|
| 1413 |
# HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
|
| 1414 |
# Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
|
| 1415 |
# runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
|
| 1416 |
+
# read-through Amazon grid answered "no read-through binding". S70-T12 re-asks the
|
| 1417 |
+
# complete Amazon provider on this restored cursor, including cached missing entries.
|
| 1418 |
+
_learn_amazon_sources(cur)
|
| 1419 |
spec = GRID_SOURCES.get(key)
|
| 1420 |
if not spec:
|
| 1421 |
return None
|
|
|
|
| 1983 |
definitions = shared_overlay.fields(table_key, st=session.runtime)
|
| 1984 |
if key not in definitions:
|
| 1985 |
raise err(400, "field_not_shared", "bulk updates on connected data require an existing shared field")
|
| 1986 |
+
if key in perm_scope.hidden_keys(session.user, table_key, list(definitions.values()),
|
| 1987 |
+
st=session.runtime):
|
| 1988 |
raise err(403, "field_hidden", "you do not have access to that field")
|
| 1989 |
value = body.get("value")
|
| 1990 |
if isinstance(value, (dict, list, tuple, set)):
|
|
|
|
| 3213 |
# two doors.
|
| 3214 |
import core.perm_scope as perm_scope
|
| 3215 |
wall_sql, wall_params = None, []
|
| 3216 |
+
_hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
|
| 3217 |
# `row_scope_applies` RATHER THAN AN ADMIN TEST SPELLED OUT HERE: it is this repo's ONE
|
| 3218 |
# statement of when the row wall bites (False for an admin, False for a record with no
|
| 3219 |
# declared filter), and a second spelling of it beside the first is how two of them come
|
|
|
|
| 3731 |
# in an overlay column that reads a hidden mirror column was outside its reach - and a formula
|
| 3732 |
# carrying a hidden value is the leak `hidden_keys` exists to close, wearing a second door's
|
| 3733 |
# name. Same wall, asked once the field list is whole.
|
| 3734 |
+
_hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
|
| 3735 |
if _hidden:
|
| 3736 |
# BOTH WIRES, and the field list is narrowed BEFORE the rows are built so a hidden column
|
| 3737 |
# is never assembled rather than assembled and then removed. `strip_row` runs anyway on
|
| 3738 |
# the overlay-merged result, because a shared-overlay cell arrives by a different door
|
| 3739 |
# than the mirror window and one narrowing cannot speak for both.
|
| 3740 |
+
fields = perm_scope.visible_fields(fields, session.user, table_key, st=session.runtime)
|
| 3741 |
rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
|
| 3742 |
overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
|
| 3743 |
for pid, cells in overlays.items()}
|
|
|
|
| 3765 |
# changes as you scroll, and nothing on screen says so.
|
| 3766 |
# ══════════════════════════════════════════════════════════
|
| 3767 |
|
| 3768 |
+
#: Maximum requested group PAGE size, never a ceiling on reachable groups. Legacy callers
|
| 3769 |
+
#: that omit groupLimit receive the complete grouping; card modes page columns explicitly.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3770 |
MAX_GROUPS = 1_000
|
| 3771 |
|
| 3772 |
#: One request asks for at most this many summaries. Each is one more aggregate expression in a
|
|
|
|
| 3895 |
return plan
|
| 3896 |
|
| 3897 |
|
| 3898 |
+
def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, limits=None,
|
| 3899 |
+
group_offset=0, group_limit=None):
|
| 3900 |
"""The C1 summary itself: `{total_rows, aggregates, groups, limits}` over ONE compiled `where`.
|
| 3901 |
|
| 3902 |
⛔⛔ PURE OF THE SESSION ON PURPOSE. Everything session-shaped (both walls, the
|
|
|
|
| 3938 |
gexpr = _fs().group_sql(gspec)
|
| 3939 |
|
| 3940 |
# ⛔⛔ THE OVERALL SUMMARY IS COMPUTED WHETHER OR NOT THIS REQUEST GROUPS, AND IT IS **NOT**
|
| 3941 |
+
# RECOVERED FROM THE GROUPS. A page may hold only some groups, and a `median` or a
|
| 3942 |
+
# `percentile` does not compose from per-group medians at all. The first cut returned an all-`null`
|
|
|
|
|
|
|
| 3943 |
# `aggregates` beside a populated `groups`, which would have put blanks in the summary bar of
|
| 3944 |
# every grouped view - caught by the gate leg that drives the same call ungrouped.
|
| 3945 |
# ⚠ AND `total_rows` COMES FROM **THIS** STATEMENT IN BOTH SHAPES. The grouped read below runs
|
|
|
|
| 3956 |
return {"total_rows": total_rows, "aggregates": answers, "groups": None,
|
| 3957 |
"limits": limits}
|
| 3958 |
|
| 3959 |
+
# S70-T12: paging bounds response memory without discarding any column. Count the NULL
|
| 3960 |
+
# bucket too. All SQL expressions/from clauses come from the already walled compiler.
|
| 3961 |
+
distinct = int(datastore.window(
|
| 3962 |
+
select=(f"count(DISTINCT {gexpr}) + CASE WHEN count(*) - count({gexpr}) > 0 "
|
| 3963 |
+
f"THEN 1 ELSE 0 END"),
|
| 3964 |
+
where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
|
| 3965 |
+
projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj)
|
| 3966 |
+
if group_limit is not None:
|
| 3967 |
+
rows = datastore.window(
|
| 3968 |
+
select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
|
| 3969 |
+
group_by=gexpr, offset=group_offset, limit=group_limit, cur=cur, **frm)["rows"]
|
| 3970 |
+
else:
|
| 3971 |
+
source = frm.get("from_sql") or frm["table"]
|
| 3972 |
+
tail = f" WHERE {where}" if where else ""
|
| 3973 |
+
rows = cur.execute(f"SELECT {projection} FROM {source}{tail} GROUP BY {gexpr} "
|
| 3974 |
+
"ORDER BY 2 DESC, 1 ASC", tuple(params)).fetchall()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3975 |
groups = []
|
| 3976 |
for r in rows:
|
| 3977 |
vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
|
|
|
|
| 3979 |
groups.append({"key": _group_key(r[0]), "label": _group_label(r[0]),
|
| 3980 |
"count": int(r[1] or 0), "aggregates": vals})
|
| 3981 |
return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
|
| 3982 |
+
"total_groups": distinct, "group_offset": group_offset,
|
| 3983 |
+
"group_limit": group_limit, "limits": limits}
|
| 3984 |
|
| 3985 |
|
| 3986 |
@router.post("/odoo-tables/{table_key}/aggregates")
|
|
|
|
| 3988 |
session: Session = Depends(require_session)):
|
| 3989 |
"""SERVER-SIDE SUMMARIES over the WHOLE predicate a server-windowed grid is showing (C1).
|
| 3990 |
|
| 3991 |
+
Body `{filters, filterConj, search, related?, groupBy?, groupOffset?, groupLimit?,
|
| 3992 |
+
aggs: [{colId, agg, p?}]}`; response `{ok, total_rows, aggregates, groups?, limits}`
|
| 3993 |
+
plus `total_groups`, `group_offset`, `group_limit` when grouped. No group is capped away.
|
| 3994 |
|
| 3995 |
⚠ `related` IS A BODY KEY HERE AND A JSON QUERY STRING ON `/rows`. Same object either way
|
| 3996 |
(`{table, record, via}`), same compiler, two transports - because this door is a POST and that
|
|
|
|
| 4005 |
⛔ Anything unanswerable is a `limits` sentence with a cause and a recommendation, in the
|
| 4006 |
SAME four-key shape `/rows` uses, and never a silent 0: an unknown column, a hidden one, a
|
| 4007 |
numeric summary over text, a percentile with no `p`, a second summary for one column, an
|
| 4008 |
+
ungroupable `groupBy`. Group pages have a bounded size and every group remains reachable.
|
| 4009 |
"""
|
| 4010 |
payload = body if isinstance(body, dict) else {}
|
| 4011 |
# ⚠ THE WIRE SHAPE IS VALIDATED BY THE SAME HELPER THE BULK DOORS USE, not re-checked by
|
|
|
|
| 4024 |
group_by = payload.get("groupBy")
|
| 4025 |
if group_by is not None and not isinstance(group_by, str):
|
| 4026 |
raise err(400, "bad_selection", "groupBy must be a column id")
|
| 4027 |
+
group_offset, group_limit = payload.get("groupOffset", 0), payload.get("groupLimit")
|
| 4028 |
+
if type(group_offset) is not int or group_offset < 0:
|
| 4029 |
+
raise err(400, "bad_selection", "groupOffset must be a nonnegative integer")
|
| 4030 |
+
if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
|
| 4031 |
+
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
|
| 4032 |
+
if group_offset and group_limit is None:
|
| 4033 |
+
raise err(400, "bad_selection", "Send groupLimit when paging groups")
|
| 4034 |
+
|
| 4035 |
+
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4036 |
+
require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 4037 |
|
| 4038 |
if _materialised_table(table_key):
|
| 4039 |
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
|
| 4040 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 4041 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 4042 |
+
limits=read["limits"], group_offset=group_offset,
|
| 4043 |
+
group_limit=group_limit)
|
| 4044 |
return {"ok": True, **out, "scope": {"table": table_key, "readThrough": True}}
|
| 4045 |
|
| 4046 |
# ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3).
|
|
|
|
| 4052 |
# come from here) no longer re-reads every definition while `user_tables` is unchanged.
|
| 4053 |
read = _compiled_read(table_key, session, tree=query["filters"],
|
| 4054 |
filterConj=query["filterConj"], search=query["search"],
|
| 4055 |
+
related=payload.get("related"), st=page[0] if page else _page_defs(session)[0])
|
| 4056 |
read = _restrict_row_ids(read, ids)
|
| 4057 |
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
|
| 4058 |
read["params"], read["frm"], aggs, group_by=group_by,
|
| 4059 |
+
limits=read["limits"], group_offset=group_offset,
|
| 4060 |
+
group_limit=group_limit)
|
| 4061 |
body_out = {"ok": True, "total_rows": out["total_rows"], "aggregates": out["aggregates"],
|
| 4062 |
"limits": out["limits"], "scope": {"table": table_key, "readThrough": True}}
|
| 4063 |
if out["groups"] is not None:
|
| 4064 |
body_out["groups"] = out["groups"]
|
| 4065 |
+
body_out.update({k: out[k] for k in ("total_groups", "group_offset", "group_limit")})
|
| 4066 |
return body_out
|
| 4067 |
|
| 4068 |
|
api/routes_products.py
CHANGED
|
@@ -109,7 +109,7 @@ def _pool_for(rt, team_id):
|
|
| 109 |
return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build)
|
| 110 |
|
| 111 |
|
| 112 |
-
def scoped_pool(session: Session):
|
| 113 |
"""`(pids, team_id, rows_src, fields_base)` — THE PRODUCT WALL, on its own.
|
| 114 |
|
| 115 |
Extracted from `product_assembly` (wave 19, item 12) so a caller that needs only "which
|
|
@@ -124,7 +124,7 @@ def scoped_pool(session: Session):
|
|
| 124 |
|
| 125 |
team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
|
| 126 |
try:
|
| 127 |
-
rows_src = _pool_for(session.runtime, team_id)
|
| 128 |
except Exception as e:
|
| 129 |
raise err(503, "pool_unavailable",
|
| 130 |
f"the product catalogue could not be built; {str(e)[:160]}")
|
|
@@ -132,7 +132,8 @@ def scoped_pool(session: Session):
|
|
| 132 |
# the field wall runs so every reader receives the same metadata and the
|
| 133 |
# Leadership account owns the permission editor from the first request.
|
| 134 |
import modules.product_data as pd
|
| 135 |
-
|
|
|
|
| 136 |
fields_base = pd_fields(consolidated=team_id is None, st=session.runtime)
|
| 137 |
# The SAME wall the customer assembly applies, in the same order: rows first (before pids
|
| 138 |
# are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
|
|
@@ -148,9 +149,10 @@ def scoped_pool(session: Session):
|
|
| 148 |
fields_base = list(fields_base) + [
|
| 149 |
field for field in enabled_raw if field.get("key") not in field_keys
|
| 150 |
]
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
|
|
|
| 154 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
| 155 |
return pids, team_id, rows_src, fields_base
|
| 156 |
|
|
@@ -397,17 +399,19 @@ def _project_enabled_raw_rows(rows, enabled, *, drop_source=False):
|
|
| 397 |
|
| 398 |
|
| 399 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 400 |
-
consume_corrections: bool = False):
|
| 401 |
from routes_customers import grid_assembly
|
| 402 |
from core import perm_scope
|
| 403 |
team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
|
| 404 |
return grid_assembly(session, scope, storage_key,
|
| 405 |
-
build=lambda borrowed: _compute_product_assembly(
|
| 406 |
-
|
|
|
|
|
|
|
| 407 |
|
| 408 |
|
| 409 |
def _compute_product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 410 |
-
consume_corrections: bool = False):
|
| 411 |
"""The product topic's mirror of `routes_customers.grid_assembly` — SAME g-dict keys, so
|
| 412 |
the /workspace and events routes consume either interchangeably.
|
| 413 |
|
|
@@ -437,7 +441,7 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
|
|
| 437 |
import aios_grid
|
| 438 |
import core.perm_scope as perm_scope
|
| 439 |
import modules.product_data as pd
|
| 440 |
-
pids, team_id, rows_src, fields_base = scoped_pool(session)
|
| 441 |
shared_defs = pd.shared_fields(st=session.runtime)
|
| 442 |
|
| 443 |
ctx = grid_events.EventCtx(
|
|
@@ -457,7 +461,7 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
|
|
| 457 |
# shared value can therefore never leak INTO the per-user bucket by way of this merge.
|
| 458 |
# ⚠ SHARED WINS per key — these are contract columns, so the tenant-wide value is the answer
|
| 459 |
# and any per-user leftover under the same key is stale by construction.
|
| 460 |
-
_shared = pd.shared_cells(pids, st=session.runtime)
|
| 461 |
if _shared:
|
| 462 |
_ov = dict(ws.get("overlays") or {})
|
| 463 |
for _pid, _cells in _shared.items():
|
|
@@ -513,15 +517,17 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
|
|
| 513 |
|
| 514 |
today = time.strftime("%Y-%m-%d")
|
| 515 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
| 516 |
-
stamp = _measure_stamp(session.runtime, team_id)
|
| 517 |
measures = product_measures(session)
|
| 518 |
|
| 519 |
# R9: the Cohorts column's cells, built from THIS topic's lists. ⭐ W37-T10 — the measure
|
| 520 |
# half of this same read-only channel is no longer empty; the customer assembly merges its
|
| 521 |
# measure columns here for exactly the same reason and through the same door.
|
| 522 |
-
derived = aios_grid.cohort_cells(lists)
|
| 523 |
-
|
| 524 |
-
|
|
|
|
|
|
|
| 525 |
derived.setdefault(pid, {}).update(cells)
|
| 526 |
|
| 527 |
# ⛔ THE CONDITION CHANNEL SHIPS WITH THE OFFER, NOT AFTER IT. `routes_grid` derives
|
|
@@ -529,14 +535,16 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
|
|
| 529 |
# filter builder offers measure conditions — and an id missing from `measureSets` renders as
|
| 530 |
# PENDING ("Calculating…") and matches nothing, permanently. Serving the offer without this
|
| 531 |
# would trade one honest absence for a spinner that never resolves.
|
| 532 |
-
measure_sets = _measure_condition_sets(views, measures, rows_src, team_id, today)
|
|
|
|
| 533 |
|
| 534 |
from core import user_tables as _zones
|
| 535 |
fields = _zones.with_zone_field("product_data", fields, st=session.runtime)
|
| 536 |
# Existing supplier-master defaults and imported overlay cells predate W43's write-time
|
| 537 |
# choice hydration. Build their effective, already-walled row values before returning the
|
| 538 |
# field contract so a populated select never opens as "no choices yet".
|
| 539 |
-
|
|
|
|
| 540 |
# ⭐⭐ D-590 (wave 46 hotfix) — **THE PER-TENANT SCHEMA OVERLAY, APPLIED TO THE GRID READ.**
|
| 541 |
# ⛔ THIS SURFACE IS THE ONE W46-T40 FAILED ON, and the staged spec named only the customer and
|
| 542 |
# user-table assemblies because lane E traced `routes_grid.py::workspace` alone. T40's evidence
|
|
|
|
| 109 |
return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build)
|
| 110 |
|
| 111 |
|
| 112 |
+
def scoped_pool(session: Session, *, with_rows: bool = True):
|
| 113 |
"""`(pids, team_id, rows_src, fields_base)` — THE PRODUCT WALL, on its own.
|
| 114 |
|
| 115 |
Extracted from `product_assembly` (wave 19, item 12) so a caller that needs only "which
|
|
|
|
| 124 |
|
| 125 |
team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
|
| 126 |
try:
|
| 127 |
+
rows_src = _pool_for(session.runtime, team_id) if with_rows else []
|
| 128 |
except Exception as e:
|
| 129 |
raise err(503, "pool_unavailable",
|
| 130 |
f"the product catalogue could not be built; {str(e)[:160]}")
|
|
|
|
| 132 |
# the field wall runs so every reader receives the same metadata and the
|
| 133 |
# Leadership account owns the permission editor from the first request.
|
| 134 |
import modules.product_data as pd
|
| 135 |
+
if with_rows:
|
| 136 |
+
pd.image_field_definition(st=session.runtime)
|
| 137 |
fields_base = pd_fields(consolidated=team_id is None, st=session.runtime)
|
| 138 |
# The SAME wall the customer assembly applies, in the same order: rows first (before pids
|
| 139 |
# are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
|
|
|
|
| 149 |
fields_base = list(fields_base) + [
|
| 150 |
field for field in enabled_raw if field.get("key") not in field_keys
|
| 151 |
]
|
| 152 |
+
if with_rows:
|
| 153 |
+
rows_src = _project_enabled_raw_rows(rows_src, enabled_raw, drop_source=True)
|
| 154 |
+
rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, fields_base,
|
| 155 |
+
st=session.runtime)
|
| 156 |
pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
|
| 157 |
return pids, team_id, rows_src, fields_base
|
| 158 |
|
|
|
|
| 399 |
|
| 400 |
|
| 401 |
def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 402 |
+
consume_corrections: bool = False, *, with_rows: bool = True, needed=None):
|
| 403 |
from routes_customers import grid_assembly
|
| 404 |
from core import perm_scope
|
| 405 |
team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
|
| 406 |
return grid_assembly(session, scope, storage_key,
|
| 407 |
+
build=lambda borrowed: _compute_product_assembly(
|
| 408 |
+
borrowed, scope, storage_key, False, with_rows=with_rows, needed=needed),
|
| 409 |
+
pool_stamp=lambda: _measure_stamp(session.runtime, team_id) if with_rows else 0,
|
| 410 |
+
with_rows=with_rows, needed=needed)
|
| 411 |
|
| 412 |
|
| 413 |
def _compute_product_assembly(session: Session, scope: str = "product", storage_key: str = "",
|
| 414 |
+
consume_corrections: bool = False, *, with_rows: bool = True, needed=None):
|
| 415 |
"""The product topic's mirror of `routes_customers.grid_assembly` — SAME g-dict keys, so
|
| 416 |
the /workspace and events routes consume either interchangeably.
|
| 417 |
|
|
|
|
| 441 |
import aios_grid
|
| 442 |
import core.perm_scope as perm_scope
|
| 443 |
import modules.product_data as pd
|
| 444 |
+
pids, team_id, rows_src, fields_base = scoped_pool(session, with_rows=with_rows)
|
| 445 |
shared_defs = pd.shared_fields(st=session.runtime)
|
| 446 |
|
| 447 |
ctx = grid_events.EventCtx(
|
|
|
|
| 461 |
# shared value can therefore never leak INTO the per-user bucket by way of this merge.
|
| 462 |
# ⚠ SHARED WINS per key — these are contract columns, so the tenant-wide value is the answer
|
| 463 |
# and any per-user leftover under the same key is stale by construction.
|
| 464 |
+
_shared = pd.shared_cells(pids, st=session.runtime) if with_rows else {}
|
| 465 |
if _shared:
|
| 466 |
_ov = dict(ws.get("overlays") or {})
|
| 467 |
for _pid, _cells in _shared.items():
|
|
|
|
| 517 |
|
| 518 |
today = time.strftime("%Y-%m-%d")
|
| 519 |
fields = aios_grid.project_date_recurrences(fields, today=today)
|
| 520 |
+
stamp = _measure_stamp(session.runtime, team_id) if with_rows else 0
|
| 521 |
measures = product_measures(session)
|
| 522 |
|
| 523 |
# R9: the Cohorts column's cells, built from THIS topic's lists. ⭐ W37-T10 — the measure
|
| 524 |
# half of this same read-only channel is no longer empty; the customer assembly merges its
|
| 525 |
# measure columns here for exactly the same reason and through the same door.
|
| 526 |
+
derived = aios_grid.cohort_cells(lists) if with_rows else {}
|
| 527 |
+
materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed]
|
| 528 |
+
for pid, cells in (_measure_cells(materialized_fields, rows_src, team_id, today, stamp,
|
| 529 |
+
session.runtime.measure_memo, measures)
|
| 530 |
+
if with_rows and (needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items():
|
| 531 |
derived.setdefault(pid, {}).update(cells)
|
| 532 |
|
| 533 |
# ⛔ THE CONDITION CHANNEL SHIPS WITH THE OFFER, NOT AFTER IT. `routes_grid` derives
|
|
|
|
| 535 |
# filter builder offers measure conditions — and an id missing from `measureSets` renders as
|
| 536 |
# PENDING ("Calculating…") and matches nothing, permanently. Serving the offer without this
|
| 537 |
# would trade one honest absence for a spinner that never resolves.
|
| 538 |
+
measure_sets = (_measure_condition_sets(views, measures, rows_src, team_id, today)
|
| 539 |
+
if with_rows and needed is None else {})
|
| 540 |
|
| 541 |
from core import user_tables as _zones
|
| 542 |
fields = _zones.with_zone_field("product_data", fields, st=session.runtime)
|
| 543 |
# Existing supplier-master defaults and imported overlay cells predate W43's write-time
|
| 544 |
# choice hydration. Build their effective, already-walled row values before returning the
|
| 545 |
# field contract so a populated select never opens as "no choices yet".
|
| 546 |
+
if with_rows:
|
| 547 |
+
fields = _hydrate_effective_choice_options(fields, rows_src, ws.get("overlays"), derived)
|
| 548 |
# ⭐⭐ D-590 (wave 46 hotfix) — **THE PER-TENANT SCHEMA OVERLAY, APPLIED TO THE GRID READ.**
|
| 549 |
# ⛔ THIS SURFACE IS THE ONE W46-T40 FAILED ON, and the staged spec named only the customer and
|
| 550 |
# user-table assemblies because lane E traced `routes_grid.py::workspace` alone. T40's evidence
|
api/routes_sheet.py
CHANGED
|
@@ -76,7 +76,7 @@ def _require_database(session: Session, database) -> str:
|
|
| 76 |
if not key:
|
| 77 |
raise err(400, "sheet_database_required", "That cell does not name a database.")
|
| 78 |
import core.perm_scope as perm_scope
|
| 79 |
-
if not perm_scope.nav_may_open(session.user, key, st=session
|
| 80 |
raise err(403, "database_unreadable", f"You cannot read {key}.")
|
| 81 |
tcfg = getattr(session.runtime.tenant, "config", None) or {}
|
| 82 |
tmods = tcfg.get("modules", "all")
|
|
@@ -96,11 +96,19 @@ def _refuse_unapplied_audience(session: Session, database) -> None:
|
|
| 96 |
declared (`audience_declared`), so this fails closed.
|
| 97 |
"""
|
| 98 |
import core.perm_scope as perm_scope
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
if perm_scope.audience_declared(database, st=session.runtime):
|
| 100 |
raise err(409, "scope_not_applicable",
|
| 101 |
f"{database} has an audience rule that limits which records each person may "
|
| 102 |
"see, and a Sheet binding cannot apply that rule yet. Open the database from "
|
| 103 |
"the navigation, where the rule is applied, or ask an administrator.")
|
|
|
|
|
|
|
| 104 |
|
| 105 |
|
| 106 |
def _mirror(session: Session):
|
|
@@ -326,7 +334,7 @@ def _amazon_stored_spec(cur, table_key):
|
|
| 326 |
return spec
|
| 327 |
|
| 328 |
|
| 329 |
-
def _registry_assembly(session: Session, table_key: str):
|
| 330 |
"""The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
|
| 331 |
|
| 332 |
⭐⭐ W53-T13 — THIS IS THE FIX, AND IT IS A SECOND CALLER OF AN ESTABLISHED PATH RATHER THAN A
|
|
@@ -366,22 +374,27 @@ def _registry_assembly(session: Session, table_key: str):
|
|
| 366 |
if memo is None:
|
| 367 |
memo = {}
|
| 368 |
session._sheet_assemblies = memo
|
| 369 |
-
if
|
| 370 |
-
|
|
|
|
|
|
|
|
|
|
| 371 |
try:
|
| 372 |
# W62 C2: a placed ut_* view needs its own field vocabulary, without fetching its rows.
|
| 373 |
if table_key.startswith("ut_"):
|
| 374 |
from routes_tables import ut_assembly
|
| 375 |
-
memo[
|
| 376 |
-
|
|
|
|
| 377 |
if table_key == "product_data":
|
| 378 |
from routes_products import product_assembly
|
| 379 |
-
memo[
|
| 380 |
-
|
| 381 |
-
return memo[
|
| 382 |
from routes_customers import grid_assembly
|
| 383 |
-
memo[
|
| 384 |
-
|
|
|
|
| 385 |
except perm_scope.Denied:
|
| 386 |
raise err(403, "database_unreadable", f"You cannot read {table_key}.")
|
| 387 |
except HTTPException:
|
|
@@ -401,6 +414,12 @@ def _registry_assembly(session: Session, table_key: str):
|
|
| 401 |
"shown rather than shown from an older set of columns. Try again in a moment.")
|
| 402 |
|
| 403 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 404 |
def _related_declarations(session, database):
|
| 405 |
"""The same permission-scoped catalogue for reads, rewrites and the picker.
|
| 406 |
|
|
@@ -416,12 +435,14 @@ def _related_declarations(session, database):
|
|
| 416 |
from core import perm_scope, relations, user_tables
|
| 417 |
from routes_grid import _source_fields
|
| 418 |
|
| 419 |
-
|
|
|
|
|
|
|
| 420 |
defs = {}
|
| 421 |
-
for key, definition in (user_tables.all_defs(st=
|
| 422 |
-
if perm_scope.nav_may_open(session.user, key, st=
|
| 423 |
declared = definition.get("fields") or []
|
| 424 |
-
hidden = perm_scope.hidden_keys(session.user, key, declared, st=
|
| 425 |
defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
|
| 426 |
entries = relations.related(database, fields=fields, defs=defs)
|
| 427 |
# S66-T13 / C2: `relations.bridge_entries` lists a bridge only from its SOURCE, so
|
|
@@ -457,12 +478,19 @@ def _legacy_linked_keys(database, entry):
|
|
| 457 |
|
| 458 |
def _related_condition_fields(session, database):
|
| 459 |
from core import perm_scope
|
|
|
|
|
|
|
|
|
|
|
|
|
| 460 |
_require_database(session, database)
|
| 461 |
-
fields = list(_registry_assembly(session, database).get("fields") or [])
|
| 462 |
-
hidden = perm_scope.hidden_keys(session.user, database, fields, st=session
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
|
|
|
|
|
|
|
|
|
| 466 |
|
| 467 |
|
| 468 |
def _rewrite_related_filters(session, database, nodes):
|
|
@@ -496,8 +524,15 @@ def _rewrite_related_filters(session, database, nodes):
|
|
| 496 |
def _related_filter_database_label(session, defs, target):
|
| 497 |
from core import registry
|
| 498 |
from routes_nav import _read_nav_meta
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 499 |
topic = registry.BY_KEY.get(target) or {}
|
| 500 |
-
return (
|
| 501 |
or defs.get(target, {}).get("label") or topic.get("label") or target)
|
| 502 |
|
| 503 |
|
|
@@ -520,6 +555,156 @@ def _distinct_page(cur, sql, params, offset, limit, search, *, numeric=False):
|
|
| 520 |
"limit": limit, "hasMore": has_more}
|
| 521 |
|
| 522 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 523 |
@router.get("/sheets/distinct/{database}")
|
| 524 |
def distinct_values(database: str, field: str = Query(...),
|
| 525 |
offset: int = Query(0, ge=0), limit: int = Query(200, ge=1, le=1000),
|
|
@@ -575,6 +760,29 @@ def distinct_values(database: str, field: str = Query(...),
|
|
| 575 |
@router.get("/sheets/filter-fields/{database}")
|
| 576 |
def linked_filter_fields(database: str, session: Session = Depends(require_session)):
|
| 577 |
"""Offer ordinary conditions grouped by their readable related database."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 578 |
from core import perm_scope, relations
|
| 579 |
database = _require_database(session, database)
|
| 580 |
_fields, defs, entries = _related_declarations(session, database)
|
|
@@ -596,7 +804,7 @@ def linked_filter_fields(database: str, session: Session = Depends(require_sessi
|
|
| 596 |
if not candidate.get("served"):
|
| 597 |
omit(candidate, "unavailable")
|
| 598 |
continue
|
| 599 |
-
if not perm_scope.nav_may_open(session.user, candidate["target"], st=session
|
| 600 |
omit(candidate, "no_access")
|
| 601 |
continue
|
| 602 |
identity = relations.edge_identity(candidate)
|
|
@@ -655,9 +863,10 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 655 |
be a second answer to that question, from the one door that never asked it.
|
| 656 |
|
| 657 |
⭐⭐ W54-T36 — `needed` IS THE COLUMN SET THIS CELL CAN POSSIBLY READ, AND `None` IS EVERY
|
| 658 |
-
COLUMN.
|
| 659 |
-
|
| 660 |
-
so a refusal can name a column the
|
|
|
|
| 661 |
|
| 662 |
⛔⛔ IT IS INTERSECTED WITH THE ASSEMBLY'S OWN FIELDS AND MUST STAY THAT WAY. `cols` is what
|
| 663 |
`_resolve_binding` subtracts a row wall's columns from to decide whether the wall can be
|
|
@@ -670,11 +879,15 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 670 |
import routes_odoo_tables as rot
|
| 671 |
from modules import sheet as sheet_mod
|
| 672 |
|
| 673 |
-
g = _registry_assembly(session, table_key)
|
| 674 |
fields = list(g.get("fields") or [])
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 678 |
if table_key == "customer_data" and (needed is None or "agent_id" in needed):
|
| 679 |
from core import perm_scope
|
| 680 |
agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
|
|
@@ -1144,7 +1357,7 @@ def _role_on_workbook(session: Session, book):
|
|
| 1144 |
|
| 1145 |
|
| 1146 |
def _require_reader(session: Session, book):
|
| 1147 |
-
"""
|
| 1148 |
from core import sheets
|
| 1149 |
if not sheets.may_read_workbook(book, _role_on_workbook(session, book)):
|
| 1150 |
raise _refuse(sheets.SheetRefusal("workbook_missing", "That workbook no longer exists."))
|
|
@@ -1377,7 +1590,9 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False):
|
|
| 1377 |
try:
|
| 1378 |
cache = sheet_binding.read_refresh_cache(_store(session), book["id"])
|
| 1379 |
except Exception:
|
| 1380 |
-
|
|
|
|
|
|
|
| 1381 |
seen = {}
|
| 1382 |
today = _sheet_today(session)
|
| 1383 |
changed_values = False
|
|
@@ -1414,11 +1629,12 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False):
|
|
| 1414 |
# new cache from the gateway instead was tried and REJECTED: it makes
|
| 1415 |
# `refreshCells` non-empty, which retires the pre-C2 passthrough below for every
|
| 1416 |
# OTHER cell and blanked a bound cell that had a perfectly good stored value.
|
| 1417 |
-
|
| 1418 |
-
|
| 1419 |
-
|
| 1420 |
-
|
| 1421 |
-
|
|
|
|
| 1422 |
legacy = None
|
| 1423 |
if legacy is not None:
|
| 1424 |
value = legacy.get("value")
|
|
@@ -1486,8 +1702,8 @@ def _response_workbook(book):
|
|
| 1486 |
def _read_for(session: Session, book):
|
| 1487 |
"""Project saved results under the current access wall without resolving any binding.
|
| 1488 |
|
| 1489 |
-
|
| 1490 |
-
|
| 1491 |
"""
|
| 1492 |
book = _response_workbook(book)
|
| 1493 |
return _reresolved_for_reader(session, book)
|
|
@@ -1552,10 +1768,9 @@ def create_sheet_from_template(key: str, body: dict = Body(default=None),
|
|
| 1552 |
# ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the
|
| 1553 |
# Sheet is granted to everyone, and the rail's folders name no database.
|
| 1554 |
#
|
| 1555 |
-
#
|
| 1556 |
-
#
|
| 1557 |
-
#
|
| 1558 |
-
# documents retain legacy visibility. Admitted non-owners still re-resolve under their own scope.
|
| 1559 |
|
| 1560 |
|
| 1561 |
@router.get("/sheets/index")
|
|
@@ -1582,12 +1797,10 @@ def write_sheet_index(body: dict = Body(default=None),
|
|
| 1582 |
payload, and the rail says so in a sentence rather than letting a workbook reappear at the
|
| 1583 |
top level with no explanation (R16). Dropping this key is what makes a silent repair.
|
| 1584 |
|
| 1585 |
-
⭐⭐ D-639 — `rev`
|
| 1586 |
-
|
| 1587 |
answers 409 `index_revision_stale` carrying `rev_current`, which `_refuse`'s existing
|
| 1588 |
-
`_stale` branch shapes for free.
|
| 1589 |
-
wins, exactly as it does on the workbook PUT: a client that has never read the rail cannot
|
| 1590 |
-
send one.
|
| 1591 |
"""
|
| 1592 |
from core import sheets
|
| 1593 |
raw = dict(body or {})
|
|
@@ -1615,6 +1828,11 @@ def write_sheet_index(body: dict = Body(default=None),
|
|
| 1615 |
"That workbook no longer exists.")
|
| 1616 |
out = sheets.merge_placements(_store(session), raw["placements"])
|
| 1617 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1618 |
folders = raw.get("folders")
|
| 1619 |
item_folders = raw.get("itemFolders")
|
| 1620 |
if not session.admin:
|
|
@@ -1680,7 +1898,7 @@ def write_sheet_index(body: dict = Body(default=None),
|
|
| 1680 |
folders = incoming
|
| 1681 |
out = sheets.save_folders(_store(session), folders,
|
| 1682 |
item_folders, starred=raw.get("starred"),
|
| 1683 |
-
expect_rev=
|
| 1684 |
writable_ids=None if session.admin else visible_ids,
|
| 1685 |
writable_folder_ids=(None if session.admin
|
| 1686 |
else visible_folder_ids))
|
|
@@ -1816,14 +2034,17 @@ def _import_sentence(report):
|
|
| 1816 |
"""What a finished import SAYS, in digits (R16). No em dash and no en dash."""
|
| 1817 |
compared = int(report.get("compared") or 0)
|
| 1818 |
matched = int(report.get("matched") or 0)
|
| 1819 |
-
|
| 1820 |
-
|
|
|
|
|
|
|
| 1821 |
if compared:
|
| 1822 |
-
line += ("
|
| 1823 |
-
|
| 1824 |
if report.get("unsupported"):
|
| 1825 |
-
|
| 1826 |
-
|
|
|
|
| 1827 |
return line
|
| 1828 |
|
| 1829 |
|
|
@@ -1832,7 +2053,7 @@ async def start_sheet_import(file: UploadFile = File(...),
|
|
| 1832 |
session: Session = Depends(require_session)):
|
| 1833 |
"""Take an .xlsx upload and START a job (contract C4). Answers `{job, state, progress}`.
|
| 1834 |
|
| 1835 |
-
⚠
|
| 1836 |
are both knowable before any work starts, so answering them as a job the caller has to poll
|
| 1837 |
would make the client wait to be told something the server already knew.
|
| 1838 |
"""
|
|
@@ -1842,6 +2063,13 @@ async def start_sheet_import(file: UploadFile = File(...),
|
|
| 1842 |
if not name.lower().endswith(".xlsx"):
|
| 1843 |
raise err(400, "import_not_xlsx",
|
| 1844 |
"Only an .xlsx workbook can be imported. Save your file as .xlsx and try again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1845 |
# Read at most one byte past the cap. UploadFile may be backed by a large temporary file; an
|
| 1846 |
# unbounded read would allocate hostile input merely to discover that it is too large.
|
| 1847 |
data = await file.read(sheet_import.MAX_IMPORT_BYTES + 1)
|
|
@@ -2327,11 +2555,10 @@ def share_sheet(workbook_id: str, body: dict = Body(default=None),
|
|
| 2327 |
rec = shares.set_grants("workbook", book["id"], entries,
|
| 2328 |
owner=(None if held.get("owner") else _workbook_owner(book)),
|
| 2329 |
granter=session.uname, st=_store(session), expect=prior)
|
| 2330 |
-
#
|
| 2331 |
if "access" not in book:
|
| 2332 |
-
#
|
| 2333 |
-
#
|
| 2334 |
-
# and lock themselves out by sharing it.
|
| 2335 |
sheets.narrow_workbook(_store(session), book["id"])
|
| 2336 |
except sheets.SheetRefusal as exc:
|
| 2337 |
raise _refuse(exc)
|
|
@@ -2803,8 +3030,14 @@ def _forget_sheet_database(session, database):
|
|
| 2803 |
getattr(session, "_sheet_assemblies", {}).pop(database, None)
|
| 2804 |
memo = getattr(session, "_sheet_read_memo", None)
|
| 2805 |
if isinstance(memo, dict):
|
| 2806 |
-
for slot in [slot for slot in memo if isinstance(slot, tuple)
|
|
|
|
| 2807 |
memo.pop(slot, None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2808 |
mirrors = getattr(session, "_sheet_mirrors", {})
|
| 2809 |
for slot in list(mirrors):
|
| 2810 |
if slot[0] == database:
|
|
@@ -2944,10 +3177,14 @@ def _sheet_request_scope(session):
|
|
| 2944 |
reads of the same document object (the Daily Sales repair projects in rounds and does not).
|
| 2945 |
"""
|
| 2946 |
previous = getattr(session, "_sheet_read_memo", None)
|
|
|
|
|
|
|
| 2947 |
memo = {"today": _sheet_today(session),
|
| 2948 |
"stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
|
| 2949 |
try:
|
| 2950 |
session._sheet_read_memo = memo
|
|
|
|
|
|
|
| 2951 |
except Exception: # noqa: BLE001
|
| 2952 |
yield None
|
| 2953 |
return
|
|
@@ -2955,6 +3192,8 @@ def _sheet_request_scope(session):
|
|
| 2955 |
yield memo
|
| 2956 |
finally:
|
| 2957 |
session._sheet_read_memo = previous
|
|
|
|
|
|
|
| 2958 |
|
| 2959 |
|
| 2960 |
def _request_scoped(route):
|
|
@@ -3318,7 +3557,7 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
|
|
| 3318 |
cell = cells.get(key)
|
| 3319 |
if not any(fr.a1_key("A1", name).rpartition("!")[0] == key.rpartition("!")[0]
|
| 3320 |
for name in names):
|
| 3321 |
-
refuse("the
|
| 3322 |
if isinstance(cell, dict) and cell.get("state") in ("refused", "stale") \
|
| 3323 |
and cell.get("kind") != "formula" and cell.get("kind") != "bound":
|
| 3324 |
refuse((cell.get("refusal") or {}).get("message") or
|
|
@@ -3456,7 +3695,7 @@ def _binding_related_entry(session, database, name):
|
|
| 3456 |
entry = relations.pick(entries, via)
|
| 3457 |
if entry is None:
|
| 3458 |
candidates = [item["key"] for item in relations.via_candidates(entries, via)
|
| 3459 |
-
if perm_scope.nav_may_open(session.user, item["target"], st=session
|
| 3460 |
offered = (" Available relationships: " + ", ".join(candidates) + ".") if candidates else ""
|
| 3461 |
# Write our own sentence: the shared pin refusal does not name via.key.
|
| 3462 |
raise err(409, "unknown_relation",
|
|
@@ -3596,7 +3835,7 @@ def _binding_filter_nodes(session, database, binding, *, memo=None):
|
|
| 3596 |
|
| 3597 |
|
| 3598 |
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
|
| 3599 |
-
book=None, tab=None, ending_seen=()):
|
| 3600 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
| 3601 |
|
| 3602 |
The order below is the contract, not a preference:
|
|
@@ -3652,10 +3891,26 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 3652 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 3653 |
if binding["aggregate"].get("field"):
|
| 3654 |
needed.add(str(binding["aggregate"]["field"]))
|
|
|
|
|
|
|
| 3655 |
|
| 3656 |
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
| 3657 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 3658 |
cols = dict(cols)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3659 |
|
| 3660 |
# ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
|
| 3661 |
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
|
|
@@ -3718,6 +3973,11 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 3718 |
"live connection, so the number is not being shown rather than shown wider "
|
| 3719 |
"than you asked for. Edit the cell's scope and try again.")
|
| 3720 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3721 |
row = cur.execute(sql, params).fetchone()
|
| 3722 |
value = row[0] if row else None
|
| 3723 |
# ⭐⭐ EVERY WAY THIS ANSWER CAME OUT WIDER THAN THE BINDING ASKED FOR TRAVELS WITH IT
|
|
|
|
| 76 |
if not key:
|
| 77 |
raise err(400, "sheet_database_required", "That cell does not name a database.")
|
| 78 |
import core.perm_scope as perm_scope
|
| 79 |
+
if not perm_scope.nav_may_open(session.user, key, st=_sheet_permission_reader(session)):
|
| 80 |
raise err(403, "database_unreadable", f"You cannot read {key}.")
|
| 81 |
tcfg = getattr(session.runtime.tenant, "config", None) or {}
|
| 82 |
tmods = tcfg.get("modules", "all")
|
|
|
|
| 96 |
declared (`audience_declared`), so this fails closed.
|
| 97 |
"""
|
| 98 |
import core.perm_scope as perm_scope
|
| 99 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 100 |
+
if not isinstance(memo, dict):
|
| 101 |
+
memo = getattr(session, "_sheet_scope_context", None)
|
| 102 |
+
slot = ("audience", database)
|
| 103 |
+
if isinstance(memo, dict) and slot in memo:
|
| 104 |
+
return
|
| 105 |
if perm_scope.audience_declared(database, st=session.runtime):
|
| 106 |
raise err(409, "scope_not_applicable",
|
| 107 |
f"{database} has an audience rule that limits which records each person may "
|
| 108 |
"see, and a Sheet binding cannot apply that rule yet. Open the database from "
|
| 109 |
"the navigation, where the rule is applied, or ask an administrator.")
|
| 110 |
+
if isinstance(memo, dict):
|
| 111 |
+
memo[slot] = True
|
| 112 |
|
| 113 |
|
| 114 |
def _mirror(session: Session):
|
|
|
|
| 334 |
return spec
|
| 335 |
|
| 336 |
|
| 337 |
+
def _registry_assembly(session: Session, table_key: str, *, with_rows: bool = True, needed=None):
|
| 338 |
"""The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
|
| 339 |
|
| 340 |
⭐⭐ W53-T13 — THIS IS THE FIX, AND IT IS A SECOND CALLER OF AN ESTABLISHED PATH RATHER THAN A
|
|
|
|
| 374 |
if memo is None:
|
| 375 |
memo = {}
|
| 376 |
session._sheet_assemblies = memo
|
| 377 |
+
needed = None if needed is None else frozenset(str(key) for key in needed)
|
| 378 |
+
shape = ("full" if needed is None else ("columns", tuple(sorted(needed)))) if with_rows else "fields"
|
| 379 |
+
slot = (table_key, shape)
|
| 380 |
+
if slot in memo:
|
| 381 |
+
return memo[slot]
|
| 382 |
try:
|
| 383 |
# W62 C2: a placed ut_* view needs its own field vocabulary, without fetching its rows.
|
| 384 |
if table_key.startswith("ut_"):
|
| 385 |
from routes_tables import ut_assembly
|
| 386 |
+
memo[slot] = ut_assembly(session, table_key, consume_corrections=False,
|
| 387 |
+
with_rows=False, fields_only=not with_rows, needed=needed)
|
| 388 |
+
return memo[slot]
|
| 389 |
if table_key == "product_data":
|
| 390 |
from routes_products import product_assembly
|
| 391 |
+
memo[slot] = product_assembly(session, scope="product",
|
| 392 |
+
consume_corrections=False, with_rows=with_rows, needed=needed)
|
| 393 |
+
return memo[slot]
|
| 394 |
from routes_customers import grid_assembly
|
| 395 |
+
memo[slot] = grid_assembly(session, scope="customer", consume_corrections=False,
|
| 396 |
+
with_rows=with_rows, needed=needed)
|
| 397 |
+
return memo[slot]
|
| 398 |
except perm_scope.Denied:
|
| 399 |
raise err(403, "database_unreadable", f"You cannot read {table_key}.")
|
| 400 |
except HTTPException:
|
|
|
|
| 414 |
"shown rather than shown from an older set of columns. Try again in a moment.")
|
| 415 |
|
| 416 |
|
| 417 |
+
def _sheet_permission_reader(session):
|
| 418 |
+
"""Lend a captured schema/grant snapshot only inside this request's offer pass."""
|
| 419 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 420 |
+
return memo.get("permission_reader", session.runtime) if isinstance(memo, dict) else session.runtime
|
| 421 |
+
|
| 422 |
+
|
| 423 |
def _related_declarations(session, database):
|
| 424 |
"""The same permission-scoped catalogue for reads, rewrites and the picker.
|
| 425 |
|
|
|
|
| 435 |
from core import perm_scope, relations, user_tables
|
| 436 |
from routes_grid import _source_fields
|
| 437 |
|
| 438 |
+
reader = _sheet_permission_reader(session)
|
| 439 |
+
fields = (_source_fields(session, database) if reader is session.runtime else
|
| 440 |
+
_source_fields(session, database, st=reader))
|
| 441 |
defs = {}
|
| 442 |
+
for key, definition in (user_tables.all_defs(st=reader) or {}).items():
|
| 443 |
+
if perm_scope.nav_may_open(session.user, key, st=reader):
|
| 444 |
declared = definition.get("fields") or []
|
| 445 |
+
hidden = perm_scope.hidden_keys(session.user, key, declared, st=reader)
|
| 446 |
defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
|
| 447 |
entries = relations.related(database, fields=fields, defs=defs)
|
| 448 |
# S66-T13 / C2: `relations.bridge_entries` lists a bridge only from its SOURCE, so
|
|
|
|
| 478 |
|
| 479 |
def _related_condition_fields(session, database):
|
| 480 |
from core import perm_scope
|
| 481 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 482 |
+
slot = ("condition_fields", database)
|
| 483 |
+
if isinstance(memo, dict) and slot in memo:
|
| 484 |
+
return memo[slot]
|
| 485 |
_require_database(session, database)
|
| 486 |
+
fields = list(_registry_assembly(session, database, with_rows=False).get("fields") or [])
|
| 487 |
+
hidden = perm_scope.hidden_keys(session.user, database, fields, st=_sheet_permission_reader(session))
|
| 488 |
+
answer = [field for field in fields if field.get("key") not in hidden
|
| 489 |
+
and field.get("type") not in {"formula", "link", "lookup", "rollup", "metric", "measure"}
|
| 490 |
+
and not field.get("measure")]
|
| 491 |
+
if isinstance(memo, dict):
|
| 492 |
+
memo[slot] = answer
|
| 493 |
+
return answer
|
| 494 |
|
| 495 |
|
| 496 |
def _rewrite_related_filters(session, database, nodes):
|
|
|
|
| 524 |
def _related_filter_database_label(session, defs, target):
|
| 525 |
from core import registry
|
| 526 |
from routes_nav import _read_nav_meta
|
| 527 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 528 |
+
if isinstance(memo, dict):
|
| 529 |
+
if "nav_meta" not in memo:
|
| 530 |
+
memo["nav_meta"] = _read_nav_meta(session.runtime)
|
| 531 |
+
metadata = memo["nav_meta"]
|
| 532 |
+
else:
|
| 533 |
+
metadata = _read_nav_meta(session.runtime)
|
| 534 |
topic = registry.BY_KEY.get(target) or {}
|
| 535 |
+
return (metadata.get(target, {}).get("name")
|
| 536 |
or defs.get(target, {}).get("label") or topic.get("label") or target)
|
| 537 |
|
| 538 |
|
|
|
|
| 555 |
"limit": limit, "hasMore": has_more}
|
| 556 |
|
| 557 |
|
| 558 |
+
def _pivot_fields(pivot):
|
| 559 |
+
from modules import sheet as sheet_mod
|
| 560 |
+
return (set(pivot.get("rows", [])) | set(pivot.get("columns", []))
|
| 561 |
+
| {value["field"] for value in pivot["values"]}
|
| 562 |
+
| sheet_mod.filter_columns_named(pivot.get("filters", [])))
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
def _pivot_unsupported():
|
| 566 |
+
"""Fields the workspace schema offers that no mirror column backs, keyed to their label.
|
| 567 |
+
|
| 568 |
+
Today that is the routing Zone overlay `user_tables.with_zone_field` appends."""
|
| 569 |
+
from core import user_tables
|
| 570 |
+
return {user_tables.ZONE_FIELD_KEY: "Zone"}
|
| 571 |
+
|
| 572 |
+
|
| 573 |
+
def _check_pivot_fields(session, pivot, spec, columns, *, filter_fields=None):
|
| 574 |
+
"""Reject every unreadable axis, value or filter, including COUNT's named field."""
|
| 575 |
+
from core import perm_scope
|
| 576 |
+
from modules import sheet as sheet_mod
|
| 577 |
+
fields = spec.get("fields") or []
|
| 578 |
+
named = set(pivot.get("rows", [])) | set(pivot.get("columns", []))
|
| 579 |
+
named.update(value["field"] for value in pivot["values"])
|
| 580 |
+
named.update(sheet_mod.filter_columns_named(pivot.get("filters", []))
|
| 581 |
+
if filter_fields is None else filter_fields)
|
| 582 |
+
hidden = perm_scope.hidden_keys(session.user, pivot["database"], fields, st=session.runtime)
|
| 583 |
+
unsupported = _pivot_unsupported()
|
| 584 |
+
for field in sorted(named):
|
| 585 |
+
if field not in hidden and field in unsupported \
|
| 586 |
+
and (field not in columns or not columns[field].get("sql")):
|
| 587 |
+
# Readable but derived outside the mirror: a named limit, never a permission refusal,
|
| 588 |
+
# which the pivot client reads as lost access and clears every offered field.
|
| 589 |
+
raise err(409, "pivot_field_unsupported",
|
| 590 |
+
f"{unsupported[field]} cannot be used in a pivot yet. Choose another field.")
|
| 591 |
+
if field in hidden or field not in columns or not columns[field].get("sql"):
|
| 592 |
+
raise err(403, "field_unreadable",
|
| 593 |
+
f"You cannot read field {field} in {pivot['database']}.")
|
| 594 |
+
definitions = {field["key"]: field for field in fields}
|
| 595 |
+
for field in set(pivot.get("rows", [])) | set(pivot.get("columns", [])):
|
| 596 |
+
if (definitions.get(field) or {}).get("type") == "formula":
|
| 597 |
+
refusal = sheet_mod.aggregate_refusal({"fn": "sum"}, definitions[field])
|
| 598 |
+
raise err(409, refusal.code, refusal.message)
|
| 599 |
+
for value in pivot["values"]:
|
| 600 |
+
# C6 cells are numbers or null. MIN/MAX over text would violate that wire contract.
|
| 601 |
+
fn = "sum" if value["agg"] in ("min", "max") else value["agg"]
|
| 602 |
+
refusal = sheet_mod.aggregate_refusal({"fn": fn}, definitions.get(value["field"]))
|
| 603 |
+
if refusal is not None:
|
| 604 |
+
raise err(409, refusal.code, refusal.message)
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
#: One pivot's dense cell matrix, which the tab paints as one table. Customers by month (about
|
| 608 |
+
#: 148,000 cells for tenant #0) fits; a 1.9 million cell pivot never painted in the browser, and
|
| 609 |
+
#: row ids by order numbers asked for billions in one process.
|
| 610 |
+
_PIVOT_MAX_CELLS = 250_000
|
| 611 |
+
|
| 612 |
+
|
| 613 |
+
def _pivot_result(cur, spec, columns, pivot, count_sql, params, limits):
|
| 614 |
+
"""Group the whole scoped source on the server, with no row or group-page truncation.
|
| 615 |
+
|
| 616 |
+
Axes are arrays of field-value tuples in the requested field order. With multiple values,
|
| 617 |
+
each column tuple ends in {field, agg}, in authored value order. Missing intersections are
|
| 618 |
+
null. With no fields on an axis its tuple is [], including an empty population's total.
|
| 619 |
+
"""
|
| 620 |
+
from modules import sheet as sheet_mod
|
| 621 |
+
import routes_odoo_tables as rot
|
| 622 |
+
|
| 623 |
+
row_fields, column_fields = pivot.get("rows", []), pivot.get("columns", [])
|
| 624 |
+
groups = [rot._fs().group_sql(columns[field]) for field in [*row_fields, *column_fields]]
|
| 625 |
+
expressions = []
|
| 626 |
+
for value in pivot["values"]:
|
| 627 |
+
field = value["field"]
|
| 628 |
+
# Native payloads can store numeric fields as JSON text. Cast the trusted source
|
| 629 |
+
# expression, as the Data aggregate compiler does, without rounding or filling nulls.
|
| 630 |
+
numeric = {**columns[field], "sql": f"TRY_CAST({columns[field]['sql']} AS DOUBLE)"}
|
| 631 |
+
expressions.append(sheet_mod.aggregate_sql(
|
| 632 |
+
spec, {"fn": value["agg"], "field": field}, {**columns, field: numeric})[0])
|
| 633 |
+
total_rows = int(cur.execute(count_sql, params).fetchone()[0])
|
| 634 |
+
tail = count_sql[len("SELECT COUNT(*)"):]
|
| 635 |
+
sql = "SELECT " + ", ".join([*groups, *expressions]) + tail
|
| 636 |
+
if groups:
|
| 637 |
+
positions = ", ".join(str(index + 1) for index in range(len(groups)))
|
| 638 |
+
sql += " GROUP BY " + positions + " ORDER BY " + positions
|
| 639 |
+
records = cur.execute(sql, params).fetchall()
|
| 640 |
+
row_keys, column_keys = {}, {}
|
| 641 |
+
if not row_fields:
|
| 642 |
+
row_keys[()] = 0
|
| 643 |
+
if not column_fields:
|
| 644 |
+
column_keys[()] = 0
|
| 645 |
+
values = {}
|
| 646 |
+
for record in records:
|
| 647 |
+
row_key = tuple(rot._group_key(value) for value in record[:len(row_fields)])
|
| 648 |
+
column_key = tuple(rot._group_key(value)
|
| 649 |
+
for value in record[len(row_fields):len(groups)])
|
| 650 |
+
row_keys.setdefault(row_key, len(row_keys))
|
| 651 |
+
column_keys.setdefault(column_key, len(column_keys))
|
| 652 |
+
values[row_key, column_key] = [rot._agg_number(value) for value in record[len(groups):]]
|
| 653 |
+
# The matrix below is dense, so its size is rows x columns x values whatever the data holds.
|
| 654 |
+
# Standing rule 1: a bound that cannot be lifted is reported with its cause and its fix.
|
| 655 |
+
cells_needed = len(row_keys) * len(column_keys) * len(pivot["values"])
|
| 656 |
+
if cells_needed > _PIVOT_MAX_CELLS:
|
| 657 |
+
raise err(409, "pivot_too_large",
|
| 658 |
+
f"This pivot needs {len(row_keys):,} rows by {len(column_keys):,} columns, "
|
| 659 |
+
f"{cells_needed:,} cells, and one pivot holds at most {_PIVOT_MAX_CELLS:,}. "
|
| 660 |
+
"Move a field to Filters, or use a field with fewer distinct values.")
|
| 661 |
+
# SQL orders the entire tuple, but a column tuple may first occur in a later row group.
|
| 662 |
+
# Sort each axis independently using its JSON spelling for deterministic mixed/null keys.
|
| 663 |
+
ordered_rows = sorted(row_keys, key=lambda key: json.dumps(key, ensure_ascii=False))
|
| 664 |
+
ordered_columns = sorted(column_keys, key=lambda key: json.dumps(key, ensure_ascii=False))
|
| 665 |
+
multi = len(pivot["values"]) > 1
|
| 666 |
+
headers = [list(key) + ([{"field": value["field"], "agg": value["agg"]}] if multi else [])
|
| 667 |
+
for key in ordered_columns for value in pivot["values"]]
|
| 668 |
+
cells = [[values.get((row, column), [None] * len(pivot["values"]))[index]
|
| 669 |
+
for column in ordered_columns for index in range(len(pivot["values"]))]
|
| 670 |
+
for row in ordered_rows]
|
| 671 |
+
return {"ok": True, "rows": [list(key) for key in ordered_rows], "columns": headers,
|
| 672 |
+
"cells": cells, "total_rows": total_rows, "limits": list(limits)}
|
| 673 |
+
|
| 674 |
+
|
| 675 |
+
@router.post("/sheets/pivot")
|
| 676 |
+
def pivot_sheet(body: dict = Body(default=None), session: Session = Depends(require_session)):
|
| 677 |
+
"""C6: read a pivot definition without saving a workbook or changing source records."""
|
| 678 |
+
from core import sheets
|
| 679 |
+
import odoo_relational as rel
|
| 680 |
+
import routes_odoo_tables as rot
|
| 681 |
+
try:
|
| 682 |
+
pivot = sheets.clean_pivot(body)
|
| 683 |
+
except sheets.SheetRefusal as exc:
|
| 684 |
+
raise _refuse(exc)
|
| 685 |
+
database = _require_database(session, pivot["database"])
|
| 686 |
+
_refuse_unapplied_audience(session, database)
|
| 687 |
+
# Storage retains authored bytes; access decisions always use the admitted canonical key.
|
| 688 |
+
pivot = {**pivot, "database": database}
|
| 689 |
+
with _sheet_request_scope(session):
|
| 690 |
+
# Native Data databases have no connector mirror. Reuse the Data aggregate door's
|
| 691 |
+
# full, already walled payload and compiler; its temporary cursor closes on exit.
|
| 692 |
+
if database.startswith("ut_") and database != rel.SHIPMENTS_KEY \
|
| 693 |
+
and rot._materialised_table(database):
|
| 694 |
+
query = {"filters": pivot.get("filters", []), "filterConj": "and", "search": ""}
|
| 695 |
+
with rot._materialised_read(database, session, query, None) as read:
|
| 696 |
+
spec = {**read["spec"], "fields": read["payload"]["fields"]}
|
| 697 |
+
_check_pivot_fields(session, pivot, spec, read["cols"])
|
| 698 |
+
count_sql = "SELECT COUNT(*) FROM " + spec["table"]
|
| 699 |
+
if read["where"]:
|
| 700 |
+
count_sql += " WHERE " + read["where"]
|
| 701 |
+
return _pivot_result(read["cur"], spec, read["cols"], pivot, count_sql,
|
| 702 |
+
read["params"], read["limits"])
|
| 703 |
+
binding = {"database": database, "aggregate": {"fn": "count"},
|
| 704 |
+
"scope": {"filter": pivot.get("filters", [])}}
|
| 705 |
+
return _resolve_binding(session, binding, _pivot=pivot)
|
| 706 |
+
|
| 707 |
+
|
| 708 |
@router.get("/sheets/distinct/{database}")
|
| 709 |
def distinct_values(database: str, field: str = Query(...),
|
| 710 |
offset: int = Query(0, ge=0), limit: int = Query(200, ge=1, le=1000),
|
|
|
|
| 760 |
@router.get("/sheets/filter-fields/{database}")
|
| 761 |
def linked_filter_fields(database: str, session: Session = Depends(require_session)):
|
| 762 |
"""Offer ordinary conditions grouped by their readable related database."""
|
| 763 |
+
from copy import copy
|
| 764 |
+
from core import shares, user_tables
|
| 765 |
+
from routes_customers import _AssemblyStore
|
| 766 |
+
borrowed = copy(session)
|
| 767 |
+
borrowed.runtime = _AssemblyStore(session.runtime, schema_only=True)
|
| 768 |
+
with _sheet_request_scope(borrowed) as memo:
|
| 769 |
+
# The permission wall reads definitions and grants only. get_one on the assembly
|
| 770 |
+
# reader would fetch each target's full stored entry, including its rows. The lend
|
| 771 |
+
# preserves the canonical wall while reusing rows-free definitions and one current
|
| 772 |
+
# grant document; it dies with this request and is never used for writes or rows.
|
| 773 |
+
definitions = user_tables.all_defs(st=borrowed.runtime)
|
| 774 |
+
try:
|
| 775 |
+
grants = borrowed.runtime.get(shares.SHARES_KEY, fresh=True) or {}
|
| 776 |
+
except Exception:
|
| 777 |
+
grants = {} # The canonical field/database grant wall fails closed on a read error.
|
| 778 |
+
reader = user_tables.lend(borrowed.runtime, **{
|
| 779 |
+
user_tables.STORE_KEY: definitions, shares.SHARES_KEY: grants})
|
| 780 |
+
if isinstance(memo, dict):
|
| 781 |
+
memo["permission_reader"] = reader
|
| 782 |
+
return _linked_filter_fields(database, borrowed)
|
| 783 |
+
|
| 784 |
+
|
| 785 |
+
def _linked_filter_fields(database, session):
|
| 786 |
from core import perm_scope, relations
|
| 787 |
database = _require_database(session, database)
|
| 788 |
_fields, defs, entries = _related_declarations(session, database)
|
|
|
|
| 804 |
if not candidate.get("served"):
|
| 805 |
omit(candidate, "unavailable")
|
| 806 |
continue
|
| 807 |
+
if not perm_scope.nav_may_open(session.user, candidate["target"], st=_sheet_permission_reader(session)):
|
| 808 |
omit(candidate, "no_access")
|
| 809 |
continue
|
| 810 |
identity = relations.edge_identity(candidate)
|
|
|
|
| 863 |
be a second answer to that question, from the one door that never asked it.
|
| 864 |
|
| 865 |
⭐⭐ W54-T36 — `needed` IS THE COLUMN SET THIS CELL CAN POSSIBLY READ, AND `None` IS EVERY
|
| 866 |
+
COLUMN. The existing assembly keeps its full permitted definitions and row-wall inputs,
|
| 867 |
+
while unused metric cells and view-condition sets are not computed. Rows retain the same
|
| 868 |
+
authorized ids and `spec["fields"]` keeps the full list so a refusal can name a column the
|
| 869 |
+
mirror did not carry. The canonical selection belongs to the existing assembly identity.
|
| 870 |
|
| 871 |
⛔⛔ IT IS INTERSECTED WITH THE ASSEMBLY'S OWN FIELDS AND MUST STAY THAT WAY. `cols` is what
|
| 872 |
`_resolve_binding` subtracts a row wall's columns from to decide whether the wall can be
|
|
|
|
| 879 |
import routes_odoo_tables as rot
|
| 880 |
from modules import sheet as sheet_mod
|
| 881 |
|
| 882 |
+
g = _registry_assembly(session, table_key, needed=needed)
|
| 883 |
fields = list(g.get("fields") or [])
|
| 884 |
+
kept = (fields if needed is None else [f for f in fields if str(f.get("key") or "") in needed])
|
| 885 |
+
metric_input = g if needed is None else {**g, "fields": kept}
|
| 886 |
+
derived, bounded, limits = _metric_derived(session, table_key, metric_input, metric_window)
|
| 887 |
+
# Authorization has already scoped rows using the full wall vocabulary. Restrict only
|
| 888 |
+
# cell materialization; an empty selection is ids-only, not rows_from_pool's all-fields default.
|
| 889 |
+
rows = (aios_grid.rows_from_pool(g["rows_src"], kept, (g.get("ws") or {}).get("overlays"),
|
| 890 |
+
derived=derived) if kept else [{"pid": row["pid"]} for row in g["rows_src"]])
|
| 891 |
if table_key == "customer_data" and (needed is None or "agent_id" in needed):
|
| 892 |
from core import perm_scope
|
| 893 |
agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
|
|
|
|
| 1357 |
|
| 1358 |
|
| 1359 |
def _require_reader(session: Session, book):
|
| 1360 |
+
"""R31 / C7: share the index predicate; hidden and missing ids answer identically."""
|
| 1361 |
from core import sheets
|
| 1362 |
if not sheets.may_read_workbook(book, _role_on_workbook(session, book)):
|
| 1363 |
raise _refuse(sheets.SheetRefusal("workbook_missing", "That workbook no longer exists."))
|
|
|
|
| 1590 |
try:
|
| 1591 |
cache = sheet_binding.read_refresh_cache(_store(session), book["id"])
|
| 1592 |
except Exception:
|
| 1593 |
+
# Preserve the legacy projection fallback on a failed whole read, once per workbook.
|
| 1594 |
+
from core import sheets as _sheets
|
| 1595 |
+
cache = _sheets._read(_store(session), _sheets._scope_cache_key(book["id"]))
|
| 1596 |
seen = {}
|
| 1597 |
today = _sheet_today(session)
|
| 1598 |
changed_values = False
|
|
|
|
| 1629 |
# new cache from the gateway instead was tried and REJECTED: it makes
|
| 1630 |
# `refreshCells` non-empty, which retires the pre-C2 passthrough below for every
|
| 1631 |
# OTHER cell and blanked a bound cell that had a perfectly good stored value.
|
| 1632 |
+
# Both cache formats share this document. The fresh request read above already
|
| 1633 |
+
# holds the legacy slots too; reopening it for every bound cell costs hundreds
|
| 1634 |
+
# of copies/PG reads on Daily Sales without improving permission freshness.
|
| 1635 |
+
legacy_slots = (cache.get("cells") or {}).get(str(tab.get("id")) + "!" + addr)
|
| 1636 |
+
legacy = legacy_slots.get(slot) if isinstance(legacy_slots, dict) else None
|
| 1637 |
+
if not isinstance(legacy, dict):
|
| 1638 |
legacy = None
|
| 1639 |
if legacy is not None:
|
| 1640 |
value = legacy.get("value")
|
|
|
|
| 1702 |
def _read_for(session: Session, book):
|
| 1703 |
"""Project saved results under the current access wall without resolving any binding.
|
| 1704 |
|
| 1705 |
+
R31 / C7 checks visibility before this projection. Admitted non-owners receive a proven
|
| 1706 |
+
scoped snapshot or a request to Refresh.
|
| 1707 |
"""
|
| 1708 |
book = _response_workbook(book)
|
| 1709 |
return _reresolved_for_reader(session, book)
|
|
|
|
| 1768 |
# ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the
|
| 1769 |
# Sheet is granted to everyone, and the rail's folders name no database.
|
| 1770 |
#
|
| 1771 |
+
# R31 / C7: both lists and all three workbook read shapes use core.sheets.may_read_workbook.
|
| 1772 |
+
# Only creators, grantees and administrators may read, including legacy documents. Admitted
|
| 1773 |
+
# non-owners still receive their own scoped results through the existing projection.
|
|
|
|
| 1774 |
|
| 1775 |
|
| 1776 |
@router.get("/sheets/index")
|
|
|
|
| 1797 |
payload, and the rail says so in a sentence rather than letting a workbook reappear at the
|
| 1798 |
top level with no explanation (R16). Dropping this key is what makes a silent repair.
|
| 1799 |
|
| 1800 |
+
⭐⭐ D-639 — `rev` IS REQUIRED FOR A WHOLESALE WRITE. Send the `rev` the GET handed you and
|
| 1801 |
+
a concurrent reorganisation
|
| 1802 |
answers 409 `index_revision_stale` carrying `rev_current`, which `_refuse`'s existing
|
| 1803 |
+
`_stale` branch shapes for free. A placements-only delta keeps its independent merge door.
|
|
|
|
|
|
|
| 1804 |
"""
|
| 1805 |
from core import sheets
|
| 1806 |
raw = dict(body or {})
|
|
|
|
| 1828 |
"That workbook no longer exists.")
|
| 1829 |
out = sheets.merge_placements(_store(session), raw["placements"])
|
| 1830 |
else:
|
| 1831 |
+
revision = raw.get("rev")
|
| 1832 |
+
if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
|
| 1833 |
+
raise sheets.SheetRefusal(
|
| 1834 |
+
"index_revision_required",
|
| 1835 |
+
"A Sheet rail revision is required for this save. Reload the rail and try again.")
|
| 1836 |
folders = raw.get("folders")
|
| 1837 |
item_folders = raw.get("itemFolders")
|
| 1838 |
if not session.admin:
|
|
|
|
| 1898 |
folders = incoming
|
| 1899 |
out = sheets.save_folders(_store(session), folders,
|
| 1900 |
item_folders, starred=raw.get("starred"),
|
| 1901 |
+
expect_rev=revision,
|
| 1902 |
writable_ids=None if session.admin else visible_ids,
|
| 1903 |
writable_folder_ids=(None if session.admin
|
| 1904 |
else visible_folder_ids))
|
|
|
|
| 2034 |
"""What a finished import SAYS, in digits (R16). No em dash and no en dash."""
|
| 2035 |
compared = int(report.get("compared") or 0)
|
| 2036 |
matched = int(report.get("matched") or 0)
|
| 2037 |
+
cells = int(report.get("cells") or 0)
|
| 2038 |
+
formulas = int(report.get("formulas") or 0)
|
| 2039 |
+
line = (f"Imported {cells} cell{'' if cells == 1 else 's'} and "
|
| 2040 |
+
f"{formulas} formula{'' if formulas == 1 else 's'}.")
|
| 2041 |
if compared:
|
| 2042 |
+
line += (f" Matched {matched} of {compared} recalculated cell{'' if compared == 1 else 's'} against the "
|
| 2043 |
+
f"{'answer' if compared == 1 else 'answers'} the file had stored.")
|
| 2044 |
if report.get("unsupported"):
|
| 2045 |
+
kinds = len(report["unsupported"])
|
| 2046 |
+
line += (f" {kinds} kind{'' if kinds == 1 else 's'} of thing "
|
| 2047 |
+
"in that file could not be carried over.")
|
| 2048 |
return line
|
| 2049 |
|
| 2050 |
|
|
|
|
| 2053 |
session: Session = Depends(require_session)):
|
| 2054 |
"""Take an .xlsx upload and START a job (contract C4). Answers `{job, state, progress}`.
|
| 2055 |
|
| 2056 |
+
⚠ UPLOAD REFUSALS ARE SYNCHRONOUS ON PURPOSE. A wrong file type and a file over the cap
|
| 2057 |
are both knowable before any work starts, so answering them as a job the caller has to poll
|
| 2058 |
would make the client wait to be told something the server already knew.
|
| 2059 |
"""
|
|
|
|
| 2063 |
if not name.lower().endswith(".xlsx"):
|
| 2064 |
raise err(400, "import_not_xlsx",
|
| 2065 |
"Only an .xlsx workbook can be imported. Save your file as .xlsx and try again.")
|
| 2066 |
+
declared_size = getattr(file, "size", None)
|
| 2067 |
+
if isinstance(declared_size, int) and declared_size > sheet_import.MAX_IMPORT_BYTES:
|
| 2068 |
+
raise HTTPException(status_code=413, detail={"error": {
|
| 2069 |
+
"code": "import_too_large",
|
| 2070 |
+
"message": ("That file is " + _mb(declared_size) + " and an import takes at most "
|
| 2071 |
+
+ _mb(sheet_import.MAX_IMPORT_BYTES)
|
| 2072 |
+
+ ", so it cannot be read. Split it into two workbooks and try again.")}})
|
| 2073 |
# Read at most one byte past the cap. UploadFile may be backed by a large temporary file; an
|
| 2074 |
# unbounded read would allocate hostile input merely to discover that it is too large.
|
| 2075 |
data = await file.read(sheet_import.MAX_IMPORT_BYTES + 1)
|
|
|
|
| 2555 |
rec = shares.set_grants("workbook", book["id"], entries,
|
| 2556 |
owner=(None if held.get("owner") else _workbook_owner(book)),
|
| 2557 |
granter=session.uname, st=_store(session), expect=prior)
|
| 2558 |
+
# Preserve explicit sharing's document marker; R31 admission uses ownership and grants.
|
| 2559 |
if "access" not in book:
|
| 2560 |
+
# Keep the existing first-share stamp on legacy documents. The owner gate above
|
| 2561 |
+
# still guards who may change grants, including on an ownerless workbook.
|
|
|
|
| 2562 |
sheets.narrow_workbook(_store(session), book["id"])
|
| 2563 |
except sheets.SheetRefusal as exc:
|
| 2564 |
raise _refuse(exc)
|
|
|
|
| 3030 |
getattr(session, "_sheet_assemblies", {}).pop(database, None)
|
| 3031 |
memo = getattr(session, "_sheet_read_memo", None)
|
| 3032 |
if isinstance(memo, dict):
|
| 3033 |
+
for slot in [slot for slot in memo if isinstance(slot, tuple)
|
| 3034 |
+
and slot[0] in {"related", "condition_fields"}]:
|
| 3035 |
memo.pop(slot, None)
|
| 3036 |
+
memo.pop(("audience", database), None)
|
| 3037 |
+
memo.pop("permission_reader", None)
|
| 3038 |
+
context = getattr(session, "_sheet_scope_context", None)
|
| 3039 |
+
if isinstance(context, dict):
|
| 3040 |
+
context.pop(("audience", database), None)
|
| 3041 |
mirrors = getattr(session, "_sheet_mirrors", {})
|
| 3042 |
for slot in list(mirrors):
|
| 3043 |
if slot[0] == database:
|
|
|
|
| 3177 |
reads of the same document object (the Daily Sales repair projects in rounds and does not).
|
| 3178 |
"""
|
| 3179 |
previous = getattr(session, "_sheet_read_memo", None)
|
| 3180 |
+
previous_assemblies = getattr(session, "_sheet_assemblies", None)
|
| 3181 |
+
previous_context = getattr(session, "_sheet_scope_context", None)
|
| 3182 |
memo = {"today": _sheet_today(session),
|
| 3183 |
"stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
|
| 3184 |
try:
|
| 3185 |
session._sheet_read_memo = memo
|
| 3186 |
+
session._sheet_assemblies = {}
|
| 3187 |
+
session._sheet_scope_context = {}
|
| 3188 |
except Exception: # noqa: BLE001
|
| 3189 |
yield None
|
| 3190 |
return
|
|
|
|
| 3192 |
yield memo
|
| 3193 |
finally:
|
| 3194 |
session._sheet_read_memo = previous
|
| 3195 |
+
session._sheet_assemblies = previous_assemblies
|
| 3196 |
+
session._sheet_scope_context = previous_context
|
| 3197 |
|
| 3198 |
|
| 3199 |
def _request_scoped(route):
|
|
|
|
| 3557 |
cell = cells.get(key)
|
| 3558 |
if not any(fr.a1_key("A1", name).rpartition("!")[0] == key.rpartition("!")[0]
|
| 3559 |
for name in names):
|
| 3560 |
+
refuse("the referenced sheet is missing.")
|
| 3561 |
if isinstance(cell, dict) and cell.get("state") in ("refused", "stale") \
|
| 3562 |
and cell.get("kind") != "formula" and cell.get("kind") != "bound":
|
| 3563 |
refuse((cell.get("refusal") or {}).get("message") or
|
|
|
|
| 3695 |
entry = relations.pick(entries, via)
|
| 3696 |
if entry is None:
|
| 3697 |
candidates = [item["key"] for item in relations.via_candidates(entries, via)
|
| 3698 |
+
if perm_scope.nav_may_open(session.user, item["target"], st=_sheet_permission_reader(session))]
|
| 3699 |
offered = (" Available relationships: " + ", ".join(candidates) + ".") if candidates else ""
|
| 3700 |
# Write our own sentence: the shared pin refusal does not name via.key.
|
| 3701 |
raise err(409, "unknown_relation",
|
|
|
|
| 3835 |
|
| 3836 |
|
| 3837 |
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
|
| 3838 |
+
book=None, tab=None, ending_seen=(), _pivot=None):
|
| 3839 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
| 3840 |
|
| 3841 |
The order below is the contract, not a preference:
|
|
|
|
| 3891 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 3892 |
if binding["aggregate"].get("field"):
|
| 3893 |
needed.add(str(binding["aggregate"]["field"]))
|
| 3894 |
+
if _pivot is not None:
|
| 3895 |
+
needed.update(_pivot_fields(_pivot))
|
| 3896 |
|
| 3897 |
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
| 3898 |
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 3899 |
cols = dict(cols)
|
| 3900 |
+
if _pivot is not None:
|
| 3901 |
+
filter_fields = (sheet_mod.filter_columns_named(scope_nodes)
|
| 3902 |
+
- sheet_mod.related_filter_ids(scope_nodes))
|
| 3903 |
+
_check_pivot_fields(session, _pivot, spec, cols, filter_fields=filter_fields)
|
| 3904 |
+
# A known column does not prove that its row-wall condition was compiled. Refuse a
|
| 3905 |
+
# skipped or aggregate wall rather than returning a pivot over a wider population.
|
| 3906 |
+
try:
|
| 3907 |
+
wall = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
|
| 3908 |
+
if wall_nodes and (wall is None or wall.uses_aggregate
|
| 3909 |
+
or sheet_mod.widen_limits(wall_nodes, cols, wall)):
|
| 3910 |
+
raise ValueError("unapplied pivot row wall")
|
| 3911 |
+
except (ValueError, sheet_mod.SheetRefusal):
|
| 3912 |
+
raise err(409, "scope_not_applicable",
|
| 3913 |
+
f"Your row restriction cannot be applied to {database}. Ask an administrator to review it.")
|
| 3914 |
|
| 3915 |
# ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
|
| 3916 |
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
|
|
|
|
| 3973 |
"live connection, so the number is not being shown rather than shown wider "
|
| 3974 |
"than you asked for. Edit the cell's scope and try again.")
|
| 3975 |
|
| 3976 |
+
if _pivot is not None:
|
| 3977 |
+
pivot_limits = list(limits) + [item for field, item in (spec.get("unbounded") or {}).items()
|
| 3978 |
+
if field in needed]
|
| 3979 |
+
return _pivot_result(cur, spec, cols, _pivot, sql, params, pivot_limits)
|
| 3980 |
+
|
| 3981 |
row = cur.execute(sql, params).fetchone()
|
| 3982 |
value = row[0] if row else None
|
| 3983 |
# ⭐⭐ EVERY WAY THIS ANSWER CAME OUT WIDER THAN THE BINDING ASKED FOR TRAVELS WITH IT
|
api/routes_tables.py
CHANGED
|
@@ -199,8 +199,11 @@ def _ut():
|
|
| 199 |
def _table_lend(runtime, table_key, defs_only=False):
|
| 200 |
"""Borrow one entry for the wall and page; old snapshot stores retain their lend."""
|
| 201 |
ut = _ut()
|
|
|
|
|
|
|
|
|
|
| 202 |
if _ut()._entry_reader(runtime) is None:
|
| 203 |
-
return ut.
|
| 204 |
try:
|
| 205 |
entry = ut.get_one(table_key, st=runtime)
|
| 206 |
except Exception as exc:
|
|
@@ -1348,17 +1351,21 @@ def _field_usage(session, table_key, fields, defn, ws_st, ws_key, st=None):
|
|
| 1348 |
|
| 1349 |
|
| 1350 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 1351 |
-
consume_corrections: bool = False, with_rows: bool = True, st=None
|
|
|
|
| 1352 |
if not with_rows and st is None:
|
| 1353 |
from routes_customers import grid_assembly
|
| 1354 |
return grid_assembly(session, table_key, storage_key,
|
| 1355 |
-
build=lambda borrowed: _compute_ut_assembly(
|
| 1356 |
-
|
| 1357 |
-
|
|
|
|
|
|
|
| 1358 |
|
| 1359 |
|
| 1360 |
def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 1361 |
-
consume_corrections: bool = False, with_rows: bool = True, st=None
|
|
|
|
| 1362 |
"""The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
|
| 1363 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 1364 |
|
|
@@ -1393,7 +1400,11 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1393 |
# reading that bucket once per question asked of it.
|
| 1394 |
if st is None:
|
| 1395 |
st = _table_lend(session.runtime, table_key, defs_only=not with_rows)
|
| 1396 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1397 |
# ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only
|
| 1398 |
# a READ route passes one — see `scoped_pool`'s own note and contract C5.
|
| 1399 |
pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st)
|
|
@@ -1544,9 +1555,11 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1544 |
# which is the honest answer for a hand-typed one rather than a descope.
|
| 1545 |
today = _recurrence_today
|
| 1546 |
measures = ut_measures(table_key, session)
|
| 1547 |
-
derived = aios_grid.cohort_cells(lists)
|
| 1548 |
-
|
| 1549 |
-
|
|
|
|
|
|
|
| 1550 |
derived.setdefault(_pid, {}).update(_cells)
|
| 1551 |
|
| 1552 |
# ⭐⭐ D-590 (wave 46 hotfix) — **THE PER-TENANT SCHEMA OVERLAY, APPLIED TO THE GRID READ.**
|
|
@@ -1583,7 +1596,8 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1583 |
# here (`table_rows`) is layering strata in a fixed order.
|
| 1584 |
"shared_cells": shared_cells,
|
| 1585 |
"measures": measures,
|
| 1586 |
-
"measure_sets": _ut_measure_sets(table_key, views, measures, pids, today)
|
|
|
|
| 1587 |
"today": today,
|
| 1588 |
# ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test
|
| 1589 |
# for is a key a consumer forgets to test for, and this one carries a refusal.
|
|
|
|
| 199 |
def _table_lend(runtime, table_key, defs_only=False):
|
| 200 |
"""Borrow one entry for the wall and page; old snapshot stores retain their lend."""
|
| 201 |
ut = _ut()
|
| 202 |
+
if defs_only:
|
| 203 |
+
# Metadata must enter through the store projection, even on entry-capable stores.
|
| 204 |
+
return ut.lend_defs(runtime)
|
| 205 |
if _ut()._entry_reader(runtime) is None:
|
| 206 |
+
return ut.lend(runtime)
|
| 207 |
try:
|
| 208 |
entry = ut.get_one(table_key, st=runtime)
|
| 209 |
except Exception as exc:
|
|
|
|
| 1351 |
|
| 1352 |
|
| 1353 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 1354 |
+
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 1355 |
+
*, fields_only: bool = False, needed=None):
|
| 1356 |
if not with_rows and st is None:
|
| 1357 |
from routes_customers import grid_assembly
|
| 1358 |
return grid_assembly(session, table_key, storage_key,
|
| 1359 |
+
build=lambda borrowed: _compute_ut_assembly(
|
| 1360 |
+
borrowed, table_key, storage_key, False, False, fields_only=fields_only, needed=needed),
|
| 1361 |
+
pool_stamp=lambda: 0, with_rows=not fields_only, needed=needed)
|
| 1362 |
+
return _compute_ut_assembly(session, table_key, storage_key, False, with_rows, st,
|
| 1363 |
+
fields_only=fields_only, needed=needed)
|
| 1364 |
|
| 1365 |
|
| 1366 |
def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 1367 |
+
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 1368 |
+
*, fields_only: bool = False, needed=None):
|
| 1369 |
"""The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
|
| 1370 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 1371 |
|
|
|
|
| 1400 |
# reading that bucket once per question asked of it.
|
| 1401 |
if st is None:
|
| 1402 |
st = _table_lend(session.runtime, table_key, defs_only=not with_rows)
|
| 1403 |
+
if fields_only:
|
| 1404 |
+
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1405 |
+
fields_base = list(defn.get("fields") or [])
|
| 1406 |
+
pids, rows_src = frozenset(), []
|
| 1407 |
+
elif with_rows:
|
| 1408 |
# ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only
|
| 1409 |
# a READ route passes one — see `scoped_pool`'s own note and contract C5.
|
| 1410 |
pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st)
|
|
|
|
| 1555 |
# which is the honest answer for a hand-typed one rather than a descope.
|
| 1556 |
today = _recurrence_today
|
| 1557 |
measures = ut_measures(table_key, session)
|
| 1558 |
+
derived = aios_grid.cohort_cells(lists) if not fields_only else {}
|
| 1559 |
+
materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed]
|
| 1560 |
+
for _pid, _cells in (_ut_measure_cells(table_key, materialized_fields, pids, today, measures,
|
| 1561 |
+
getattr(session.runtime, 'measure_memo', {}))
|
| 1562 |
+
if not fields_only and (needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items():
|
| 1563 |
derived.setdefault(_pid, {}).update(_cells)
|
| 1564 |
|
| 1565 |
# ⭐⭐ D-590 (wave 46 hotfix) — **THE PER-TENANT SCHEMA OVERLAY, APPLIED TO THE GRID READ.**
|
|
|
|
| 1596 |
# here (`table_rows`) is layering strata in a fixed order.
|
| 1597 |
"shared_cells": shared_cells,
|
| 1598 |
"measures": measures,
|
| 1599 |
+
"measure_sets": (_ut_measure_sets(table_key, views, measures, pids, today)
|
| 1600 |
+
if not fields_only and needed is None else {}),
|
| 1601 |
"today": today,
|
| 1602 |
# ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test
|
| 1603 |
# for is a key a consumer forgets to test for, and this one carries a refusal.
|
api/telnyx_whatsapp.py
CHANGED
|
@@ -107,6 +107,10 @@ def _send(to, message, *, purpose, tenant, username, transport, store_):
|
|
| 107 |
"name": message["template"], "language": {"code": "en_US"},
|
| 108 |
"components": [{"type": "body", "parameters": [
|
| 109 |
{"type": "text", "text": p} for p in message["params"]]}]}}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
else:
|
| 111 |
whatsapp_message = {"type": "text", "text": {"body": message["text"]}}
|
| 112 |
payload = {"from": sender, "to": to, "whatsapp_message": whatsapp_message}
|
|
@@ -134,6 +138,107 @@ def _send(to, message, *, purpose, tenant, username, transport, store_):
|
|
| 134 |
return copy.deepcopy(entry)
|
| 135 |
|
| 136 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
def send_template(to, template, params, *, purpose, tenant="", username="",
|
| 138 |
transport=None, store_=None):
|
| 139 |
_check_text(template)
|
|
|
|
| 107 |
"name": message["template"], "language": {"code": "en_US"},
|
| 108 |
"components": [{"type": "body", "parameters": [
|
| 109 |
{"type": "text", "text": p} for p in message["params"]]}]}}
|
| 110 |
+
if purpose == "auth_code" and message["template"] == auth_template():
|
| 111 |
+
whatsapp_message["template"]["components"].append({
|
| 112 |
+
"type": "button", "sub_type": "url", "index": 0,
|
| 113 |
+
"parameters": [{"type": "text", "text": message["params"][0]}]})
|
| 114 |
else:
|
| 115 |
whatsapp_message = {"type": "text", "text": {"body": message["text"]}}
|
| 116 |
payload = {"from": sender, "to": to, "whatsapp_message": whatsapp_message}
|
|
|
|
| 138 |
return copy.deepcopy(entry)
|
| 139 |
|
| 140 |
|
| 141 |
+
TELNYX_FAILED = {"delivery_failed", "sending_failed", "expired"}
|
| 142 |
+
UNDELIVERED = "WhatsApp could not deliver this message."
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _first_code(errors):
|
| 146 |
+
if isinstance(errors, list) and errors and isinstance(errors[0], dict):
|
| 147 |
+
return errors[0].get("code")
|
| 148 |
+
return None
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _status_updates(event_type, payload):
|
| 152 |
+
"""Read (providerId, outcome, telnyxCode, metaCode) rows from one status event."""
|
| 153 |
+
if not isinstance(payload, dict):
|
| 154 |
+
return []
|
| 155 |
+
if event_type == "message.finalized":
|
| 156 |
+
provider_id = payload.get("id")
|
| 157 |
+
if payload.get("type") != "WHATSAPP" or not isinstance(provider_id, str) or not provider_id:
|
| 158 |
+
return []
|
| 159 |
+
recipients = payload.get("to") if isinstance(payload.get("to"), list) else []
|
| 160 |
+
states = {row.get("status") for row in recipients
|
| 161 |
+
if isinstance(row, dict) and isinstance(row.get("status"), str)}
|
| 162 |
+
if states & TELNYX_FAILED:
|
| 163 |
+
code = _first_code(payload.get("errors"))
|
| 164 |
+
return [(provider_id, "failed", None if code in (None, "") else str(code), None)]
|
| 165 |
+
return [(provider_id, "delivered", None, None)] if "delivered" in states else []
|
| 166 |
+
if event_type == "whatsapp.messages":
|
| 167 |
+
rows = []
|
| 168 |
+
for row in payload.get("statuses") if isinstance(payload.get("statuses"), list) else []:
|
| 169 |
+
# A malformed row is skipped: a status event never answers 400.
|
| 170 |
+
if (not isinstance(row, dict) or not isinstance(row.get("id"), str) or not row["id"]
|
| 171 |
+
or not isinstance(row.get("status"), str)):
|
| 172 |
+
continue
|
| 173 |
+
if row.get("status") == "failed":
|
| 174 |
+
code = _first_code(row.get("errors"))
|
| 175 |
+
if isinstance(code, str) and code.isdigit():
|
| 176 |
+
code = int(code)
|
| 177 |
+
meta = code if isinstance(code, int) and not isinstance(code, bool) else None
|
| 178 |
+
rows.append((row["id"], "failed", None, meta))
|
| 179 |
+
elif row.get("status") in {"delivered", "read"}:
|
| 180 |
+
rows.append((row["id"], "delivered", None, None))
|
| 181 |
+
return rows
|
| 182 |
+
return []
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _apply_status(entry, outcome, telnyx_code, meta_code):
|
| 186 |
+
before = dict(entry)
|
| 187 |
+
if outcome == "failed":
|
| 188 |
+
# Failed is terminal: a later event adds a code but never clears one.
|
| 189 |
+
entry.update(status="failed", error=UNDELIVERED)
|
| 190 |
+
if telnyx_code is not None:
|
| 191 |
+
entry["errorCode"] = telnyx_code
|
| 192 |
+
if meta_code is not None:
|
| 193 |
+
entry["metaCode"] = meta_code
|
| 194 |
+
elif entry.get("status") != "failed":
|
| 195 |
+
entry["status"] = "delivered"
|
| 196 |
+
return entry != before
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def record_status(event_type, payload, *, store_=None):
|
| 200 |
+
"""Apply a signed Telnyx delivery event to outbox entries by providerId; return entries changed."""
|
| 201 |
+
rows = _status_updates(event_type, payload)
|
| 202 |
+
if not rows:
|
| 203 |
+
return 0
|
| 204 |
+
selected = store_ if store_ is not None else _store
|
| 205 |
+
current = (selected.get(OUTBOX_KEY) or {}).get("messages") or []
|
| 206 |
+
|
| 207 |
+
def changes(messages):
|
| 208 |
+
changed = 0
|
| 209 |
+
for entry in messages:
|
| 210 |
+
if not isinstance(entry, dict) or not entry.get("providerId"):
|
| 211 |
+
continue
|
| 212 |
+
for provider_id, outcome, telnyx_code, meta_code in rows:
|
| 213 |
+
if entry["providerId"] == provider_id:
|
| 214 |
+
changed += _apply_status(entry, outcome, telnyx_code, meta_code)
|
| 215 |
+
return changed
|
| 216 |
+
|
| 217 |
+
# An unknown id or a repeated event writes nothing at all.
|
| 218 |
+
if not changes(copy.deepcopy(current)):
|
| 219 |
+
return 0
|
| 220 |
+
outcome = {"changed": 0}
|
| 221 |
+
|
| 222 |
+
def apply(doc):
|
| 223 |
+
doc["messages"] = list(doc.get("messages") or [])
|
| 224 |
+
outcome["changed"] = changes(doc["messages"])
|
| 225 |
+
return doc
|
| 226 |
+
|
| 227 |
+
selected.update(OUTBOX_KEY, apply)
|
| 228 |
+
return outcome["changed"]
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def delivery_status(provider_id, *, store_=None):
|
| 232 |
+
"""Return the outbox status of the newest entry sent as provider_id, or None."""
|
| 233 |
+
if not provider_id:
|
| 234 |
+
return None
|
| 235 |
+
messages = ((store_ if store_ is not None else _store).get(OUTBOX_KEY) or {}).get("messages") or []
|
| 236 |
+
for entry in reversed(messages):
|
| 237 |
+
if isinstance(entry, dict) and entry.get("providerId") == provider_id:
|
| 238 |
+
return entry.get("status")
|
| 239 |
+
return None
|
| 240 |
+
|
| 241 |
+
|
| 242 |
def send_template(to, template, params, *, purpose, tenant="", username="",
|
| 243 |
transport=None, store_=None):
|
| 244 |
_check_text(template)
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -171,7 +171,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 171 |
events.append(('sleep', seconds))
|
| 172 |
cycle[0] += 1
|
| 173 |
sleeps.append(seconds)
|
| 174 |
-
def sync(manual, passes):
|
|
|
|
| 175 |
events.append(('sync', passes))
|
| 176 |
self.assertFalse(manual)
|
| 177 |
self.assertEqual(passes, len(reports) + 1)
|
|
@@ -193,7 +194,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 193 |
# every pass would ask for a zero wait.
|
| 194 |
clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60,
|
| 195 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 196 |
-
ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
|
|
|
|
| 197 |
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
|
| 198 |
tenant='synthetic', tenantSource='test', tenantSeconds=60,
|
| 199 |
envSeconds=60, disagreement=None),
|
|
@@ -499,7 +501,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 499 |
raise StopIteration('bounded synthetic resident loop')
|
| 500 |
return False
|
| 501 |
|
| 502 |
-
def sync(manual_flag, passes):
|
|
|
|
| 503 |
events.append(('sync', passes))
|
| 504 |
err = failures.pop(0) if failures else None
|
| 505 |
if err is not None:
|
|
@@ -514,6 +517,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 514 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 515 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 516 |
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None],
|
|
|
|
| 517 |
_resolve_resync_cadence=lambda: dict(
|
| 518 |
resolvedSeconds=cadence, manual=manual, tenant='synthetic',
|
| 519 |
tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
|
|
|
|
| 171 |
events.append(('sleep', seconds))
|
| 172 |
cycle[0] += 1
|
| 173 |
sleeps.append(seconds)
|
| 174 |
+
def sync(manual, passes, entities):
|
| 175 |
+
self.assertIsNone(entities) # Existing tenants retain the full mirror selection.
|
| 176 |
events.append(('sync', passes))
|
| 177 |
self.assertFalse(manual)
|
| 178 |
self.assertEqual(passes, len(reports) + 1)
|
|
|
|
| 194 |
# every pass would ask for a zero wait.
|
| 195 |
clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60,
|
| 196 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 197 |
+
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None],
|
| 198 |
+
_configured_mirror_entities=lambda *_: None,
|
| 199 |
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
|
| 200 |
tenant='synthetic', tenantSource='test', tenantSeconds=60,
|
| 201 |
envSeconds=60, disagreement=None),
|
|
|
|
| 501 |
raise StopIteration('bounded synthetic resident loop')
|
| 502 |
return False
|
| 503 |
|
| 504 |
+
def sync(manual_flag, passes, entities):
|
| 505 |
+
self.assertIsNone(entities)
|
| 506 |
events.append(('sync', passes))
|
| 507 |
err = failures.pop(0) if failures else None
|
| 508 |
if err is not None:
|
|
|
|
| 517 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 518 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 519 |
ns = dict(RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None],
|
| 520 |
+
_configured_mirror_entities=lambda *_: None,
|
| 521 |
_resolve_resync_cadence=lambda: dict(
|
| 522 |
resolvedSeconds=cadence, manual=manual, tenant='synthetic',
|
| 523 |
tenantSource='test', tenantSeconds=cadence, envSeconds=cadence,
|
api/whatsapp_door.py
CHANGED
|
@@ -146,13 +146,23 @@ def accept(raw: bytes, signature: str, timestamp: str, *, now=None):
|
|
| 146 |
raise WhatsAppDoorRefusal("Telnyx webhook signature verification failed")
|
| 147 |
try:
|
| 148 |
data = json.loads(raw)["data"]
|
|
|
|
|
|
|
| 149 |
if data["event_type"] != "message.received":
|
| 150 |
return None
|
| 151 |
payload = data["payload"]
|
| 152 |
message_id = payload["id"]
|
| 153 |
phone = whatsapp_identity.normalize_phone(payload["from"]["phone_number"])
|
| 154 |
-
|
| 155 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
raise ValueError("Invalid message fields")
|
| 157 |
except (ValueError, TypeError, KeyError) as exc:
|
| 158 |
raise ValueError("The WhatsApp webhook payload is invalid.") from exc
|
|
|
|
| 146 |
raise WhatsAppDoorRefusal("Telnyx webhook signature verification failed")
|
| 147 |
try:
|
| 148 |
data = json.loads(raw)["data"]
|
| 149 |
+
# A delivery status updates the outbox only: no receipt claim, no window stamp, no turn.
|
| 150 |
+
telnyx_whatsapp.record_status(data["event_type"], data.get("payload"))
|
| 151 |
if data["event_type"] != "message.received":
|
| 152 |
return None
|
| 153 |
payload = data["payload"]
|
| 154 |
message_id = payload["id"]
|
| 155 |
phone = whatsapp_identity.normalize_phone(payload["from"]["phone_number"])
|
| 156 |
+
if not isinstance(message_id, str) or not message_id.strip():
|
| 157 |
+
raise ValueError("Invalid message fields")
|
| 158 |
+
body = payload.get("body", "")
|
| 159 |
+
# Edits and media can carry decoy text; ignore them before claiming receipt/window.
|
| 160 |
+
if isinstance(body, dict) and body.get("type") != "text":
|
| 161 |
+
return None
|
| 162 |
+
text = payload.get("text", body)
|
| 163 |
+
if isinstance(body, dict) and not isinstance(text, str):
|
| 164 |
+
text = body["text"]["body"]
|
| 165 |
+
if not isinstance(text, str):
|
| 166 |
raise ValueError("Invalid message fields")
|
| 167 |
except (ValueError, TypeError, KeyError) as exc:
|
| 168 |
raise ValueError("The WhatsApp webhook payload is invalid.") from exc
|
api/whatsapp_identity.py
CHANGED
|
@@ -22,6 +22,7 @@ LOCK_S = 900
|
|
| 22 |
PHONE_TAKEN = "This number is already connected to another Loopable account."
|
| 23 |
CODE_WRONG = "That code is not right. Check the latest WhatsApp message and try again."
|
| 24 |
CODE_EXPIRED = "That code has expired. Send a new code."
|
|
|
|
| 25 |
|
| 26 |
|
| 27 |
class WhatsAppRefusal(ValueError):
|
|
@@ -145,6 +146,16 @@ def start(tenant, username, phone, *, now=None, sender=None):
|
|
| 145 |
p: row for p, row in doc.items()
|
| 146 |
if p != phone or row.get("codeHash") != pending["codeHash"]})
|
| 147 |
raise WhatsAppRefusal("The code could not be sent. Try again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
return {"state": "pending", "phone": phone, "expiresAt": pending["expiresAt"]}
|
| 149 |
|
| 150 |
|
|
@@ -201,4 +212,6 @@ def status(tenant, username, *, now=None):
|
|
| 201 |
"message": _lock_sentence(pending, now)}
|
| 202 |
if pending["expiresAt"] <= now:
|
| 203 |
return {"state": "none"}
|
|
|
|
|
|
|
| 204 |
return {"state": "pending", "phone": phone, "expiresAt": pending["expiresAt"]}
|
|
|
|
| 22 |
PHONE_TAKEN = "This number is already connected to another Loopable account."
|
| 23 |
CODE_WRONG = "That code is not right. Check the latest WhatsApp message and try again."
|
| 24 |
CODE_EXPIRED = "That code has expired. Send a new code."
|
| 25 |
+
CODE_UNDELIVERED = "The code could not be delivered to this number."
|
| 26 |
|
| 27 |
|
| 28 |
class WhatsAppRefusal(ValueError):
|
|
|
|
| 146 |
p: row for p, row in doc.items()
|
| 147 |
if p != phone or row.get("codeHash") != pending["codeHash"]})
|
| 148 |
raise WhatsAppRefusal("The code could not be sent. Try again.")
|
| 149 |
+
provider_id = delivery.get("providerId") if isinstance(delivery, dict) else None
|
| 150 |
+
if provider_id:
|
| 151 |
+
# status() reads the outbox by this id, so a later refusal can reach the caller.
|
| 152 |
+
def remember(doc):
|
| 153 |
+
row = doc.get(phone)
|
| 154 |
+
if isinstance(row, dict) and row.get("codeHash") == pending["codeHash"]:
|
| 155 |
+
row["providerId"] = provider_id
|
| 156 |
+
return doc
|
| 157 |
+
|
| 158 |
+
_store.update(CODES_KEY, remember)
|
| 159 |
return {"state": "pending", "phone": phone, "expiresAt": pending["expiresAt"]}
|
| 160 |
|
| 161 |
|
|
|
|
| 212 |
"message": _lock_sentence(pending, now)}
|
| 213 |
if pending["expiresAt"] <= now:
|
| 214 |
return {"state": "none"}
|
| 215 |
+
if telnyx_whatsapp.delivery_status(pending.get("providerId")) == "failed":
|
| 216 |
+
return {"state": "undelivered", "phone": phone, "message": CODE_UNDELIVERED}
|
| 217 |
return {"state": "pending", "phone": phone, "expiresAt": pending["expiresAt"]}
|
platform/aios_grid.py
CHANGED
|
@@ -1921,7 +1921,12 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
|
|
| 1921 |
row[k] = v if field["type"] in {"text", "status", "date"} else (
|
| 1922 |
_round2(v) if field["type"] in {"currency", "pct"} else _round(v))
|
| 1923 |
for field in overlay_fields:
|
| 1924 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1925 |
got = derived.get(pid) or {}
|
| 1926 |
for k in derived_keys:
|
| 1927 |
# '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a
|
|
|
|
| 1921 |
row[k] = v if field["type"] in {"text", "status", "date"} else (
|
| 1922 |
_round2(v) if field["type"] in {"currency", "pct"} else _round(v))
|
| 1923 |
for field in overlay_fields:
|
| 1924 |
+
k = field["key"]
|
| 1925 |
+
# The pool may carry the mirror's sync stamp. It wins over a stale overlay,
|
| 1926 |
+
# including an explicit blank; never manufacture freshness from _created or now.
|
| 1927 |
+
row[k] = r.get(k, saved.get(k, "")) if (
|
| 1928 |
+
k == "refreshed" and (field.get("automation") or {}).get("preset")
|
| 1929 |
+
) else saved.get(k, "")
|
| 1930 |
got = derived.get(pid) or {}
|
| 1931 |
for k in derived_keys:
|
| 1932 |
# '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a
|
platform/core/audit.py
CHANGED
|
@@ -5,6 +5,7 @@ import re
|
|
| 5 |
import time
|
| 6 |
|
| 7 |
from harness import runtime
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
AUDIT_KINDS = frozenset({
|
|
@@ -142,6 +143,8 @@ def ordered(doc, tenant, limit=200) -> list[dict]:
|
|
| 142 |
def record(kind, tenant, principal, detail=None) -> dict | None:
|
| 143 |
"""Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
|
| 144 |
item = event(kind, tenant, principal, detail)
|
|
|
|
|
|
|
| 145 |
tenant = item["tenant"]
|
| 146 |
|
| 147 |
def _append(doc):
|
|
|
|
| 5 |
import time
|
| 6 |
|
| 7 |
from harness import runtime
|
| 8 |
+
from core import freeze_notice
|
| 9 |
|
| 10 |
|
| 11 |
AUDIT_KINDS = frozenset({
|
|
|
|
| 143 |
def record(kind, tenant, principal, detail=None) -> dict | None:
|
| 144 |
"""Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
|
| 145 |
item = event(kind, tenant, principal, detail)
|
| 146 |
+
if freeze_notice.enabled():
|
| 147 |
+
return None
|
| 148 |
tenant = item["tenant"]
|
| 149 |
|
| 150 |
def _append(doc):
|
platform/core/data_binding.py
CHANGED
|
@@ -49,6 +49,7 @@ per D-160 the Space environment cannot be read from outside, so the container ha
|
|
| 49 |
saw. One deploy then answers "does SPACE_ID exist here", instead of three.
|
| 50 |
"""
|
| 51 |
import os
|
|
|
|
| 52 |
|
| 53 |
#: The deployment that OWNS production data. A frozenset rather than a string because the product
|
| 54 |
#: has had two public addresses before and will again.
|
|
@@ -284,6 +285,7 @@ def write_refusal(repo):
|
|
| 284 |
|
| 285 |
def check_write(repo, operation='write'):
|
| 286 |
"""Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door."""
|
|
|
|
| 287 |
reason = gate_write_refusal(repo) or write_refusal(repo)
|
| 288 |
if reason:
|
| 289 |
raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}')
|
|
|
|
| 49 |
saw. One deploy then answers "does SPACE_ID exist here", instead of three.
|
| 50 |
"""
|
| 51 |
import os
|
| 52 |
+
from core import freeze_notice
|
| 53 |
|
| 54 |
#: The deployment that OWNS production data. A frozenset rather than a string because the product
|
| 55 |
#: has had two public addresses before and will again.
|
|
|
|
| 285 |
|
| 286 |
def check_write(repo, operation='write'):
|
| 287 |
"""Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door."""
|
| 288 |
+
freeze_notice.check_write()
|
| 289 |
reason = gate_write_refusal(repo) or write_refusal(repo)
|
| 290 |
if reason:
|
| 291 |
raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}')
|
platform/core/external_oauth.py
CHANGED
|
@@ -14,7 +14,7 @@ import secrets
|
|
| 14 |
import time
|
| 15 |
from urllib.parse import urlsplit, urlunsplit
|
| 16 |
|
| 17 |
-
from core import external_access
|
| 18 |
|
| 19 |
|
| 20 |
BUCKET = "external_oauth"
|
|
@@ -400,7 +400,9 @@ def authorize(st, username: str, tenant: str, *, client_id, redirect_uri, state,
|
|
| 400 |
raise ExternalOAuthError("oauth_client_invalid")
|
| 401 |
if redirect_uri not in (client.get("redirect_uris") or []):
|
| 402 |
raise ExternalOAuthError("oauth_redirect_uri_invalid")
|
| 403 |
-
|
|
|
|
|
|
|
| 404 |
scopes = _normalise_scopes(scope, access)
|
| 405 |
states = {key: row for key, row in book["states"].items()
|
| 406 |
if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at}
|
|
@@ -604,6 +606,78 @@ def _intersect_databases(left, right):
|
|
| 604 |
return tuple(item for item in (left or ()) if item in allowed)
|
| 605 |
|
| 606 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 607 |
def verify_access(token: str, *, resource, runtime_resolver) -> VerifiedOAuth:
|
| 608 |
try:
|
| 609 |
_, tenant_key, grant_id, secret = _parse_token(token, {"lpo1"})
|
|
@@ -626,17 +700,11 @@ def verify_access(token: str, *, resource, runtime_resolver) -> VerifiedOAuth:
|
|
| 626 |
or client.get("username") != row.get("username")
|
| 627 |
or client.get("tenant") != tenant_key):
|
| 628 |
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
client_access, client_databases = external_access.scope_of(client)
|
| 635 |
-
access = (external_access.ACCESS_READWRITE
|
| 636 |
-
if grant_access == client_access == external_access.ACCESS_READWRITE
|
| 637 |
-
and "mcp:write" in (row.get("scopes") or ())
|
| 638 |
-
else external_access.ACCESS_READ)
|
| 639 |
-
databases = _intersect_databases(grant_databases, client_databases)
|
| 640 |
if now - float(row.get("last_used_at") or 0) >= _STAMP_EVERY:
|
| 641 |
def _stamp(value):
|
| 642 |
latest = _book_value(value)
|
|
@@ -652,7 +720,7 @@ def verify_access(token: str, *, resource, runtime_resolver) -> VerifiedOAuth:
|
|
| 652 |
return latest
|
| 653 |
_update(st, _stamp, auth=True)
|
| 654 |
return VerifiedOAuth(tenant_key, grant_id, row["client_id"], public, st, access,
|
| 655 |
-
databases,
|
| 656 |
except ExternalOAuthAuthError:
|
| 657 |
raise
|
| 658 |
except Exception as exc:
|
|
|
|
| 14 |
import time
|
| 15 |
from urllib.parse import urlsplit, urlunsplit
|
| 16 |
|
| 17 |
+
from core import external_access, perm_scope, perms, registry, shares, user_tables, users
|
| 18 |
|
| 19 |
|
| 20 |
BUCKET = "external_oauth"
|
|
|
|
| 400 |
raise ExternalOAuthError("oauth_client_invalid")
|
| 401 |
if redirect_uri not in (client.get("redirect_uris") or []):
|
| 402 |
raise ExternalOAuthError("oauth_redirect_uri_invalid")
|
| 403 |
+
# R15/C5: approval delegates the live account authority. Saved client settings and
|
| 404 |
+
# old grants remain readable metadata, never a cap on a future Manage users change.
|
| 405 |
+
access, databases = external_access.ACCESS_READWRITE, external_access.ALL_DATABASES
|
| 406 |
scopes = _normalise_scopes(scope, access)
|
| 407 |
states = {key: row for key, row in book["states"].items()
|
| 408 |
if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at}
|
|
|
|
| 606 |
return tuple(item for item in (left or ()) if item in allowed)
|
| 607 |
|
| 608 |
|
| 609 |
+
def _live_permission_user(username: str, tenant: str) -> dict:
|
| 610 |
+
"""Read and validate the raw account before its public projection can discard bad perms."""
|
| 611 |
+
records = users.registry()
|
| 612 |
+
record = records.get(username) if isinstance(records, dict) else None
|
| 613 |
+
if not isinstance(record, dict) or record.get("active", True) is not True:
|
| 614 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 615 |
+
version = record.get("perms_v", 0)
|
| 616 |
+
if type(version) is not int or version < 0:
|
| 617 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 618 |
+
block = record.get("perms", {})
|
| 619 |
+
if not isinstance(block, dict):
|
| 620 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 621 |
+
for key, entry in block.items():
|
| 622 |
+
if not isinstance(key, str) or not key or not isinstance(entry, dict):
|
| 623 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 624 |
+
if "access" in entry and type(entry["access"]) is not bool:
|
| 625 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 626 |
+
if "metrics" in entry and type(entry["metrics"]) is not bool:
|
| 627 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 628 |
+
tree = entry.get("filter")
|
| 629 |
+
if tree is not None and (not isinstance(tree, dict)
|
| 630 |
+
or not isinstance(tree.get("nodes"), list)
|
| 631 |
+
or tree.get("conj", "and") not in {"and", "or"}):
|
| 632 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 633 |
+
if "hiddenFields" in entry and (not isinstance(entry["hiddenFields"], list)
|
| 634 |
+
or any(not isinstance(field, str) for field in entry["hiddenFields"])):
|
| 635 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 636 |
+
modules = record.get("modules", "all")
|
| 637 |
+
if modules != "all" and (not isinstance(modules, (list, tuple)) or not modules
|
| 638 |
+
or any(not isinstance(key, str) or not key for key in modules)):
|
| 639 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 640 |
+
public = users._public(username, record)
|
| 641 |
+
if public.get("tenant") != tenant:
|
| 642 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 643 |
+
return public
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def _live_permission_scope(user: dict, st) -> tuple[str, object]:
|
| 647 |
+
"""C5: resolve this call's database and editor authority, without reading database rows.
|
| 648 |
+
|
| 649 |
+
The canonical evaluators preserve migrated topic denies and the distinct ut_* share rule.
|
| 650 |
+
The coarse write capability also permits owned Sheet work; each writer still checks its
|
| 651 |
+
own object, database, row and field authority. Neither client nor grant snapshots enter.
|
| 652 |
+
"""
|
| 653 |
+
if perms.is_admin(user):
|
| 654 |
+
return external_access.ACCESS_READWRITE, external_access.ALL_DATABASES
|
| 655 |
+
definitions = st.get_projection(user_tables.STORE_KEY, drop=user_tables.DEF_ONLY_DROPS)
|
| 656 |
+
if definitions is None:
|
| 657 |
+
definitions = {}
|
| 658 |
+
if not isinstance(definitions, dict) or any(
|
| 659 |
+
not isinstance(key, str) or not isinstance(value, dict)
|
| 660 |
+
for key, value in definitions.items()):
|
| 661 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 662 |
+
grants = st.get(shares.SHARES_KEY, fresh=True)
|
| 663 |
+
if grants is None:
|
| 664 |
+
grants = {}
|
| 665 |
+
if not isinstance(grants, dict):
|
| 666 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 667 |
+
if "database" in grants and not isinstance(grants["database"], dict):
|
| 668 |
+
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 669 |
+
lent = user_tables.lend(st, **{user_tables.STORE_KEY: definitions, shares.SHARES_KEY: grants})
|
| 670 |
+
topics = {row["key"] for row in registry.governable_modules().values()
|
| 671 |
+
if perms.tenant_has_module(st, row["key"])}
|
| 672 |
+
candidates = topics | {key for key in definitions if key.startswith(user_tables.KEY_PREFIX)}
|
| 673 |
+
databases = tuple(key for key in sorted(candidates) if perm_scope.may_read(user, key, st=lent))
|
| 674 |
+
editable = any(key in topics or user_tables.may_edit_database(
|
| 675 |
+
key, user["username"], st=lent) for key in databases)
|
| 676 |
+
editable = editable or (perms.tenant_has_module(st, "sheets")
|
| 677 |
+
and perm_scope.nav_may_open(user, "sheets", st=lent))
|
| 678 |
+
return (external_access.ACCESS_READWRITE if editable else external_access.ACCESS_READ), databases
|
| 679 |
+
|
| 680 |
+
|
| 681 |
def verify_access(token: str, *, resource, runtime_resolver) -> VerifiedOAuth:
|
| 682 |
try:
|
| 683 |
_, tenant_key, grant_id, secret = _parse_token(token, {"lpo1"})
|
|
|
|
| 700 |
or client.get("username") != row.get("username")
|
| 701 |
or client.get("tenant") != tenant_key):
|
| 702 |
raise ExternalOAuthAuthError(AUTH_ERROR)
|
| 703 |
+
public = _live_permission_user(row["username"], tenant_key)
|
| 704 |
+
access, databases = _live_permission_scope(public, st)
|
| 705 |
+
scopes = ("mcp:read",) + (("mcp:write",) if external_access.may_write(access) else ())
|
| 706 |
+
if "offline_access" in (row.get("scopes") or ()):
|
| 707 |
+
scopes += ("offline_access",)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 708 |
if now - float(row.get("last_used_at") or 0) >= _STAMP_EVERY:
|
| 709 |
def _stamp(value):
|
| 710 |
latest = _book_value(value)
|
|
|
|
| 720 |
return latest
|
| 721 |
_update(st, _stamp, auth=True)
|
| 722 |
return VerifiedOAuth(tenant_key, grant_id, row["client_id"], public, st, access,
|
| 723 |
+
databases, scopes, audience)
|
| 724 |
except ExternalOAuthAuthError:
|
| 725 |
raise
|
| 726 |
except Exception as exc:
|
platform/core/field_permissions.py
CHANGED
|
@@ -1157,7 +1157,8 @@ def reconcile_shared_permissions(shared_key, grant_topic, st=None, shared_docume
|
|
| 1157 |
# agree without turning ungoverned contract fields into read-only fields.
|
| 1158 |
updated[grant_mark] = True
|
| 1159 |
updated["permissions"] = derived
|
| 1160 |
-
|
|
|
|
| 1161 |
if isinstance(shared_document, dict):
|
| 1162 |
shared_document.setdefault("fields", {})[key] = updated
|
| 1163 |
rewritten += 1
|
|
@@ -1374,6 +1375,30 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None
|
|
| 1374 |
if not key or st is None:
|
| 1375 |
return {"promoted": 0, "normalized": 0, "reclassified": 0,
|
| 1376 |
"suppressed": 0, "purged": 0, "folded": 0}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1377 |
# ⭐⭐ W50-T23 — THE FOLD RUNS BEFORE THE RECONCILE, AND THE ORDER IS THE POINT. A definition
|
| 1378 |
# that arrives from the legacy bucket must be classified by the same pass that classifies
|
| 1379 |
# every other one; folding after the reconcile would leave the newcomer unreconciled until
|
|
|
|
| 1157 |
# agree without turning ungoverned contract fields into read-only fields.
|
| 1158 |
updated[grant_mark] = True
|
| 1159 |
updated["permissions"] = derived
|
| 1160 |
+
if not getattr(st, "schema_only", False):
|
| 1161 |
+
shared_overlay.put_field(bucket, key, updated, st=st)
|
| 1162 |
if isinstance(shared_document, dict):
|
| 1163 |
shared_document.setdefault("fields", {})[key] = updated
|
| 1164 |
rewritten += 1
|
|
|
|
| 1375 |
if not key or st is None:
|
| 1376 |
return {"promoted": 0, "normalized": 0, "reclassified": 0,
|
| 1377 |
"suppressed": 0, "purged": 0, "folded": 0}
|
| 1378 |
+
if getattr(st, "schema_only", False):
|
| 1379 |
+
# The same migration decisions and grant classifier, over detached definitions only.
|
| 1380 |
+
# A pending copy-before-remove operation belongs to the full read door.
|
| 1381 |
+
fold_legacy_shared_bucket(shared_key, st=st)
|
| 1382 |
+
document = (workspace_document if isinstance(workspace_document, dict)
|
| 1383 |
+
else st.get_projection(key, drop=("overlays",)))
|
| 1384 |
+
shared = (shared_document if isinstance(shared_document, dict)
|
| 1385 |
+
else shared_overlay.document(shared_key, st=st))
|
| 1386 |
+
tombstoned = _live_tombstones(document)
|
| 1387 |
+
for owner, workspace in (document or {}).items():
|
| 1388 |
+
if owner == WORKSPACE_SHARED_MEMBER or not isinstance(workspace, dict):
|
| 1389 |
+
continue
|
| 1390 |
+
for field_key, original in (workspace.get("fields") or {}).items():
|
| 1391 |
+
if (str(field_key) not in tombstoned and _field_is_migratable(original)
|
| 1392 |
+
and stored_permissions(original, fallback="collaborative",
|
| 1393 |
+
known_users=known_users)["edit"] != "personal"):
|
| 1394 |
+
raise shared_overlay.SchemaUnavailable(
|
| 1395 |
+
'field definitions need a full workspace refresh to finish migration')
|
| 1396 |
+
reclassified = reconcile_shared_permissions(
|
| 1397 |
+
shared_key, grant_key, st=st, shared_document=shared)
|
| 1398 |
+
for dead in tombstoned:
|
| 1399 |
+
(shared.get("fields") or {}).pop(dead, None)
|
| 1400 |
+
return {"promoted": 0, "normalized": 0, "reclassified": reclassified,
|
| 1401 |
+
"suppressed": 0, "purged": 0, "folded": 0}
|
| 1402 |
# ⭐⭐ W50-T23 — THE FOLD RUNS BEFORE THE RECONCILE, AND THE ORDER IS THE POINT. A definition
|
| 1403 |
# that arrives from the legacy bucket must be classified by the same pass that classifies
|
| 1404 |
# every other one; folding after the reconcile would leave the newcomer unreconciled until
|
platform/core/freeze_notice.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deployment maintenance state, independent of sessions and stored documents."""
|
| 2 |
+
import os
|
| 3 |
+
from contextlib import contextmanager
|
| 4 |
+
from contextvars import ContextVar
|
| 5 |
+
|
| 6 |
+
ENV_NAME = "AIOS_FREEZE_NOTICE"
|
| 7 |
+
DEFAULT_TEXT = "Data input is frozen for maintenance."
|
| 8 |
+
REFUSAL_TEXT = "Data input is frozen for maintenance. Try again shortly."
|
| 9 |
+
ERROR_CODE = "data_input_frozen"
|
| 10 |
+
STATUS_CODE = 423
|
| 11 |
+
_REFUSAL = ContextVar("aios_freeze_refusal", default=None)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class DataInputFrozen(RuntimeError):
|
| 15 |
+
"""A maintenance freeze refused a durable mutation before acceptance."""
|
| 16 |
+
|
| 17 |
+
def __init__(self):
|
| 18 |
+
refusal = _REFUSAL.get()
|
| 19 |
+
if refusal is not None:
|
| 20 |
+
refusal[0] = True
|
| 21 |
+
super().__init__(REFUSAL_TEXT)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@contextmanager
|
| 25 |
+
def refusal_scope():
|
| 26 |
+
"""Track a refusal through route wrappers and copied threadpool contexts."""
|
| 27 |
+
refusal = [False]
|
| 28 |
+
token = _REFUSAL.set(refusal)
|
| 29 |
+
try:
|
| 30 |
+
yield refusal
|
| 31 |
+
finally:
|
| 32 |
+
_REFUSAL.reset(token)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def enabled():
|
| 36 |
+
return bool(os.environ.get(ENV_NAME))
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def notice():
|
| 40 |
+
"""Return only the public on/off state and notice text."""
|
| 41 |
+
value = str(os.environ.get(ENV_NAME) or "")
|
| 42 |
+
text = DEFAULT_TEXT if value.lower() in {"1", "on"} else value
|
| 43 |
+
return {"enabled": bool(value), "text": text}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def check_write():
|
| 47 |
+
"""Refuse before any store permission check, cache mutation or network call."""
|
| 48 |
+
if enabled():
|
| 49 |
+
raise DataInputFrozen()
|
platform/core/grid_events.py
CHANGED
|
@@ -1181,7 +1181,7 @@ def _refuse(ctx, code, message, key='', *, event='field_upsert'):
|
|
| 1181 |
#: the same 24-event window still overwrites it, because that branch assigns the slot directly.
|
| 1182 |
#: Widening `EventResult` to a message LIST is the real fix and it is not this ticket's file.
|
| 1183 |
def _folder_refuse(ctx, kind, code, message, key=''):
|
| 1184 |
-
"""Refuse
|
| 1185 |
try:
|
| 1186 |
out = getattr(ctx, 'out', None)
|
| 1187 |
if out is not None:
|
|
@@ -1489,6 +1489,12 @@ def handle_one(event, ctx):
|
|
| 1489 |
requested_name = name
|
| 1490 |
if not view_id or not name:
|
| 1491 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1492 |
if 'permissions' in raw:
|
| 1493 |
return _refuse(
|
| 1494 |
ctx, 'view_permissions_retired',
|
|
@@ -3350,8 +3356,12 @@ def handle_one(event, ctx):
|
|
| 3350 |
|
| 3351 |
if kind == 'overlay_patch':
|
| 3352 |
pid = event.get('pid')
|
| 3353 |
-
if not isinstance(pid, int) or pid
|
| 3354 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3355 |
_history_table = str(ctx.scope_key or 'customer')
|
| 3356 |
if _history_table.startswith('ut_') and ctx.table is not None:
|
| 3357 |
import core.user_tables as _ut_history
|
|
|
|
| 1181 |
#: the same 24-event window still overwrites it, because that branch assigns the slot directly.
|
| 1182 |
#: Widening `EventResult` to a message LIST is the real fix and it is not this ticket's file.
|
| 1183 |
def _folder_refuse(ctx, kind, code, message, key=''):
|
| 1184 |
+
"""Refuse one event out loud: the machine record AND the rendered sentence."""
|
| 1185 |
try:
|
| 1186 |
out = getattr(ctx, 'out', None)
|
| 1187 |
if out is not None:
|
|
|
|
| 1489 |
requested_name = name
|
| 1490 |
if not view_id or not name:
|
| 1491 |
return False
|
| 1492 |
+
display = raw['config'].get('display')
|
| 1493 |
+
if isinstance(display, dict) and display.get('mode') == 'timeseries':
|
| 1494 |
+
return _folder_refuse(
|
| 1495 |
+
ctx, kind, 'view_mode_retired',
|
| 1496 |
+
'Time-series Views are retired. Choose an available View display mode.',
|
| 1497 |
+
key=view_id)
|
| 1498 |
if 'permissions' in raw:
|
| 1499 |
return _refuse(
|
| 1500 |
ctx, 'view_permissions_retired',
|
|
|
|
| 3356 |
|
| 3357 |
if kind == 'overlay_patch':
|
| 3358 |
pid = event.get('pid')
|
| 3359 |
+
if not isinstance(pid, int) or isinstance(pid, bool):
|
| 3360 |
return False
|
| 3361 |
+
if pid not in allowed_pids:
|
| 3362 |
+
return _folder_refuse(
|
| 3363 |
+
ctx, kind, 'row_outside_audience',
|
| 3364 |
+
'This edit was not applied because the record is outside your permitted audience.')
|
| 3365 |
_history_table = str(ctx.scope_key or 'customer')
|
| 3366 |
if _history_table.startswith('ut_') and ctx.table is not None:
|
| 3367 |
import core.user_tables as _ut_history
|
platform/core/keychain.py
CHANGED
|
@@ -253,10 +253,13 @@ def next_slot(rt, etype):
|
|
| 253 |
return top + 1
|
| 254 |
|
| 255 |
|
| 256 |
-
def add_entry(rt, label, etype, fields, username):
|
| 257 |
"""Encrypt + store. Returns the metadata row, or raises ValueError on shape problems and
|
| 258 |
KeychainLocked when there is no key (a secret must never be stored in the clear).
|
| 259 |
|
|
|
|
|
|
|
|
|
|
| 260 |
⭐ W54-T44 / C4 — the row is stamped with its `connection_id` and its `slot`, and a caller
|
| 261 |
that supplies NO label gets the default the slot names: the tenant's first Odoo connection is
|
| 262 |
`Odoo 1`, the second `Odoo 2`. An explicitly supplied label always wins, and the label stays
|
|
@@ -268,6 +271,13 @@ def add_entry(rt, label, etype, fields, username):
|
|
| 268 |
etype = str(etype or '').strip().lower()
|
| 269 |
if etype not in ENTRY_TYPES:
|
| 270 |
raise ValueError(f"type must be one of {', '.join(sorted(ENTRY_TYPES))}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
if etype == 'amazon_sp' and not isinstance(fields, dict):
|
| 272 |
fields = {}
|
| 273 |
if not isinstance(fields, dict) or (not fields and etype != 'amazon_sp'):
|
|
@@ -305,14 +315,33 @@ def add_entry(rt, label, etype, fields, username):
|
|
| 305 |
if etype == 'amazon_sp':
|
| 306 |
row[AMAZON_ROTATION_DEADLINE_FIELD] = deadline
|
| 307 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
def _up(cur):
|
| 309 |
entries = cur.setdefault('entries', {})
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
if len(entries) >= MAX_ENTRIES:
|
| 311 |
raise ValueError(f'this keychain is at its {MAX_ENTRIES}-entry cap')
|
| 312 |
entries[eid] = row
|
| 313 |
return cur
|
| 314 |
|
| 315 |
-
rt.update(KEY, _up, flush='sync')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
return {'id': eid, 'connection_id': eid, 'slot': slot,
|
| 317 |
'label': label, 'type': etype, 'preview': preview,
|
| 318 |
'created': row['created'], 'createdBy': row['createdBy']}
|
|
@@ -451,14 +480,14 @@ def odoo_creds(rt, connection_id=None):
|
|
| 451 |
|
| 452 |
|
| 453 |
def meta_creds(rt, connection_id=None):
|
| 454 |
-
"""
|
| 455 |
|
| 456 |
⭐⭐ W49-T13 / D-202 — IT HAS A PRODUCTION CALLER NOW, AND THAT WAS THE WHOLE DEFECT.
|
| 457 |
`harness/meta_store.py::token` read `META_ADS_ACCESS_TOKEN` unconditionally, so this resolver
|
| 458 |
existed, was correct, and was reached by nothing: a tenant's stored Meta credential was
|
| 459 |
accepted, previewed and testable while every sync ran on the deployment's own token. The
|
| 460 |
-
loader now asks here FIRST
|
| 461 |
-
because "nothing calls it" would have been the wrong repair
|
| 462 |
|
| 463 |
⛔ IT IS DELIBERATELY THE SAME SHAPE AS `odoo_creds` AND SHARES ITS IMPLEMENTATION, because
|
| 464 |
"which stored credential serves this connector?" is ONE question and this module was one
|
|
@@ -466,11 +495,10 @@ def meta_creds(rt, connection_id=None):
|
|
| 466 |
difference between the two connectors belongs in the CONNECTOR, not in the lookup.
|
| 467 |
|
| 468 |
⚠ AND IT FAILS CLOSED WITH NO ENVIRONMENT FALLBACK, which is where it deliberately DIFFERS
|
| 469 |
-
from
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
exists to prevent. A tenant with no entry gets None, and the connector says so.
|
| 474 |
|
| 475 |
⭐ W54-T45 / R4 — `connection_id` NAMES WHICH Meta connection, and no id means the tenant's
|
| 476 |
DESIGNATED DEFAULT (the lowest-`slot` `meta_ads` connection), not "whichever row sorts first".
|
|
|
|
| 253 |
return top + 1
|
| 254 |
|
| 255 |
|
| 256 |
+
def add_entry(rt, label, etype, fields, username, *, scope=None, write_scope=None):
|
| 257 |
"""Encrypt + store. Returns the metadata row, or raises ValueError on shape problems and
|
| 258 |
KeychainLocked when there is no key (a secret must never be stored in the clear).
|
| 259 |
|
| 260 |
+
A scoped setup writer supplies `scope` and the API's `write_scope` callback. Personal
|
| 261 |
+
ownership commits before the credential; callers omitting these keep their existing path.
|
| 262 |
+
|
| 263 |
⭐ W54-T44 / C4 — the row is stamped with its `connection_id` and its `slot`, and a caller
|
| 264 |
that supplies NO label gets the default the slot names: the tenant's first Odoo connection is
|
| 265 |
`Odoo 1`, the second `Odoo 2`. An explicitly supplied label always wins, and the label stays
|
|
|
|
| 271 |
etype = str(etype or '').strip().lower()
|
| 272 |
if etype not in ENTRY_TYPES:
|
| 273 |
raise ValueError(f"type must be one of {', '.join(sorted(ENTRY_TYPES))}")
|
| 274 |
+
if scope is not None and scope not in ('business', 'personal'):
|
| 275 |
+
raise ValueError('invalid credential scope')
|
| 276 |
+
if scope == 'personal':
|
| 277 |
+
if etype in TENANT_WIDE_TYPES:
|
| 278 |
+
raise ValueError('this connector requires business scope')
|
| 279 |
+
if not username or not callable(write_scope):
|
| 280 |
+
raise ValueError('personal scope requires an owner and a scoped writer')
|
| 281 |
if etype == 'amazon_sp' and not isinstance(fields, dict):
|
| 282 |
fields = {}
|
| 283 |
if not isinstance(fields, dict) or (not fields and etype != 'amazon_sp'):
|
|
|
|
| 315 |
if etype == 'amazon_sp':
|
| 316 |
row[AMAZON_ROTATION_DEADLINE_FIELD] = deadline
|
| 317 |
|
| 318 |
+
# Setup links publish a personal entry only AFTER its existing scope writer has
|
| 319 |
+
# committed the ownership row. If either write fails, keep that reservation:
|
| 320 |
+
# deleting it could turn an uncertain credential commit into a business entry.
|
| 321 |
+
if scope == 'personal':
|
| 322 |
+
if not write_scope(rt, eid, scope, username):
|
| 323 |
+
raise ValueError('credential scope could not be saved')
|
| 324 |
+
scopes = rt.get(SCOPE_KEY)
|
| 325 |
+
saved = scopes.get(eid) if isinstance(scopes, dict) else None
|
| 326 |
+
if saved != {'scope': 'personal', 'owner': str(username)}:
|
| 327 |
+
raise ValueError('credential scope could not be saved')
|
| 328 |
+
|
| 329 |
def _up(cur):
|
| 330 |
entries = cur.setdefault('entries', {})
|
| 331 |
+
if eid in entries:
|
| 332 |
+
if entries[eid] == row:
|
| 333 |
+
return cur
|
| 334 |
+
raise ValueError('credential identity already exists')
|
| 335 |
if len(entries) >= MAX_ENTRIES:
|
| 336 |
raise ValueError(f'this keychain is at its {MAX_ENTRIES}-entry cap')
|
| 337 |
entries[eid] = row
|
| 338 |
return cur
|
| 339 |
|
| 340 |
+
stored = rt.update(KEY, _up, flush='sync')
|
| 341 |
+
if scope is not None:
|
| 342 |
+
saved_entries = stored.get('entries') if isinstance(stored, dict) else None
|
| 343 |
+
if not isinstance(saved_entries, dict) or saved_entries.get(eid) != row:
|
| 344 |
+
raise RuntimeError('credential write was not confirmed')
|
| 345 |
return {'id': eid, 'connection_id': eid, 'slot': slot,
|
| 346 |
'label': label, 'type': etype, 'preview': preview,
|
| 347 |
'created': row['created'], 'createdBy': row['createdBy']}
|
|
|
|
| 480 |
|
| 481 |
|
| 482 |
def meta_creds(rt, connection_id=None):
|
| 483 |
+
"""The named or designated default `meta_ads` connection's fields, else None.
|
| 484 |
|
| 485 |
⭐⭐ W49-T13 / D-202 — IT HAS A PRODUCTION CALLER NOW, AND THAT WAS THE WHOLE DEFECT.
|
| 486 |
`harness/meta_store.py::token` read `META_ADS_ACCESS_TOKEN` unconditionally, so this resolver
|
| 487 |
existed, was correct, and was reached by nothing: a tenant's stored Meta credential was
|
| 488 |
accepted, previewed and testable while every sync ran on the deployment's own token. The
|
| 489 |
+
loader now asks here FIRST; its environment fallback is restricted to Royal Imports.
|
| 490 |
+
Deleting this function because "nothing calls it" would have been the wrong repair.
|
| 491 |
|
| 492 |
⛔ IT IS DELIBERATELY THE SAME SHAPE AS `odoo_creds` AND SHARES ITS IMPLEMENTATION, because
|
| 493 |
"which stored credential serves this connector?" is ONE question and this module was one
|
|
|
|
| 495 |
difference between the two connectors belongs in the CONNECTOR, not in the lookup.
|
| 496 |
|
| 497 |
⚠ AND IT FAILS CLOSED WITH NO ENVIRONMENT FALLBACK, which is where it deliberately DIFFERS
|
| 498 |
+
from its caller: environment fallback is a Royal-only connector decision, never a keychain
|
| 499 |
+
lookup decision. `META_ADS_ACCESS_TOKEN` is the OWNER's measurement token (R8), and handing
|
| 500 |
+
it to another tenant is precisely the leak the R3 rule exists to prevent. A tenant with no
|
| 501 |
+
entry gets None here.
|
|
|
|
| 502 |
|
| 503 |
⭐ W54-T45 / R4 — `connection_id` NAMES WHICH Meta connection, and no id means the tenant's
|
| 504 |
DESIGNATED DEFAULT (the lowest-`slot` `meta_ads` connection), not "whichever row sorts first".
|
platform/core/route_system.py
CHANGED
|
@@ -591,7 +591,13 @@ def reconcile_inbox_tasks(st, *, today=None):
|
|
| 591 |
|
| 592 |
|
| 593 |
def inbox_tasks(st, assignee, *, today=None):
|
| 594 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 595 |
if str(task.get("assignee") or "").lower() == str(assignee or "").lower()]
|
| 596 |
|
| 597 |
|
|
|
|
| 591 |
|
| 592 |
|
| 593 |
def inbox_tasks(st, assignee, *, today=None):
|
| 594 |
+
from core import freeze_notice
|
| 595 |
+
if freeze_notice.enabled():
|
| 596 |
+
# Reconcile only the detached read; durable receipts remain unchanged.
|
| 597 |
+
tasks = _reconcile_inbox_state(copy.deepcopy(read(st)), _inbox_day(today, st=st))
|
| 598 |
+
else:
|
| 599 |
+
tasks = reconcile_inbox_tasks(st, today=today)
|
| 600 |
+
return [task for task in tasks
|
| 601 |
if str(task.get("assignee") or "").lower() == str(assignee or "").lower()]
|
| 602 |
|
| 603 |
|
platform/core/setup_links.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Single-use connector setup capabilities. Credential writes are supplied by the API.
|
| 2 |
+
|
| 3 |
+
The forms bucket holds only a token digest and server-owned metadata. Store callbacks
|
| 4 |
+
may be replayed after a conflict, so claiming is idempotent for its own operation and
|
| 5 |
+
refuses every competing operation before any credential writer can run.
|
| 6 |
+
"""
|
| 7 |
+
import hashlib
|
| 8 |
+
import math
|
| 9 |
+
import re
|
| 10 |
+
import secrets
|
| 11 |
+
import threading
|
| 12 |
+
import time
|
| 13 |
+
from urllib.parse import urlsplit
|
| 14 |
+
|
| 15 |
+
from core import keychain
|
| 16 |
+
|
| 17 |
+
KIND = "connector_setup"
|
| 18 |
+
TTL_SECONDS = 24 * 60 * 60
|
| 19 |
+
_TOKEN = re.compile(r"^[A-Za-z0-9_-]{32}$")
|
| 20 |
+
# Serialize setup metadata mutations within a process too. Durable store.update and
|
| 21 |
+
# its conflict replay enforce the claim across processes; no secret write holds this lock.
|
| 22 |
+
_TOKEN_LOCK = threading.Lock()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class InvalidSetupLink(ValueError):
|
| 26 |
+
pass
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class InvalidSetupValues(ValueError):
|
| 30 |
+
pass
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class SetupWriteFailed(RuntimeError):
|
| 34 |
+
pass
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def digest(token):
|
| 38 |
+
if not isinstance(token, str) or not _TOKEN.fullmatch(token):
|
| 39 |
+
return ""
|
| 40 |
+
return "setup:" + hashlib.sha256(token.encode("ascii")).hexdigest()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def public_origin(value):
|
| 44 |
+
"""Accept a configured HTTPS origin, never a caller-supplied host or relative URL."""
|
| 45 |
+
value = str(value or "").strip().rstrip("/")
|
| 46 |
+
try:
|
| 47 |
+
parts = urlsplit(value)
|
| 48 |
+
port = parts.port
|
| 49 |
+
except ValueError:
|
| 50 |
+
raise InvalidSetupLink() from None
|
| 51 |
+
if (parts.scheme != "https" or not parts.hostname or parts.username is not None
|
| 52 |
+
or parts.password is not None or parts.path or parts.query or parts.fragment
|
| 53 |
+
or any(ch.isspace() or ord(ch) < 32 for ch in value) or "\\" in value
|
| 54 |
+
or port == 0):
|
| 55 |
+
raise InvalidSetupLink()
|
| 56 |
+
return value
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _target_ok(connector, scope):
|
| 60 |
+
return (isinstance(connector, str) and connector in keychain.ENTRY_TYPES
|
| 61 |
+
and scope in ("personal", "business")
|
| 62 |
+
and not (scope == "personal" and connector in keychain.TENANT_WIDE_TYPES))
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def read_record(index, token, tenant, *, now=None):
|
| 66 |
+
row = index.get(digest(token)) if isinstance(index, dict) and digest(token) else None
|
| 67 |
+
if not isinstance(row, dict):
|
| 68 |
+
return None
|
| 69 |
+
expiry = row.get("expiresAt")
|
| 70 |
+
now = time.time() if now is None else now
|
| 71 |
+
if (row.get("kind") != KIND or row.get("status") != "ready"
|
| 72 |
+
or row.get("tenant") != tenant or not row.get("owner")
|
| 73 |
+
or not _target_ok(row.get("connector"), row.get("scope"))
|
| 74 |
+
or type(expiry) not in (int, float) or not math.isfinite(expiry)
|
| 75 |
+
or now >= expiry):
|
| 76 |
+
return None
|
| 77 |
+
return {key: row[key] for key in
|
| 78 |
+
("kind", "tenant", "owner", "connector", "scope", "expiresAt", "status")}
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def mint(rt, *, tokens_key, tenant, owner, connector, scope, origin, now=None):
|
| 82 |
+
origin = public_origin(origin)
|
| 83 |
+
if not tenant or not owner or not _target_ok(connector, scope):
|
| 84 |
+
raise InvalidSetupLink()
|
| 85 |
+
now = time.time() if now is None else now
|
| 86 |
+
token = secrets.token_urlsafe(24)
|
| 87 |
+
hashed = digest(token)
|
| 88 |
+
row = {"kind": KIND, "tenant": tenant, "owner": owner, "connector": connector,
|
| 89 |
+
"scope": scope, "expiresAt": now + TTL_SECONDS, "status": "ready"}
|
| 90 |
+
|
| 91 |
+
def _mint(cur):
|
| 92 |
+
cur = dict(cur or {})
|
| 93 |
+
if hashed in cur and cur[hashed] != row:
|
| 94 |
+
raise InvalidSetupLink()
|
| 95 |
+
cur[hashed] = dict(row)
|
| 96 |
+
return cur
|
| 97 |
+
|
| 98 |
+
with _TOKEN_LOCK:
|
| 99 |
+
stored = rt.update(tokens_key, _mint, flush="sync")
|
| 100 |
+
if not isinstance(stored, dict) or stored.get(hashed) != row:
|
| 101 |
+
raise InvalidSetupLink()
|
| 102 |
+
return {"url": f"{origin}/#/form/{token}", "expiresAt": row["expiresAt"],
|
| 103 |
+
"connector": connector, "scope": scope}
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def clean_values(field_specs, values):
|
| 107 |
+
"""Use the supplied connector schema; never echo a submitted value in an error."""
|
| 108 |
+
if not isinstance(values, dict) or not field_specs:
|
| 109 |
+
raise InvalidSetupValues()
|
| 110 |
+
keys = {field["key"] for field in field_specs}
|
| 111 |
+
if set(values) - keys:
|
| 112 |
+
raise InvalidSetupValues()
|
| 113 |
+
clean = {}
|
| 114 |
+
for field in field_specs:
|
| 115 |
+
key = field["key"]
|
| 116 |
+
value = values.get(key, "")
|
| 117 |
+
if (not isinstance(value, str) or len(value) > keychain.MAX_FIELD_LEN
|
| 118 |
+
or (field.get("required", True) and not value.strip())):
|
| 119 |
+
raise InvalidSetupValues()
|
| 120 |
+
if value:
|
| 121 |
+
clean[key] = value
|
| 122 |
+
return clean
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def submit(rt, token, record, values, *, tokens_key, field_specs, write_credentials):
|
| 126 |
+
clean = clean_values(field_specs, values)
|
| 127 |
+
hashed, claim_id = digest(token), secrets.token_hex(16)
|
| 128 |
+
|
| 129 |
+
def _claim(cur):
|
| 130 |
+
cur = dict(cur or {})
|
| 131 |
+
current = cur.get(hashed)
|
| 132 |
+
# HF conflict replay of THIS successful claim is harmless; another claimant refuses.
|
| 133 |
+
if (isinstance(current, dict) and current.get("status") == "used"
|
| 134 |
+
and current.get("claimId") == claim_id):
|
| 135 |
+
return cur
|
| 136 |
+
fresh = read_record(cur, token, record["tenant"])
|
| 137 |
+
if fresh is None or fresh != record:
|
| 138 |
+
raise InvalidSetupLink()
|
| 139 |
+
cur[hashed] = {**fresh, "status": "used", "claimId": claim_id}
|
| 140 |
+
return cur
|
| 141 |
+
|
| 142 |
+
try:
|
| 143 |
+
with _TOKEN_LOCK:
|
| 144 |
+
stored = rt.update(tokens_key, _claim, flush="sync")
|
| 145 |
+
claimed = stored.get(hashed) if isinstance(stored, dict) else None
|
| 146 |
+
if (not isinstance(claimed, dict) or claimed.get("claimId") != claim_id
|
| 147 |
+
or claimed.get("status") != "used"):
|
| 148 |
+
raise SetupWriteFailed()
|
| 149 |
+
except InvalidSetupLink:
|
| 150 |
+
clean.clear()
|
| 151 |
+
raise
|
| 152 |
+
except Exception:
|
| 153 |
+
clean.clear()
|
| 154 |
+
raise SetupWriteFailed() from None
|
| 155 |
+
# A failed or uncertain write must never revive this capability or claim success.
|
| 156 |
+
try:
|
| 157 |
+
write_credentials(record, clean)
|
| 158 |
+
except Exception:
|
| 159 |
+
raise SetupWriteFailed() from None
|
| 160 |
+
finally:
|
| 161 |
+
clean.clear()
|
| 162 |
+
return {"ok": True, "kind": KIND}
|
platform/core/shared_overlay.py
CHANGED
|
@@ -64,6 +64,10 @@ BUCKET_SUFFIX = '__shared'
|
|
| 64 |
#: in the tenant, so an unbounded one is an unbounded cost for all of them.
|
| 65 |
MAX_VALUE_CHARS = 32_768
|
| 66 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
# ══════════════════════════ W41-T04 / CONTRACT C3 — ONE COLUMN-CREATION DOOR ══════════════════
|
| 68 |
#
|
| 69 |
# ⭐⭐ WHAT THIS ALLOWLIST IS FOR, AND WHY IT IS *HERE* RATHER THAN AT EACH CALLER. Before it,
|
|
@@ -141,6 +145,15 @@ def _read(table_key, st=None):
|
|
| 141 |
"""The whole stratum, always in its full two-key shape. Lenient like every other display
|
| 142 |
read: an unreachable store degrades to "nothing shared yet", never to an exception on a page
|
| 143 |
that would otherwise render."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
try:
|
| 145 |
data = _st(st).get(bucket(table_key)) or {}
|
| 146 |
except Exception:
|
|
@@ -251,11 +264,15 @@ def fold_legacy_bucket(legacy_key, table_key, st=None):
|
|
| 251 |
if not _may_hold_anything(legacy, st=st):
|
| 252 |
return {'fields': 0, 'cells': 0}
|
| 253 |
src = _read(legacy, st=st)
|
| 254 |
-
if not src['fields'] and not src['cells']:
|
| 255 |
return {'fields': 0, 'cells': 0}
|
| 256 |
dst = _read(canon, st=st)
|
| 257 |
add_fields = {key: defn for key, defn in src['fields'].items()
|
| 258 |
if isinstance(defn, dict) and key not in dst['fields']}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
add_cells = {}
|
| 260 |
for pid, values in src['cells'].items():
|
| 261 |
if not isinstance(values, dict):
|
|
@@ -299,13 +316,21 @@ def _value(value):
|
|
| 299 |
⛔ A dict or a list RAISES rather than being dropped. This is called by our own code, so a
|
| 300 |
non-scalar is a programming error, and silently storing nothing would surface later as "the
|
| 301 |
shared column is blank for everyone" with no failure anywhere near the cause.
|
|
|
|
|
|
|
|
|
|
| 302 |
"""
|
| 303 |
if isinstance(value, (dict, list, tuple, set)):
|
| 304 |
raise ValueError(f'shared_overlay: a cell holds a scalar, not {type(value).__name__}')
|
| 305 |
if value is None or isinstance(value, bool) or isinstance(value, (int, float)):
|
| 306 |
return value
|
| 307 |
text = str(value)
|
| 308 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
|
| 310 |
|
| 311 |
# ----------------------------------------------------------------- the column DEFINITIONS
|
|
@@ -553,7 +578,7 @@ def snapshot(table_key, pids, st=None):
|
|
| 553 |
|
| 554 |
|
| 555 |
def put_cell(table_key, pid, field_key, value, st=None):
|
| 556 |
-
"""Write ONE shared cell.
|
| 557 |
return put_cells(table_key, pid, {field_key: value}, st=st).get(str(field_key or ''))
|
| 558 |
|
| 559 |
|
|
|
|
| 64 |
#: in the tenant, so an unbounded one is an unbounded cost for all of them.
|
| 65 |
MAX_VALUE_CHARS = 32_768
|
| 66 |
|
| 67 |
+
|
| 68 |
+
class SchemaUnavailable(RuntimeError):
|
| 69 |
+
"""Definitions cannot be answered without an unfinished full-workspace migration."""
|
| 70 |
+
|
| 71 |
# ══════════════════════════ W41-T04 / CONTRACT C3 — ONE COLUMN-CREATION DOOR ══════════════════
|
| 72 |
#
|
| 73 |
# ⭐⭐ WHAT THIS ALLOWLIST IS FOR, AND WHY IT IS *HERE* RATHER THAN AT EACH CALLER. Before it,
|
|
|
|
| 145 |
"""The whole stratum, always in its full two-key shape. Lenient like every other display
|
| 146 |
read: an unreachable store degrades to "nothing shared yet", never to an exception on a page
|
| 147 |
that would otherwise render."""
|
| 148 |
+
if getattr(_st(st), 'schema_only', False):
|
| 149 |
+
reader = getattr(_st(st), 'get_entry', None)
|
| 150 |
+
if not callable(reader):
|
| 151 |
+
raise SchemaUnavailable('field definitions cannot be read without loading row cells')
|
| 152 |
+
fields = reader(bucket(table_key), 'fields')
|
| 153 |
+
if fields is not None and not isinstance(fields, dict):
|
| 154 |
+
raise SchemaUnavailable('the stored field definitions are unavailable')
|
| 155 |
+
# Asking this metadata document for cells must remain a loud programming error.
|
| 156 |
+
return store._Projected({'fields': dict(fields or {})}, ('cells',))
|
| 157 |
try:
|
| 158 |
data = _st(st).get(bucket(table_key)) or {}
|
| 159 |
except Exception:
|
|
|
|
| 264 |
if not _may_hold_anything(legacy, st=st):
|
| 265 |
return {'fields': 0, 'cells': 0}
|
| 266 |
src = _read(legacy, st=st)
|
| 267 |
+
if not src['fields'] and (getattr(_st(st), 'schema_only', False) or not src['cells']):
|
| 268 |
return {'fields': 0, 'cells': 0}
|
| 269 |
dst = _read(canon, st=st)
|
| 270 |
add_fields = {key: defn for key, defn in src['fields'].items()
|
| 271 |
if isinstance(defn, dict) and key not in dst['fields']}
|
| 272 |
+
if getattr(_st(st), 'schema_only', False):
|
| 273 |
+
if add_fields:
|
| 274 |
+
raise SchemaUnavailable('field definitions need a full workspace refresh to finish migration')
|
| 275 |
+
return {'fields': 0, 'cells': 0}
|
| 276 |
add_cells = {}
|
| 277 |
for pid, values in src['cells'].items():
|
| 278 |
if not isinstance(values, dict):
|
|
|
|
| 316 |
⛔ A dict or a list RAISES rather than being dropped. This is called by our own code, so a
|
| 317 |
non-scalar is a programming error, and silently storing nothing would surface later as "the
|
| 318 |
shared column is blank for everyone" with no failure anywhere near the cause.
|
| 319 |
+
|
| 320 |
+
An oversized string refuses before any mutation. The shared-cell HTTP door already relays
|
| 321 |
+
ValueError as a sentence; cutting here would silently change text and could corrupt JSON.
|
| 322 |
"""
|
| 323 |
if isinstance(value, (dict, list, tuple, set)):
|
| 324 |
raise ValueError(f'shared_overlay: a cell holds a scalar, not {type(value).__name__}')
|
| 325 |
if value is None or isinstance(value, bool) or isinstance(value, (int, float)):
|
| 326 |
return value
|
| 327 |
text = str(value)
|
| 328 |
+
if len(text) > MAX_VALUE_CHARS:
|
| 329 |
+
raise ValueError(
|
| 330 |
+
f"This cell value has {len(text):,} characters and would be truncated at the "
|
| 331 |
+
f"{MAX_VALUE_CHARS:,}-character cell limit. It was not saved. "
|
| 332 |
+
"Shorten the value or split it across cells and try again.")
|
| 333 |
+
return text
|
| 334 |
|
| 335 |
|
| 336 |
# ----------------------------------------------------------------- the column DEFINITIONS
|
|
|
|
| 578 |
|
| 579 |
|
| 580 |
def put_cell(table_key, pid, field_key, value, st=None):
|
| 581 |
+
"""Write ONE shared cell. Oversized values refuse before writing; accepted values are exact."""
|
| 582 |
return put_cells(table_key, pid, {field_key: value}, st=st).get(str(field_key or ''))
|
| 583 |
|
| 584 |
|
platform/core/sheet_import.py
CHANGED
|
@@ -987,15 +987,20 @@ def _address_in_document(ref):
|
|
| 987 |
|
| 988 |
|
| 989 |
def _guard_worksheet_xml(stream):
|
| 990 |
-
"""Stream one worksheet and
|
| 991 |
|
| 992 |
ElementTree sees only bounded, central-directory-admitted bytes. It is used here as an
|
| 993 |
admission parser, not as the workbook parser: no cell value, formula or style is interpreted.
|
| 994 |
"""
|
| 995 |
dimension_seen = False
|
| 996 |
cells = 0
|
|
|
|
|
|
|
| 997 |
try:
|
| 998 |
-
for
|
|
|
|
|
|
|
|
|
|
| 999 |
tag = str(element.tag).rsplit("}", 1)[-1]
|
| 1000 |
if tag == "dimension":
|
| 1001 |
ref = str(element.get("ref") or "").replace("$", "").upper()
|
|
@@ -1004,7 +1009,9 @@ def _guard_worksheet_xml(stream):
|
|
| 1004 |
or (len(parts) == 2 and rect_of(ref) is None)):
|
| 1005 |
_archive_refusal(
|
| 1006 |
"import_declared_dimension_invalid",
|
| 1007 |
-
"A worksheet declares cells outside Excel's supported dimensions
|
|
|
|
|
|
|
| 1008 |
dimension_seen = True
|
| 1009 |
elif tag == "c":
|
| 1010 |
cells += 1
|
|
@@ -1012,13 +1019,39 @@ def _guard_worksheet_xml(stream):
|
|
| 1012 |
if not _address_in_document(ref):
|
| 1013 |
_archive_refusal(
|
| 1014 |
"import_declared_dimension_invalid",
|
| 1015 |
-
"A worksheet contains a cell outside Excel's supported dimensions
|
|
|
|
|
|
|
| 1016 |
if cells > sheets._MAX_CELLS_PER_SHEET:
|
| 1017 |
_archive_refusal(
|
| 1018 |
"sheet_too_many_cells",
|
| 1019 |
"A worksheet declares more than " + str(sheets._MAX_CELLS_PER_SHEET)
|
| 1020 |
-
+ " populated cells, so it cannot be imported."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1021 |
element.clear()
|
|
|
|
|
|
|
| 1022 |
except ImportRefusal:
|
| 1023 |
raise
|
| 1024 |
except ET.ParseError as exc:
|
|
@@ -1040,32 +1073,42 @@ def guard_import_archive(path):
|
|
| 1040 |
raise ImportRefusal(
|
| 1041 |
"import_too_large",
|
| 1042 |
"That file is larger than the " + str(MAX_IMPORT_BYTES // (1024 * 1024))
|
| 1043 |
-
+ " MB import limit, so it cannot be read.")
|
| 1044 |
try:
|
| 1045 |
with zipfile.ZipFile(path) as archive:
|
| 1046 |
members = archive.infolist()
|
| 1047 |
if len(members) > MAX_ARCHIVE_MEMBERS:
|
| 1048 |
_archive_refusal("import_zip_bloat",
|
| 1049 |
-
"That workbook archive
|
|
|
|
| 1050 |
names = [entry.filename for entry in members]
|
| 1051 |
if len(set(names)) != len(names):
|
| 1052 |
_archive_refusal("import_zip_bloat",
|
| 1053 |
-
"That workbook archive repeats a part name, so it cannot be imported
|
|
|
|
| 1054 |
total = sum(max(0, int(entry.file_size)) for entry in members)
|
| 1055 |
if total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
| 1056 |
_archive_refusal("import_zip_bloat",
|
| 1057 |
-
"That workbook expands beyond the
|
|
|
|
|
|
|
| 1058 |
for entry in members:
|
| 1059 |
if entry.flag_bits & 0x1:
|
| 1060 |
_archive_refusal("import_encrypted",
|
| 1061 |
-
"Password protected workbooks cannot be imported."
|
|
|
|
| 1062 |
if entry.file_size > MAX_ARCHIVE_MEMBER_BYTES:
|
| 1063 |
_archive_refusal("import_zip_bloat",
|
| 1064 |
-
"One workbook part expands beyond the
|
|
|
|
|
|
|
| 1065 |
ratio = entry.file_size / max(1, entry.compress_size)
|
| 1066 |
if entry.file_size and ratio > MAX_ARCHIVE_COMPRESSION_RATIO:
|
| 1067 |
_archive_refusal("import_zip_bloat",
|
| 1068 |
-
"That workbook archive
|
|
|
|
|
|
|
|
|
|
| 1069 |
worksheets = [entry for entry in members
|
| 1070 |
if re.fullmatch(r"xl/worksheets/sheet[^/]*\.xml", entry.filename)]
|
| 1071 |
for entry in worksheets:
|
|
|
|
| 987 |
|
| 988 |
|
| 989 |
def _guard_worksheet_xml(stream):
|
| 990 |
+
"""Stream one worksheet and bound authored cells plus merge allocation before openpyxl.
|
| 991 |
|
| 992 |
ElementTree sees only bounded, central-directory-admitted bytes. It is used here as an
|
| 993 |
admission parser, not as the workbook parser: no cell value, formula or style is interpreted.
|
| 994 |
"""
|
| 995 |
dimension_seen = False
|
| 996 |
cells = 0
|
| 997 |
+
merged_cells = 0
|
| 998 |
+
ancestors = []
|
| 999 |
try:
|
| 1000 |
+
for event, element in ET.iterparse(stream, events=("start", "end")):
|
| 1001 |
+
if event == "start":
|
| 1002 |
+
ancestors.append(element)
|
| 1003 |
+
continue
|
| 1004 |
tag = str(element.tag).rsplit("}", 1)[-1]
|
| 1005 |
if tag == "dimension":
|
| 1006 |
ref = str(element.get("ref") or "").replace("$", "").upper()
|
|
|
|
| 1009 |
or (len(parts) == 2 and rect_of(ref) is None)):
|
| 1010 |
_archive_refusal(
|
| 1011 |
"import_declared_dimension_invalid",
|
| 1012 |
+
"A worksheet declares cells outside Excel's supported dimensions of "
|
| 1013 |
+
"1,048,576 rows and 16,384 columns. Remove the out-of-range rows or columns "
|
| 1014 |
+
"in Excel and save the workbook again.")
|
| 1015 |
dimension_seen = True
|
| 1016 |
elif tag == "c":
|
| 1017 |
cells += 1
|
|
|
|
| 1019 |
if not _address_in_document(ref):
|
| 1020 |
_archive_refusal(
|
| 1021 |
"import_declared_dimension_invalid",
|
| 1022 |
+
"A worksheet contains a cell outside Excel's supported dimensions of "
|
| 1023 |
+
"1,048,576 rows and 16,384 columns. Remove the out-of-range rows or columns "
|
| 1024 |
+
"in Excel and save the workbook again.")
|
| 1025 |
if cells > sheets._MAX_CELLS_PER_SHEET:
|
| 1026 |
_archive_refusal(
|
| 1027 |
"sheet_too_many_cells",
|
| 1028 |
"A worksheet declares more than " + str(sheets._MAX_CELLS_PER_SHEET)
|
| 1029 |
+
+ " populated cells, so it cannot be imported. Delete unused cells or "
|
| 1030 |
+
"split the worksheet across tabs and try again.")
|
| 1031 |
+
elif tag == "mergeCell":
|
| 1032 |
+
ref = str(element.get("ref") or "").replace("$", "").upper()
|
| 1033 |
+
parts = ref.split(":")
|
| 1034 |
+
if len(parts) not in (1, 2) or not all(_address_in_document(part) for part in parts):
|
| 1035 |
+
_archive_refusal(
|
| 1036 |
+
"import_declared_dimension_invalid",
|
| 1037 |
+
"A merged range is outside Excel's supported dimensions of "
|
| 1038 |
+
"1,048,576 rows and 16,384 columns. Unmerge that range in Excel and try again.")
|
| 1039 |
+
rect = rect_of(ref)
|
| 1040 |
+
# Normal-mode openpyxl allocates every merged placeholder. A sparse dimension
|
| 1041 |
+
# alone allocates nothing and must remain legal, including Excel's last cell.
|
| 1042 |
+
merged_cells += ((rect["col1"] - rect["col0"] + 1)
|
| 1043 |
+
* (rect["row1"] - rect["row0"] + 1) - 1)
|
| 1044 |
+
if cells + merged_cells > sheets._MAX_CELLS_PER_SHEET:
|
| 1045 |
+
_archive_refusal(
|
| 1046 |
+
"import_merged_cells_too_large",
|
| 1047 |
+
"Opening this worksheet's merged ranges would exceed the "
|
| 1048 |
+
+ str(sheets._MAX_CELLS_PER_SHEET)
|
| 1049 |
+
+ "-cell import limit. Unmerge large ranges in Excel or split the worksheet "
|
| 1050 |
+
"across tabs and try again.")
|
| 1051 |
+
ancestors.pop()
|
| 1052 |
element.clear()
|
| 1053 |
+
if ancestors:
|
| 1054 |
+
ancestors[-1].remove(element)
|
| 1055 |
except ImportRefusal:
|
| 1056 |
raise
|
| 1057 |
except ET.ParseError as exc:
|
|
|
|
| 1073 |
raise ImportRefusal(
|
| 1074 |
"import_too_large",
|
| 1075 |
"That file is larger than the " + str(MAX_IMPORT_BYTES // (1024 * 1024))
|
| 1076 |
+
+ " MB import limit, so it cannot be read. Split it into smaller workbooks and try again.")
|
| 1077 |
try:
|
| 1078 |
with zipfile.ZipFile(path) as archive:
|
| 1079 |
members = archive.infolist()
|
| 1080 |
if len(members) > MAX_ARCHIVE_MEMBERS:
|
| 1081 |
_archive_refusal("import_zip_bloat",
|
| 1082 |
+
f"That workbook archive exceeds the {MAX_ARCHIVE_MEMBERS:,}-part "
|
| 1083 |
+
"import limit. Split it into smaller workbooks and try again.")
|
| 1084 |
names = [entry.filename for entry in members]
|
| 1085 |
if len(set(names)) != len(names):
|
| 1086 |
_archive_refusal("import_zip_bloat",
|
| 1087 |
+
"That workbook archive repeats a part name, so it cannot be imported "
|
| 1088 |
+
"safely. Save it again as .xlsx and try once more.")
|
| 1089 |
total = sum(max(0, int(entry.file_size)) for entry in members)
|
| 1090 |
if total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
| 1091 |
_archive_refusal("import_zip_bloat",
|
| 1092 |
+
"That workbook expands beyond the "
|
| 1093 |
+
+ str(MAX_ARCHIVE_UNCOMPRESSED_BYTES // (1024 * 1024))
|
| 1094 |
+
+ " MB uncompressed import limit. Split it into smaller workbooks and try again.")
|
| 1095 |
for entry in members:
|
| 1096 |
if entry.flag_bits & 0x1:
|
| 1097 |
_archive_refusal("import_encrypted",
|
| 1098 |
+
"Password protected workbooks cannot be imported. Remove the "
|
| 1099 |
+
"password in Excel and save the workbook again.")
|
| 1100 |
if entry.file_size > MAX_ARCHIVE_MEMBER_BYTES:
|
| 1101 |
_archive_refusal("import_zip_bloat",
|
| 1102 |
+
"One workbook part expands beyond the "
|
| 1103 |
+
+ str(MAX_ARCHIVE_MEMBER_BYTES // (1024 * 1024))
|
| 1104 |
+
+ " MB uncompressed part limit. Split it into smaller workbooks and try again.")
|
| 1105 |
ratio = entry.file_size / max(1, entry.compress_size)
|
| 1106 |
if entry.file_size and ratio > MAX_ARCHIVE_COMPRESSION_RATIO:
|
| 1107 |
_archive_refusal("import_zip_bloat",
|
| 1108 |
+
"That workbook archive exceeds the "
|
| 1109 |
+
+ str(MAX_ARCHIVE_COMPRESSION_RATIO)
|
| 1110 |
+
+ " to 1 compression limit. Remove unused repeated content "
|
| 1111 |
+
"or split it into smaller workbooks and try again.")
|
| 1112 |
worksheets = [entry for entry in members
|
| 1113 |
if re.fullmatch(r"xl/worksheets/sheet[^/]*\.xml", entry.filename)]
|
| 1114 |
for entry in worksheets:
|
platform/core/sheets.py
CHANGED
|
@@ -261,7 +261,10 @@ SHEET_CHART_STACKING = ("clustered", "stacked", "percent")
|
|
| 261 |
#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
|
| 262 |
SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
|
| 263 |
"rules", "rowGroups", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
|
| 264 |
-
"visibility", "view", "print", "excel")
|
|
|
|
|
|
|
|
|
|
| 265 |
|
| 266 |
#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
|
| 267 |
#: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab
|
|
@@ -1502,6 +1505,53 @@ def _clean_row_groups(raw, rows):
|
|
| 1502 |
return out or None
|
| 1503 |
|
| 1504 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1505 |
def _clean_sheet(raw, index=0, *, lossless=False):
|
| 1506 |
"""Normalize a tab's known strata and retain additive JSON metadata without interpretation.
|
| 1507 |
|
|
@@ -1532,6 +1582,10 @@ def _clean_sheet(raw, index=0, *, lossless=False):
|
|
| 1532 |
visibility = raw.get("visibility")
|
| 1533 |
if visibility is not None and visibility not in ("visible", "hidden", "veryHidden"):
|
| 1534 |
raise SheetRefusal("sheet_visibility_unknown", "A sheet must be visible, hidden or very hidden.")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1535 |
|
| 1536 |
#: ⚠ THE ORDER OF THIS DICT IS THE ORDER THE CLEANERS RUN IN, and it is load bearing: two of
|
| 1537 |
#: them REFUSE by name (`_clean_rules` on an unknown op, `_clean_imported` on a record that is
|
|
@@ -1552,6 +1606,9 @@ def _clean_sheet(raw, index=0, *, lossless=False):
|
|
| 1552 |
"view": _clean_view(raw.get("view")),
|
| 1553 |
"print": _clean_print(raw.get("print")),
|
| 1554 |
"excel": excel,
|
|
|
|
|
|
|
|
|
|
| 1555 |
}
|
| 1556 |
|
| 1557 |
#: ⛔ THE WALK IS THROUGH `SHEET_TAB_KEYS`, NOT THROUGH `optional`, AND THAT IS THE WHOLE
|
|
@@ -1839,7 +1896,7 @@ def put_workbook(st, workbook_id, raw, expect_rev=None, *, actor="", actor_name=
|
|
| 1839 |
if expect_rev is not None and int(expect_rev) < int(prior.get("structureRev") or 0):
|
| 1840 |
raise SheetStaleRefusal(
|
| 1841 |
"sheet_structure_stale",
|
| 1842 |
-
"Rows or columns changed since you opened this workbook. This save was not applied.
|
| 1843 |
rev_current=int(prior.get("rev") or 0))
|
| 1844 |
if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev):
|
| 1845 |
raise SheetStaleRefusal(
|
|
@@ -2167,12 +2224,12 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None, *, actor="
|
|
| 2167 |
if revision < int(prior.get("structureRev") or 0):
|
| 2168 |
raise SheetStaleRefusal(
|
| 2169 |
"sheet_structure_stale",
|
| 2170 |
-
"Rows or columns changed since you opened this workbook. This save was not applied.
|
| 2171 |
rev_current=current)
|
| 2172 |
if revision < int(tab.get("writesFrom") or 0):
|
| 2173 |
raise SheetStaleRefusal(
|
| 2174 |
"sheet_revision_stale",
|
| 2175 |
-
"This tab's retained change history was truncated after your revision.
|
| 2176 |
rev_current=current)
|
| 2177 |
for write in writes:
|
| 2178 |
if int(write["rev"]) > revision and write["by"] != actor:
|
|
@@ -2421,7 +2478,7 @@ def narrow_workbook(st, workbook_id):
|
|
| 2421 |
state.setdefault("access", "private")
|
| 2422 |
return state, state
|
| 2423 |
|
| 2424 |
-
#
|
| 2425 |
book = _persist(st, workbook_key(wid), _apply)
|
| 2426 |
_index_upsert(st, book)
|
| 2427 |
|
|
@@ -2477,21 +2534,20 @@ def workbook_role(st, book, viewer, *, is_admin=False, rec=None):
|
|
| 2477 |
|
| 2478 |
|
| 2479 |
def may_read_workbook(book, role):
|
| 2480 |
-
"""
|
| 2481 |
|
| 2482 |
-
|
| 2483 |
-
|
| 2484 |
-
Legacy visibility grants no ownership or editing role.
|
| 2485 |
"""
|
| 2486 |
-
return
|
| 2487 |
|
| 2488 |
|
| 2489 |
def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=None,
|
| 2490 |
grant_records=None):
|
| 2491 |
"""Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched.
|
| 2492 |
|
| 2493 |
-
|
| 2494 |
-
The route
|
| 2495 |
`list_workbooks` itself stays a bare pointer list on purpose:
|
| 2496 |
`iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry
|
| 2497 |
read per workbook there would put a permission lookup inside a scan that asks no permission
|
|
@@ -2503,16 +2559,13 @@ def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=No
|
|
| 2503 |
documents was pulled in full — all its tabs, all its cells — so that three fields could be
|
| 2504 |
consulted, and the list got slower with every cell anybody typed.
|
| 2505 |
|
| 2506 |
-
⭐ THE ROW ALREADY ANSWERS
|
| 2507 |
the document on every save, and `workbook_role` reads nothing else; so does the route's
|
| 2508 |
-
injected helper.
|
| 2509 |
-
|
| 2510 |
-
|
| 2511 |
-
|
| 2512 |
-
|
| 2513 |
-
R28 / W62-T25: admission uses the document facts rebuilt on save and explicit sharing.
|
| 2514 |
-
`_index_drop` removes a deleted document's row. Older rows still check document existence
|
| 2515 |
-
when taking the fallback; opening any workbook always reads its document for real.
|
| 2516 |
"""
|
| 2517 |
out = []
|
| 2518 |
bulk = grant_records
|
|
@@ -2537,18 +2590,9 @@ def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=No
|
|
| 2537 |
light = {"id": row["id"], "created_by": row.get("created_by") or ""}
|
| 2538 |
role = (role_on_workbook(light) if role_on_workbook is not None
|
| 2539 |
else workbook_role(st, light, viewer, is_admin=is_admin, rec=rec))
|
| 2540 |
-
if not
|
| 2541 |
-
|
| 2542 |
-
|
| 2543 |
-
continue
|
| 2544 |
-
else:
|
| 2545 |
-
# R28 / W62-T25: old rows lack this fact, so retain their document check.
|
| 2546 |
-
# R28: this fallback set is expected to be empty after every workbook
|
| 2547 |
-
# has been saved once; guessing would change R28's legacy admission.
|
| 2548 |
-
book = _read(st, workbook_key(row["id"]))
|
| 2549 |
-
if not book or not book.get("id") or not may_read_workbook(book, role):
|
| 2550 |
-
continue
|
| 2551 |
-
joined["role"] = role or "view"
|
| 2552 |
joined["shares"] = shares_from_record(rec)
|
| 2553 |
out.append(joined)
|
| 2554 |
return out
|
|
@@ -2843,7 +2887,8 @@ def validate_folder_tree(folders, item_folders):
|
|
| 2843 |
if len(rows_in) > _MAX_FOLDERS:
|
| 2844 |
raise SheetFolderRefusal(
|
| 2845 |
"too_many_folders",
|
| 2846 |
-
"That rail holds
|
|
|
|
| 2847 |
reason="too_many")
|
| 2848 |
|
| 2849 |
# ---- pass 1: shape. Ids, duplicates, and a `parent` resolved against the FINISHED set.
|
|
@@ -3127,9 +3172,9 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None, *,
|
|
| 3127 |
inside `_apply` for the same reason. `store.update` discards the callback's work when it
|
| 3128 |
raises, so a refused save still writes nothing.
|
| 3129 |
|
| 3130 |
-
⚠
|
| 3131 |
-
|
| 3132 |
-
|
| 3133 |
"""
|
| 3134 |
tree, placed, repairs = validate_folder_tree(folders, item_folders)
|
| 3135 |
writable = None if writable_ids is None else {str(wid) for wid in writable_ids}
|
|
|
|
| 261 |
#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
|
| 262 |
SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
|
| 263 |
"rules", "rowGroups", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
|
| 264 |
+
"visibility", "view", "print", "excel", "kind", "pivot")
|
| 265 |
+
|
| 266 |
+
SHEET_TAB_KINDS = ("grid", "pivot")
|
| 267 |
+
PIVOT_AGGREGATES = ("sum", "count", "avg", "min", "max")
|
| 268 |
|
| 269 |
#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
|
| 270 |
#: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab
|
|
|
|
| 1505 |
return out or None
|
| 1506 |
|
| 1507 |
|
| 1508 |
+
def clean_pivot(raw, *, draft=False):
|
| 1509 |
+
"""C6's authored pivot definition, validated without changing its JSON representation.
|
| 1510 |
+
|
| 1511 |
+
Field and database access are checked by the read door, never by workbook storage. A saved
|
| 1512 |
+
pivot is a definition, not a cached aggregate, and additive metadata remains intact.
|
| 1513 |
+
`draft=True` (workbook storage) admits a tab with no database or value yet, which is what
|
| 1514 |
+
Insert > PivotTable creates; the read door keeps refusing it.
|
| 1515 |
+
"""
|
| 1516 |
+
if not isinstance(raw, dict):
|
| 1517 |
+
raise SheetRefusal("sheet_pivot_invalid", "A pivot must be an object.")
|
| 1518 |
+
database = raw.get("database")
|
| 1519 |
+
if not isinstance(database, str) or (not database.strip() and not draft):
|
| 1520 |
+
raise SheetRefusal("sheet_pivot_invalid", "A pivot must name a database.")
|
| 1521 |
+
for axis in ("rows", "columns"):
|
| 1522 |
+
fields = raw.get(axis, [])
|
| 1523 |
+
if not isinstance(fields, list) or any(not isinstance(field, str) or not field.strip()
|
| 1524 |
+
for field in fields):
|
| 1525 |
+
raise SheetRefusal("sheet_pivot_invalid", "Pivot rows and columns must list field names.")
|
| 1526 |
+
values = raw.get("values")
|
| 1527 |
+
if not isinstance(values, list) or (not values and not draft):
|
| 1528 |
+
raise SheetRefusal("sheet_pivot_invalid", "A pivot must have at least one value.")
|
| 1529 |
+
for value in values:
|
| 1530 |
+
if not isinstance(value, dict) or not isinstance(value.get("field"), str) \
|
| 1531 |
+
or not value["field"].strip():
|
| 1532 |
+
raise SheetRefusal("sheet_pivot_invalid", "Each pivot value must name a field.")
|
| 1533 |
+
if value.get("agg") not in PIVOT_AGGREGATES:
|
| 1534 |
+
raise SheetRefusal("sheet_pivot_aggregate_unknown",
|
| 1535 |
+
"Choose sum, count, avg, min or max for the pivot value.")
|
| 1536 |
+
|
| 1537 |
+
def validate_filters(nodes):
|
| 1538 |
+
if not isinstance(nodes, list):
|
| 1539 |
+
raise SheetRefusal("sheet_pivot_invalid", "Pivot filters must be a list.")
|
| 1540 |
+
for node in nodes:
|
| 1541 |
+
if not isinstance(node, dict):
|
| 1542 |
+
raise SheetRefusal("sheet_pivot_invalid", "Each pivot filter must be an object.")
|
| 1543 |
+
if "children" in node:
|
| 1544 |
+
if node.get("conj", "and") not in ("and", "or"):
|
| 1545 |
+
raise SheetRefusal("sheet_pivot_invalid", "A pivot filter group must use and or or.")
|
| 1546 |
+
validate_filters(node["children"])
|
| 1547 |
+
elif not isinstance(node.get("colId"), str) or not node["colId"].strip() \
|
| 1548 |
+
or not isinstance(node.get("op"), str) or not node["op"].strip():
|
| 1549 |
+
raise SheetRefusal("sheet_pivot_invalid", "Each pivot filter must name a field and condition.")
|
| 1550 |
+
|
| 1551 |
+
validate_filters(raw.get("filters", []))
|
| 1552 |
+
return copy.deepcopy(_json_safe(raw, "pivot"))
|
| 1553 |
+
|
| 1554 |
+
|
| 1555 |
def _clean_sheet(raw, index=0, *, lossless=False):
|
| 1556 |
"""Normalize a tab's known strata and retain additive JSON metadata without interpretation.
|
| 1557 |
|
|
|
|
| 1582 |
visibility = raw.get("visibility")
|
| 1583 |
if visibility is not None and visibility not in ("visible", "hidden", "veryHidden"):
|
| 1584 |
raise SheetRefusal("sheet_visibility_unknown", "A sheet must be visible, hidden or very hidden.")
|
| 1585 |
+
kind = raw.get("kind", "grid")
|
| 1586 |
+
if kind not in SHEET_TAB_KINDS:
|
| 1587 |
+
raise SheetRefusal("sheet_tab_kind_unknown", "Choose grid or pivot for the sheet tab.")
|
| 1588 |
+
pivot = clean_pivot(raw.get("pivot"), draft=True) if kind == "pivot" or "pivot" in raw else None
|
| 1589 |
|
| 1590 |
#: ⚠ THE ORDER OF THIS DICT IS THE ORDER THE CLEANERS RUN IN, and it is load bearing: two of
|
| 1591 |
#: them REFUSE by name (`_clean_rules` on an unknown op, `_clean_imported` on a record that is
|
|
|
|
| 1606 |
"view": _clean_view(raw.get("view")),
|
| 1607 |
"print": _clean_print(raw.get("print")),
|
| 1608 |
"excel": excel,
|
| 1609 |
+
# Absence remains the legacy grid default; an explicitly authored kind round-trips.
|
| 1610 |
+
"kind": kind if "kind" in raw else None,
|
| 1611 |
+
"pivot": pivot,
|
| 1612 |
}
|
| 1613 |
|
| 1614 |
#: ⛔ THE WALK IS THROUGH `SHEET_TAB_KEYS`, NOT THROUGH `optional`, AND THAT IS THE WHOLE
|
|
|
|
| 1896 |
if expect_rev is not None and int(expect_rev) < int(prior.get("structureRev") or 0):
|
| 1897 |
raise SheetStaleRefusal(
|
| 1898 |
"sheet_structure_stale",
|
| 1899 |
+
"Rows or columns changed since you opened this workbook. This save was not applied. Try again using the refreshed workbook.",
|
| 1900 |
rev_current=int(prior.get("rev") or 0))
|
| 1901 |
if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev):
|
| 1902 |
raise SheetStaleRefusal(
|
|
|
|
| 2224 |
if revision < int(prior.get("structureRev") or 0):
|
| 2225 |
raise SheetStaleRefusal(
|
| 2226 |
"sheet_structure_stale",
|
| 2227 |
+
"Rows or columns changed since you opened this workbook. This save was not applied. Try again using the refreshed workbook.",
|
| 2228 |
rev_current=current)
|
| 2229 |
if revision < int(tab.get("writesFrom") or 0):
|
| 2230 |
raise SheetStaleRefusal(
|
| 2231 |
"sheet_revision_stale",
|
| 2232 |
+
"This tab's retained change history was truncated after your revision. This save was not applied. Try again using the refreshed tab.",
|
| 2233 |
rev_current=current)
|
| 2234 |
for write in writes:
|
| 2235 |
if int(write["rev"]) > revision and write["by"] != actor:
|
|
|
|
| 2478 |
state.setdefault("access", "private")
|
| 2479 |
return state, state
|
| 2480 |
|
| 2481 |
+
# Preserve the existing explicit-share marker and index facts; R31 admission uses roles.
|
| 2482 |
book = _persist(st, workbook_key(wid), _apply)
|
| 2483 |
_index_upsert(st, book)
|
| 2484 |
|
|
|
|
| 2534 |
|
| 2535 |
|
| 2536 |
def may_read_workbook(book, role):
|
| 2537 |
+
"""R31 / C7: only an owner, grantee or administrator may read a workbook.
|
| 2538 |
|
| 2539 |
+
The existing role resolver includes administrators as owners. Legacy access markers and
|
| 2540 |
+
missing creators confer no read access; document reads and index projections use this rule.
|
|
|
|
| 2541 |
"""
|
| 2542 |
+
return role in ("owner", "view", "edit")
|
| 2543 |
|
| 2544 |
|
| 2545 |
def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=None,
|
| 2546 |
grant_records=None):
|
| 2547 |
"""Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched.
|
| 2548 |
|
| 2549 |
+
R31 / C7: a named viewer filters rows with the workbook read predicate and receives their role.
|
| 2550 |
+
The route uses the bulk role join; callers may also supply a workbook role helper.
|
| 2551 |
`list_workbooks` itself stays a bare pointer list on purpose:
|
| 2552 |
`iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry
|
| 2553 |
read per workbook there would put a permission lookup inside a scan that asks no permission
|
|
|
|
| 2559 |
documents was pulled in full — all its tabs, all its cells — so that three fields could be
|
| 2560 |
consulted, and the list got slower with every cell anybody typed.
|
| 2561 |
|
| 2562 |
+
⭐ THE ROW ALREADY ANSWERS THE ROLE QUESTION. `_index_row` rebuilds `id` and `created_by` off
|
| 2563 |
the document on every save, and `workbook_role` reads nothing else; so does the route's
|
| 2564 |
+
injected helper. R31 removes the legacy-visibility exception, so older rows lacking
|
| 2565 |
+
`has_access` need no workbook-document fallback either.
|
| 2566 |
+
|
| 2567 |
+
`_index_drop` removes a deleted document's row. Opening any workbook always reads its
|
| 2568 |
+
document for real; the index remains a pointer list.
|
|
|
|
|
|
|
|
|
|
| 2569 |
"""
|
| 2570 |
out = []
|
| 2571 |
bulk = grant_records
|
|
|
|
| 2590 |
light = {"id": row["id"], "created_by": row.get("created_by") or ""}
|
| 2591 |
role = (role_on_workbook(light) if role_on_workbook is not None
|
| 2592 |
else workbook_role(st, light, viewer, is_admin=is_admin, rec=rec))
|
| 2593 |
+
if not may_read_workbook(light, role):
|
| 2594 |
+
continue
|
| 2595 |
+
joined["role"] = role
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2596 |
joined["shares"] = shares_from_record(rec)
|
| 2597 |
out.append(joined)
|
| 2598 |
return out
|
|
|
|
| 2887 |
if len(rows_in) > _MAX_FOLDERS:
|
| 2888 |
raise SheetFolderRefusal(
|
| 2889 |
"too_many_folders",
|
| 2890 |
+
f"That rail holds {len(rows_in)} folders and a workspace accepts at most {_MAX_FOLDERS}. "
|
| 2891 |
+
"Delete unused folders or combine their contents into fewer folders and try again.",
|
| 2892 |
reason="too_many")
|
| 2893 |
|
| 2894 |
# ---- pass 1: shape. Ids, duplicates, and a `parent` resolved against the FINISHED set.
|
|
|
|
| 3172 |
inside `_apply` for the same reason. `store.update` discards the callback's work when it
|
| 3173 |
raises, so a refused save still writes nothing.
|
| 3174 |
|
| 3175 |
+
⚠ Trusted core callers may omit `expect_rev`. The HTTP wholesale door requires the integer
|
| 3176 |
+
revision from `read_index`, including revision zero for the first folder. Placements-only
|
| 3177 |
+
deltas use `merge_placements` instead of replacing the rail.
|
| 3178 |
"""
|
| 3179 |
tree, placed, repairs = validate_folder_tree(folders, item_folders)
|
| 3180 |
writable = None if writable_ids is None else {str(wid) for wid in writable_ids}
|
platform/core/store.py
CHANGED
|
@@ -380,6 +380,34 @@ class Store:
|
|
| 380 |
self._base[name] = sha # D-305 — set with the cache, never apart from it
|
| 381 |
return _project(data, drop)
|
| 382 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
def _key_lock(self, name):
|
| 384 |
"""The write lock for ONE key (W31-T13). Created on demand; never removed."""
|
| 385 |
n = str(name)
|
|
@@ -1183,6 +1211,11 @@ def get_projection(name, drop=()):
|
|
| 1183 |
return _d().get_projection(name, drop=drop)
|
| 1184 |
|
| 1185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1186 |
def revision(name):
|
| 1187 |
"""The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing.
|
| 1188 |
|
|
|
|
| 380 |
self._base[name] = sha # D-305 — set with the cache, never apart from it
|
| 381 |
return _project(data, drop)
|
| 382 |
|
| 383 |
+
def get_entry(self, name, entry):
|
| 384 |
+
"""One detached dictionary entry, or None, with the existing document cache/lock.
|
| 385 |
+
|
| 386 |
+
Cold HF reads retain the whole downloaded document in the existing cache. Only the
|
| 387 |
+
selected entry is serialized; sibling rows/cells never enter the returned copy.
|
| 388 |
+
"""
|
| 389 |
+
def selected(data):
|
| 390 |
+
value = data.get(str(entry)) if isinstance(data, dict) else None
|
| 391 |
+
return json.loads(json.dumps(value)) if isinstance(value, dict) else None
|
| 392 |
+
|
| 393 |
+
with self._lock:
|
| 394 |
+
if name in self._cache:
|
| 395 |
+
return selected(self._cache[name])
|
| 396 |
+
try:
|
| 397 |
+
data, sha = self._download(name)
|
| 398 |
+
except Exception as exc:
|
| 399 |
+
try:
|
| 400 |
+
import harness.telemetry as telemetry
|
| 401 |
+
telemetry.error(f'store:get_entry:{name}', exc,
|
| 402 |
+
fallback='cached' if name in self._cache else 'empty')
|
| 403 |
+
except Exception:
|
| 404 |
+
pass
|
| 405 |
+
with self._lock:
|
| 406 |
+
return selected(self._cache.get(name))
|
| 407 |
+
with self._lock:
|
| 408 |
+
self._cache[name], self._base[name] = data, sha
|
| 409 |
+
return selected(data)
|
| 410 |
+
|
| 411 |
def _key_lock(self, name):
|
| 412 |
"""The write lock for ONE key (W31-T13). Created on demand; never removed."""
|
| 413 |
n = str(name)
|
|
|
|
| 1211 |
return _d().get_projection(name, drop=drop)
|
| 1212 |
|
| 1213 |
|
| 1214 |
+
def get_entry(name, entry):
|
| 1215 |
+
"""One dictionary entry through the default tenant's existing store handle."""
|
| 1216 |
+
return _d().get_entry(name, entry)
|
| 1217 |
+
|
| 1218 |
+
|
| 1219 |
def revision(name):
|
| 1220 |
"""The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing.
|
| 1221 |
|
platform/core/store_backend.py
CHANGED
|
@@ -113,6 +113,11 @@ def get_projection(name_, drop=()):
|
|
| 113 |
return active().get_projection(name_, drop=drop)
|
| 114 |
|
| 115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
def revision(name_):
|
| 117 |
"""The bucket's change token (wave 29, C6) — `{'rev', 'updated_at', 'token'}`.
|
| 118 |
|
|
|
|
| 113 |
return active().get_projection(name_, drop=drop)
|
| 114 |
|
| 115 |
|
| 116 |
+
def get_entry(name_, entry):
|
| 117 |
+
"""One dictionary entry through the existing active-backend selector."""
|
| 118 |
+
return active().get_entry(name_, entry)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
def revision(name_):
|
| 122 |
"""The bucket's change token (wave 29, C6) — `{'rev', 'updated_at', 'token'}`.
|
| 123 |
|
platform/core/store_pg.py
CHANGED
|
@@ -49,6 +49,7 @@ from pathlib import Path
|
|
| 49 |
from urllib.parse import urlparse
|
| 50 |
|
| 51 |
import core.data_binding as data_binding
|
|
|
|
| 52 |
from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER
|
| 53 |
|
| 54 |
_URL_ENV = 'DATABASE_URL'
|
|
@@ -254,6 +255,7 @@ def check_write(tenant_slug=None, operation='write'):
|
|
| 254 |
a session-scoped fallback and tell the user their change was saved. A new type would have had
|
| 255 |
to re-earn both properties, and would have failed the second one silently.
|
| 256 |
"""
|
|
|
|
| 257 |
reason = gate_write_refusal(tenant_slug) or write_refusal(tenant_slug)
|
| 258 |
if reason:
|
| 259 |
raise data_binding.StoreWriteRefused(
|
|
@@ -550,6 +552,11 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
|
|
| 550 |
}
|
| 551 |
|
| 552 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 553 |
def _get_entry(name, entry, tenant_slug=None):
|
| 554 |
"""ONE top-level entry of document `name` (`value -> entry`), or None when either is absent.
|
| 555 |
|
|
@@ -650,21 +657,15 @@ class StoreDocumentTooLarge(RuntimeError):
|
|
| 650 |
|
| 651 |
def _payload(name, data, con=None, tenant_slug=None):
|
| 652 |
"""Serialize `data` for `name` through the byte budget and the ceiling. Raises when refused."""
|
| 653 |
-
payload = json.dumps(data)
|
| 654 |
if str(name) == 'user_tables' and isinstance(data, dict):
|
| 655 |
from core import user_tables as _ut
|
| 656 |
-
#
|
| 657 |
-
|
| 658 |
-
if
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
payload = json.dumps(data)
|
| 664 |
-
if report['evicted']:
|
| 665 |
-
print('[store_pg] user_tables byte budget: stored no rows for '
|
| 666 |
-
+ ', '.join(f'{k} ({v / 1048576:.1f} MB)'
|
| 667 |
-
for k, v in sorted(report['evicted'].items())))
|
| 668 |
if len(payload) > DOCUMENT_CEILING_BYTES:
|
| 669 |
before = None
|
| 670 |
if con is not None:
|
|
|
|
| 49 |
from urllib.parse import urlparse
|
| 50 |
|
| 51 |
import core.data_binding as data_binding
|
| 52 |
+
from core import freeze_notice
|
| 53 |
from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER
|
| 54 |
|
| 55 |
_URL_ENV = 'DATABASE_URL'
|
|
|
|
| 255 |
a session-scoped fallback and tell the user their change was saved. A new type would have had
|
| 256 |
to re-earn both properties, and would have failed the second one silently.
|
| 257 |
"""
|
| 258 |
+
freeze_notice.check_write()
|
| 259 |
reason = gate_write_refusal(tenant_slug) or write_refusal(tenant_slug)
|
| 260 |
if reason:
|
| 261 |
raise data_binding.StoreWriteRefused(
|
|
|
|
| 552 |
}
|
| 553 |
|
| 554 |
|
| 555 |
+
def get_entry(name, entry, tenant_slug=None):
|
| 556 |
+
"""Dictionary entry parity with the HF store through the existing PG primitive."""
|
| 557 |
+
return _get_entry(name, entry, tenant_slug=tenant_slug)
|
| 558 |
+
|
| 559 |
+
|
| 560 |
def _get_entry(name, entry, tenant_slug=None):
|
| 561 |
"""ONE top-level entry of document `name` (`value -> entry`), or None when either is absent.
|
| 562 |
|
|
|
|
| 657 |
|
| 658 |
def _payload(name, data, con=None, tenant_slug=None):
|
| 659 |
"""Serialize `data` for `name` through the byte budget and the ceiling. Raises when refused."""
|
|
|
|
| 660 |
if str(name) == 'user_tables' and isinstance(data, dict):
|
| 661 |
from core import user_tables as _ut
|
| 662 |
+
# Read-through copies must never persist, even below the size budget.
|
| 663 |
+
report = _ut.enforce_byte_budget(data)
|
| 664 |
+
if report['evicted']:
|
| 665 |
+
print('[store_pg] user_tables byte budget: stored no rows for '
|
| 666 |
+
+ ', '.join(f'{k} ({v / 1048576:.1f} MB)'
|
| 667 |
+
for k, v in sorted(report['evicted'].items())))
|
| 668 |
+
payload = json.dumps(data)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 669 |
if len(payload) > DOCUMENT_CEILING_BYTES:
|
| 670 |
before = None
|
| 671 |
if con is not None:
|
platform/core/table_store.py
CHANGED
|
@@ -774,7 +774,7 @@ class TableStore:
|
|
| 774 |
# then splice a bounded copy into that payload only. Keeping it out of `fields`
|
| 775 |
# prevents an old request id surviving forever and overriding a later rename.
|
| 776 |
corrections = {}
|
| 777 |
-
if consume_corrections and ws.get('fieldCorrections'):
|
| 778 |
def _take(current):
|
| 779 |
current_ws = current.get(username) or {}
|
| 780 |
pending = current_ws.get('fieldCorrections') or {}
|
|
@@ -824,7 +824,8 @@ class TableStore:
|
|
| 824 |
out = {
|
| 825 |
'views': dict(ws.get('views') or {}),
|
| 826 |
'fields': fields,
|
| 827 |
-
'overlays':
|
|
|
|
| 828 |
# wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
|
| 829 |
# stratum rather than a key on each item — see aios_grid.clean_folders for why
|
| 830 |
# (a cohort lives in another store, and filing is an organising act, not part of
|
|
@@ -1402,6 +1403,9 @@ class TableStore:
|
|
| 1402 |
permission_plans = self._personal_permission_plans(source)
|
| 1403 |
if not plans and not permission_plans and not has_blank_legacy:
|
| 1404 |
return {'migrated': 0, 'status': 'clean'}
|
|
|
|
|
|
|
|
|
|
| 1405 |
|
| 1406 |
import core.shares as shares
|
| 1407 |
|
|
|
|
| 774 |
# then splice a bounded copy into that payload only. Keeping it out of `fields`
|
| 775 |
# prevents an old request id surviving forever and overriding a later rename.
|
| 776 |
corrections = {}
|
| 777 |
+
if consume_corrections and not getattr(self._st, 'schema_only', False) and ws.get('fieldCorrections'):
|
| 778 |
def _take(current):
|
| 779 |
current_ws = current.get(username) or {}
|
| 780 |
pending = current_ws.get('fieldCorrections') or {}
|
|
|
|
| 824 |
out = {
|
| 825 |
'views': dict(ws.get('views') or {}),
|
| 826 |
'fields': fields,
|
| 827 |
+
'overlays': ({} if getattr(self._st, 'schema_only', False)
|
| 828 |
+
else dict(ws.get('overlays') or {})),
|
| 829 |
# wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
|
| 830 |
# stratum rather than a key on each item — see aios_grid.clean_folders for why
|
| 831 |
# (a cohort lives in another store, and filing is an organising act, not part of
|
|
|
|
| 1403 |
permission_plans = self._personal_permission_plans(source)
|
| 1404 |
if not plans and not permission_plans and not has_blank_legacy:
|
| 1405 |
return {'migrated': 0, 'status': 'clean'}
|
| 1406 |
+
if getattr(self._st, 'schema_only', False):
|
| 1407 |
+
from core.shared_overlay import SchemaUnavailable
|
| 1408 |
+
raise SchemaUnavailable('saved fields and views need a full workspace refresh to finish migration')
|
| 1409 |
|
| 1410 |
import core.shares as shares
|
| 1411 |
|
platform/core/user_tables.py
CHANGED
|
@@ -327,6 +327,20 @@ def capped_table_count(tables):
|
|
| 327 |
#: for whole tables. Every table that fails one is REPORTED, per table, on the status door.
|
| 328 |
_READ_THROUGH = set()
|
| 329 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
|
| 331 |
def register_read_through(*table_keys):
|
| 332 |
"""Declare that these databases are SERVED from the mirror and store no rows here."""
|
|
@@ -378,14 +392,13 @@ def strip_materialised(st=None):
|
|
| 378 |
the permission wall and the nav entry with it.
|
| 379 |
"""
|
| 380 |
out = {'dropped': {}, 'before': 0, 'after': 0}
|
| 381 |
-
if not _READ_THROUGH:
|
| 382 |
-
return out
|
| 383 |
s = _st(st)
|
| 384 |
try:
|
| 385 |
cur = s.get(STORE_KEY) or {}
|
| 386 |
except Exception: # noqa: BLE001
|
| 387 |
return out
|
| 388 |
-
fat = {k: len(
|
|
|
|
| 389 |
fat = {k: n for k, n in fat.items() if n}
|
| 390 |
if not fat:
|
| 391 |
return out # the common case: nothing to do, and nothing measured
|
|
@@ -394,14 +407,14 @@ def strip_materialised(st=None):
|
|
| 394 |
out['before'] = len(_json.dumps(cur, default=str))
|
| 395 |
|
| 396 |
def _strip(doc):
|
| 397 |
-
for key in
|
| 398 |
-
t
|
| 399 |
-
if isinstance(t, dict) and t.get('rows'):
|
| 400 |
out['dropped'][key] = len(t['rows'])
|
| 401 |
t['rows'] = {}
|
| 402 |
# ⛔ THE STAMP IS THE POINT — see `materialises`. Without it the next process to
|
| 403 |
# boot reads an empty `rows` and calls that the answer.
|
| 404 |
t['readThrough'] = True
|
|
|
|
| 405 |
return doc
|
| 406 |
|
| 407 |
s.update(STORE_KEY, _strip, flush='sync')
|
|
@@ -431,9 +444,9 @@ def _connector_fed(key, table):
|
|
| 431 |
|
| 432 |
|
| 433 |
def enforce_byte_budget(doc, now=None):
|
| 434 |
-
"""
|
| 435 |
|
| 436 |
-
Idempotent
|
| 437 |
`{'evicted': {key: bytes}, 'cleared': [key, ...]}`.
|
| 438 |
"""
|
| 439 |
out = {'evicted': {}, 'cleared': []}
|
|
@@ -444,6 +457,20 @@ def enforce_byte_budget(doc, now=None):
|
|
| 444 |
for key, table in doc.items():
|
| 445 |
if not isinstance(table, dict):
|
| 446 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
rows = table.get('rows')
|
| 448 |
if not rows:
|
| 449 |
continue
|
|
@@ -1585,9 +1612,17 @@ def all_defs(st=None):
|
|
| 1585 |
this function, so an empty answer here stops being a blank nav and becomes a 404 on every
|
| 1586 |
database in the tenant.
|
| 1587 |
"""
|
|
|
|
|
|
|
|
|
|
| 1588 |
try:
|
| 1589 |
-
return dict(
|
| 1590 |
except Exception:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1591 |
# ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was
|
| 1592 |
# only ever ONE of the ways this can fail, and singling it out is what made the other ways
|
| 1593 |
# silent. `all_tables` is correct for all of them.
|
|
@@ -4758,8 +4793,9 @@ def may_edit_database(table_key, viewer, is_admin=False, st=None):
|
|
| 4758 |
#: LAZILY (an admin, or a table's creator, is admitted before that leg runs and never pays for
|
| 4759 |
#: it) and memoised for the pass.
|
| 4760 |
#:
|
| 4761 |
-
#: ⚠ THE ALLOW-LIST
|
| 4762 |
-
#: to the real runtime uncached
|
|
|
|
| 4763 |
#: runtime through `__getattr__` — so this can serve a stale value for the two named keys and
|
| 4764 |
#: dynamically named `<key>__shared` overlay buckets, for
|
| 4765 |
#: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level
|
|
@@ -4769,6 +4805,9 @@ def may_edit_database(table_key, viewer, is_admin=False, st=None):
|
|
| 4769 |
#: `fresh=True` and write verbs still reach the runtime, never the snapshot.
|
| 4770 |
_LENDABLE = (STORE_KEY, 'object_shares')
|
| 4771 |
|
|
|
|
|
|
|
|
|
|
| 4772 |
|
| 4773 |
class _Lent:
|
| 4774 |
"""A read-only `st` that serves already-read documents and passes everything else through."""
|
|
@@ -4777,7 +4816,8 @@ class _Lent:
|
|
| 4777 |
self._runtime, self._docs = runtime, dict(docs or {})
|
| 4778 |
|
| 4779 |
def get(self, name, fresh=False):
|
| 4780 |
-
if fresh or (name not in
|
|
|
|
| 4781 |
return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
|
| 4782 |
if name not in self._docs:
|
| 4783 |
self._docs[name] = self._runtime.get(name) # lazily, once, for this pass
|
|
@@ -4797,7 +4837,7 @@ class _Lent:
|
|
| 4797 |
A re-projection would cost a second copy to convert the first case and change nothing
|
| 4798 |
about the second.
|
| 4799 |
"""
|
| 4800 |
-
if name not in _LENDABLE and not str(name).endswith('__shared'):
|
| 4801 |
return self._runtime.get_projection(name, drop=drop)
|
| 4802 |
if name not in self._docs:
|
| 4803 |
self._docs[name] = self._runtime.get_projection(name, drop=drop)
|
|
@@ -4808,7 +4848,7 @@ class _Lent:
|
|
| 4808 |
|
| 4809 |
|
| 4810 |
def lend(st=None, **docs):
|
| 4811 |
-
"""Lend `_LENDABLE`
|
| 4812 |
|
| 4813 |
`lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and
|
| 4814 |
hand the result to every predicate in that pass; see the note above for what makes it safe.
|
|
|
|
| 327 |
#: for whole tables. Every table that fails one is REPORTED, per table, on the status door.
|
| 328 |
_READ_THROUGH = set()
|
| 329 |
|
| 330 |
+
# C8 / R33: these reserved connector databases always serve the mirror. A cold
|
| 331 |
+
# definition write must retire their legacy rows even before the API registry loads.
|
| 332 |
+
_MIRROR_ONLY_KEYS = frozenset(('ut_amzn_orders', 'ut_amzn_order_items'))
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def _mirror_only_definition(key, table):
|
| 336 |
+
return (key in _READ_THROUGH or table.get('readThrough') is True
|
| 337 |
+
or (key in _MIRROR_ONLY_KEYS and (
|
| 338 |
+
_connector_fed(key, table)
|
| 339 |
+
or (table.get('source') == AUTOMATION_SOURCE
|
| 340 |
+
and table.get('recordMode') == AUTOMATION_RECORD_MODE
|
| 341 |
+
and any((field.get('automation') or {}).get('flowId') == 'amazon_relational'
|
| 342 |
+
for field in table.get('fields') or [] if isinstance(field, dict))))))
|
| 343 |
+
|
| 344 |
|
| 345 |
def register_read_through(*table_keys):
|
| 346 |
"""Declare that these databases are SERVED from the mirror and store no rows here."""
|
|
|
|
| 392 |
the permission wall and the nav entry with it.
|
| 393 |
"""
|
| 394 |
out = {'dropped': {}, 'before': 0, 'after': 0}
|
|
|
|
|
|
|
| 395 |
s = _st(st)
|
| 396 |
try:
|
| 397 |
cur = s.get(STORE_KEY) or {}
|
| 398 |
except Exception: # noqa: BLE001
|
| 399 |
return out
|
| 400 |
+
fat = {k: len(t.get('rows') or {}) for k, t in cur.items()
|
| 401 |
+
if isinstance(t, dict) and _mirror_only_definition(k, t)}
|
| 402 |
fat = {k: n for k, n in fat.items() if n}
|
| 403 |
if not fat:
|
| 404 |
return out # the common case: nothing to do, and nothing measured
|
|
|
|
| 407 |
out['before'] = len(_json.dumps(cur, default=str))
|
| 408 |
|
| 409 |
def _strip(doc):
|
| 410 |
+
for key, t in (doc or {}).items():
|
| 411 |
+
if isinstance(t, dict) and _mirror_only_definition(key, t) and t.get('rows'):
|
|
|
|
| 412 |
out['dropped'][key] = len(t['rows'])
|
| 413 |
t['rows'] = {}
|
| 414 |
# ⛔ THE STAMP IS THE POINT — see `materialises`. Without it the next process to
|
| 415 |
# boot reads an empty `rows` and calls that the answer.
|
| 416 |
t['readThrough'] = True
|
| 417 |
+
t.pop(OVERSIZE_FIELD, None)
|
| 418 |
return doc
|
| 419 |
|
| 420 |
s.update(STORE_KEY, _strip, flush='sync')
|
|
|
|
| 444 |
|
| 445 |
|
| 446 |
def enforce_byte_budget(doc, now=None):
|
| 447 |
+
"""Remove read-through copies, then evict connector-fed rows over the byte budget in place.
|
| 448 |
|
| 449 |
+
Idempotent; `core.store_pg` calls this from its payload-budget check. Returns
|
| 450 |
`{'evicted': {key: bytes}, 'cleared': [key, ...]}`.
|
| 451 |
"""
|
| 452 |
out = {'evicted': {}, 'cleared': []}
|
|
|
|
| 457 |
for key, table in doc.items():
|
| 458 |
if not isinstance(table, dict):
|
| 459 |
continue
|
| 460 |
+
# A read-through definition cannot retain old rows below the byte budget.
|
| 461 |
+
# The flag and removal share the caller's atomic definition write; the
|
| 462 |
+
# source population remains in the mirror and has no row ceiling.
|
| 463 |
+
if _mirror_only_definition(key, table):
|
| 464 |
+
rows = table.get('rows') or {}
|
| 465 |
+
if rows:
|
| 466 |
+
out['evicted'][key] = rows_bytes(rows)
|
| 467 |
+
table['rows'] = {}
|
| 468 |
+
if table.get('readThrough') is not True:
|
| 469 |
+
out['cleared'].append(key)
|
| 470 |
+
table['readThrough'] = True
|
| 471 |
+
if table.pop(OVERSIZE_FIELD, None) is not None and key not in out['cleared']:
|
| 472 |
+
out['cleared'].append(key)
|
| 473 |
+
continue
|
| 474 |
rows = table.get('rows')
|
| 475 |
if not rows:
|
| 476 |
continue
|
|
|
|
| 1612 |
this function, so an empty answer here stops being a blank nav and becomes a 404 on every
|
| 1613 |
database in the tenant.
|
| 1614 |
"""
|
| 1615 |
+
# Schema-only handles cannot use the historical whole-reader fallback described above:
|
| 1616 |
+
# their caller must distinguish an unavailable definition snapshot from an empty one.
|
| 1617 |
+
reader = _st(st)
|
| 1618 |
try:
|
| 1619 |
+
return dict(reader.get_projection(STORE_KEY, drop=DEF_ONLY_DROPS) or {})
|
| 1620 |
except Exception:
|
| 1621 |
+
# A schema-only request promises not to materialize rows, including on an outage.
|
| 1622 |
+
# Its existing route maps this failure to a named unavailable response. Ordinary
|
| 1623 |
+
# full readers retain the correct-but-slow fallback below.
|
| 1624 |
+
if getattr(reader, 'schema_only', False):
|
| 1625 |
+
raise
|
| 1626 |
# ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was
|
| 1627 |
# only ever ONE of the ways this can fail, and singling it out is what made the other ways
|
| 1628 |
# silent. `all_tables` is correct for all of them.
|
|
|
|
| 4793 |
#: LAZILY (an admin, or a table's creator, is admitted before that leg runs and never pays for
|
| 4794 |
#: it) and memoised for the pass.
|
| 4795 |
#:
|
| 4796 |
+
#: ⚠ THE ALLOW-LIST CONTROLS AUTOMATIC REUSE. Any other UNSUPPLIED key passes through
|
| 4797 |
+
#: to the real runtime uncached. Explicit snapshots are caller-captured read-only documents for
|
| 4798 |
+
#: this pass only; `fresh=True` always bypasses, and every WRITE verb reaches the
|
| 4799 |
#: runtime through `__getattr__` — so this can serve a stale value for the two named keys and
|
| 4800 |
#: dynamically named `<key>__shared` overlay buckets, for
|
| 4801 |
#: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level
|
|
|
|
| 4805 |
#: `fresh=True` and write verbs still reach the runtime, never the snapshot.
|
| 4806 |
_LENDABLE = (STORE_KEY, 'object_shares')
|
| 4807 |
|
| 4808 |
+
# A caller may also supply a captured read-only document explicitly. That grants no automatic
|
| 4809 |
+
# reuse to its bucket: unsupplied keys outside the allow-list still read the real runtime.
|
| 4810 |
+
|
| 4811 |
|
| 4812 |
class _Lent:
|
| 4813 |
"""A read-only `st` that serves already-read documents and passes everything else through."""
|
|
|
|
| 4816 |
self._runtime, self._docs = runtime, dict(docs or {})
|
| 4817 |
|
| 4818 |
def get(self, name, fresh=False):
|
| 4819 |
+
if fresh or (name not in self._docs and name not in _LENDABLE
|
| 4820 |
+
and not str(name).endswith('__shared')):
|
| 4821 |
return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
|
| 4822 |
if name not in self._docs:
|
| 4823 |
self._docs[name] = self._runtime.get(name) # lazily, once, for this pass
|
|
|
|
| 4837 |
A re-projection would cost a second copy to convert the first case and change nothing
|
| 4838 |
about the second.
|
| 4839 |
"""
|
| 4840 |
+
if name not in self._docs and name not in _LENDABLE and not str(name).endswith('__shared'):
|
| 4841 |
return self._runtime.get_projection(name, drop=drop)
|
| 4842 |
if name not in self._docs:
|
| 4843 |
self._docs[name] = self._runtime.get_projection(name, drop=drop)
|
|
|
|
| 4848 |
|
| 4849 |
|
| 4850 |
def lend(st=None, **docs):
|
| 4851 |
+
"""Lend explicit snapshots plus `_LENDABLE` and `__shared` buckets; defer other keys to `st`.
|
| 4852 |
|
| 4853 |
`lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and
|
| 4854 |
hand the result to every predicate in that pass; see the note above for what makes it safe.
|
platform/core/users.py
CHANGED
|
@@ -21,6 +21,7 @@ import time
|
|
| 21 |
from urllib.parse import quote
|
| 22 |
|
| 23 |
import core.store as store
|
|
|
|
| 24 |
|
| 25 |
BU_LABELS = {5: 'Fisch', 6: 'Royal'}
|
| 26 |
_ITER = 200_000
|
|
@@ -327,6 +328,7 @@ def set_mfa(username, secret, recovery_codes, *, require_unenrolled=False, expec
|
|
| 327 |
enrolment check lets two workers both issue a first set of codes; an epoch check prevents
|
| 328 |
regeneration racing an admin reset from silently reinstalling the lost factor.
|
| 329 |
"""
|
|
|
|
| 330 |
username = (username or '').strip().lower()
|
| 331 |
# The replay closure captures only hashes, never printable recovery codes: the HF store
|
| 332 |
# journals closures for conflict replay, and that journal must not retain clear codes.
|
|
@@ -380,6 +382,14 @@ def clear_mfa(username):
|
|
| 380 |
def consume_recovery_code(username, code) -> bool:
|
| 381 |
username = (username or '').strip().lower()
|
| 382 |
wanted = _recovery_key(code)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
consumed = [False]
|
| 384 |
|
| 385 |
def _set(reg):
|
|
@@ -483,6 +493,8 @@ def login_lockout_remaining(identifier, now=None):
|
|
| 483 |
|
| 484 |
def note_login_failure(identifier, now=None):
|
| 485 |
"""Atomically count one failed password/MFA proof and return lock seconds remaining."""
|
|
|
|
|
|
|
| 486 |
now = time.time() if now is None else float(now)
|
| 487 |
key = _login_lockout_key(identifier)
|
| 488 |
|
|
@@ -552,6 +564,8 @@ def note_login_failure(identifier, now=None):
|
|
| 552 |
|
| 553 |
def clear_login_failures(identifier):
|
| 554 |
"""Clear the account's shared guessing budget after a complete successful proof."""
|
|
|
|
|
|
|
| 555 |
key = _login_lockout_key(identifier)
|
| 556 |
|
| 557 |
def _clear(doc):
|
|
@@ -747,7 +761,8 @@ def create_user(username, pw, name, role='user', bus='all', modules='all',
|
|
| 747 |
the carry-and-bump lives here rather than in each caller: an overwrite is at least as
|
| 748 |
session-invalidating as a password change, and it usually IS one.
|
| 749 |
"""
|
| 750 |
-
|
|
|
|
| 751 |
if not username:
|
| 752 |
raise ValueError('username and password are required')
|
| 753 |
# Refuse before the update so even an empty password names its broken rules and writes nothing.
|
|
@@ -769,6 +784,8 @@ def create_user(username, pw, name, role='user', bus='all', modules='all',
|
|
| 769 |
# their own plane. None = leave as it was; True/False = set it deliberately.
|
| 770 |
platform_admin=(prior.get('platform_admin') is True
|
| 771 |
if platform_admin is None else platform_admin is True))
|
|
|
|
|
|
|
| 772 |
if prior:
|
| 773 |
rec['epoch'] = int(prior.get('epoch') or 0) + 1
|
| 774 |
reg[username] = rec
|
|
@@ -782,6 +799,7 @@ def set_password(username, pw, enforce=True):
|
|
| 782 |
`enforce=False` is only for a caller that already validated (or a future migration) to rotate
|
| 783 |
without re-reading. The admin password route is that one caller; all others enforce here.
|
| 784 |
"""
|
|
|
|
| 785 |
username = (username or '').strip().lower()
|
| 786 |
if enforce:
|
| 787 |
try:
|
|
@@ -915,6 +933,8 @@ _STAMPS_LOCK = __import__('threading').Lock()
|
|
| 915 |
|
| 916 |
def _stamp(username, fields):
|
| 917 |
"""Merge `fields` into ONE account record, on a background thread, with a fresh read."""
|
|
|
|
|
|
|
| 918 |
username = (username or '').strip().lower()
|
| 919 |
if not username or not fields:
|
| 920 |
return None
|
|
@@ -926,6 +946,8 @@ def _stamp(username, fields):
|
|
| 926 |
return reg
|
| 927 |
|
| 928 |
def _work():
|
|
|
|
|
|
|
| 929 |
try:
|
| 930 |
store.update('users', _set) # sync: fresh strict read + blocking upload
|
| 931 |
except Exception:
|
|
|
|
| 21 |
from urllib.parse import quote
|
| 22 |
|
| 23 |
import core.store as store
|
| 24 |
+
from core import freeze_notice
|
| 25 |
|
| 26 |
BU_LABELS = {5: 'Fisch', 6: 'Royal'}
|
| 27 |
_ITER = 200_000
|
|
|
|
| 328 |
enrolment check lets two workers both issue a first set of codes; an epoch check prevents
|
| 329 |
regeneration racing an admin reset from silently reinstalling the lost factor.
|
| 330 |
"""
|
| 331 |
+
freeze_notice.check_write()
|
| 332 |
username = (username or '').strip().lower()
|
| 333 |
# The replay closure captures only hashes, never printable recovery codes: the HF store
|
| 334 |
# journals closures for conflict replay, and that journal must not retain clear codes.
|
|
|
|
| 382 |
def consume_recovery_code(username, code) -> bool:
|
| 383 |
username = (username or '').strip().lower()
|
| 384 |
wanted = _recovery_key(code)
|
| 385 |
+
if freeze_notice.enabled():
|
| 386 |
+
rec = (registry() or {}).get(username) or {}
|
| 387 |
+
matched = False
|
| 388 |
+
for stored in rec.get('recovery_codes') or []:
|
| 389 |
+
matched |= hmac.compare_digest(stored, wanted)
|
| 390 |
+
if matched:
|
| 391 |
+
freeze_notice.check_write()
|
| 392 |
+
return False
|
| 393 |
consumed = [False]
|
| 394 |
|
| 395 |
def _set(reg):
|
|
|
|
| 493 |
|
| 494 |
def note_login_failure(identifier, now=None):
|
| 495 |
"""Atomically count one failed password/MFA proof and return lock seconds remaining."""
|
| 496 |
+
if freeze_notice.enabled():
|
| 497 |
+
return login_lockout_remaining(identifier, now=now)
|
| 498 |
now = time.time() if now is None else float(now)
|
| 499 |
key = _login_lockout_key(identifier)
|
| 500 |
|
|
|
|
| 564 |
|
| 565 |
def clear_login_failures(identifier):
|
| 566 |
"""Clear the account's shared guessing budget after a complete successful proof."""
|
| 567 |
+
if freeze_notice.enabled():
|
| 568 |
+
return None
|
| 569 |
key = _login_lockout_key(identifier)
|
| 570 |
|
| 571 |
def _clear(doc):
|
|
|
|
| 761 |
the carry-and-bump lives here rather than in each caller: an overwrite is at least as
|
| 762 |
session-invalidating as a password change, and it usually IS one.
|
| 763 |
"""
|
| 764 |
+
original_username = (username or '').strip()
|
| 765 |
+
username = original_username.lower()
|
| 766 |
if not username:
|
| 767 |
raise ValueError('username and password are required')
|
| 768 |
# Refuse before the update so even an empty password names its broken rules and writes nothing.
|
|
|
|
| 784 |
# their own plane. None = leave as it was; True/False = set it deliberately.
|
| 785 |
platform_admin=(prior.get('platform_admin') is True
|
| 786 |
if platform_admin is None else platform_admin is True))
|
| 787 |
+
# Display spelling is separate from the key used by auth, grants and workspaces.
|
| 788 |
+
rec['username_original'] = prior.get('username_original') or original_username
|
| 789 |
if prior:
|
| 790 |
rec['epoch'] = int(prior.get('epoch') or 0) + 1
|
| 791 |
reg[username] = rec
|
|
|
|
| 799 |
`enforce=False` is only for a caller that already validated (or a future migration) to rotate
|
| 800 |
without re-reading. The admin password route is that one caller; all others enforce here.
|
| 801 |
"""
|
| 802 |
+
freeze_notice.check_write()
|
| 803 |
username = (username or '').strip().lower()
|
| 804 |
if enforce:
|
| 805 |
try:
|
|
|
|
| 933 |
|
| 934 |
def _stamp(username, fields):
|
| 935 |
"""Merge `fields` into ONE account record, on a background thread, with a fresh read."""
|
| 936 |
+
if freeze_notice.enabled():
|
| 937 |
+
return None
|
| 938 |
username = (username or '').strip().lower()
|
| 939 |
if not username or not fields:
|
| 940 |
return None
|
|
|
|
| 946 |
return reg
|
| 947 |
|
| 948 |
def _work():
|
| 949 |
+
if freeze_notice.enabled():
|
| 950 |
+
return
|
| 951 |
try:
|
| 952 |
store.update('users', _set) # sync: fresh strict read + blocking upload
|
| 953 |
except Exception:
|