Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +515 -5
- VERSION +1 -1
- api/amazon_relational.py +96 -63
- api/registry_source.py +15 -1
- api/routes_automation.py +6 -7
- api/routes_customers.py +4 -1
- api/routes_sheet.py +77 -66
- platform/core/amazon_rivals.py +2 -2
- platform/core/connector_spec.py +25 -23
- platform/core/received_changes.py +63 -34
- platform/core/sheet_binding.py +1 -1
- platform/harness/amazon_ads_client.py +470 -158
- platform/harness/amazon_ads_schema.py +96 -38
- platform/harness/amazon_ads_store.py +238 -171
- platform/harness/amazon_client.py +1 -1
- platform/harness/canonical_source.py +82 -16
- platform/modules/competitor_watch.py +2 -8
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,11 +1,521 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
-
"version": "
|
| 6 |
-
"sha": "
|
| 7 |
-
"date": "2026-11-
|
| 8 |
-
"subject": "release
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
}
|
| 10 |
]
|
| 11 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "caef761dd5af969155c5c31f930259757738efe4 (caef761dd5af9)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
+
"version": "v87",
|
| 6 |
+
"sha": "288b6b8793041",
|
| 7 |
+
"date": "2026-11-12",
|
| 8 |
+
"subject": "release(sdlc85): release lessons and log line"
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"version": "v86",
|
| 12 |
+
"sha": "108bd7510dcd2",
|
| 13 |
+
"date": "2026-10-27",
|
| 14 |
+
"subject": "release v86: SDLC 80 (owner override, 3 booked FAILs to SDLC 81)"
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"version": "v85",
|
| 18 |
+
"sha": "b36dd4992153e",
|
| 19 |
+
"date": "2026-10-25",
|
| 20 |
+
"subject": "Merge remote-tracking branch 'origin/master' into hfstage/sdlc79-release"
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"version": "v84",
|
| 24 |
+
"sha": "7bb3a2e09e138",
|
| 25 |
+
"date": "2026-10-24",
|
| 26 |
+
"subject": "release(sdlc78): v84 QA records the staging build change and what v84 excludes"
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"version": "v83",
|
| 30 |
+
"sha": "9b7471968e79d",
|
| 31 |
+
"date": "2026-10-21",
|
| 32 |
+
"subject": "release v83"
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"version": "v82",
|
| 36 |
+
"sha": "9c61b1ee94d0c",
|
| 37 |
+
"date": "2026-10-17",
|
| 38 |
+
"subject": "release v82"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"version": "v81",
|
| 42 |
+
"sha": "e141ccc85ef0c",
|
| 43 |
+
"date": "2026-10-15",
|
| 44 |
+
"subject": "release v81"
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"version": "v80",
|
| 48 |
+
"sha": "869b8fcc62ac7",
|
| 49 |
+
"date": "2026-10-13",
|
| 50 |
+
"subject": "release v80"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"version": "v79",
|
| 54 |
+
"sha": "406417b8ded0e",
|
| 55 |
+
"date": "2026-10-13",
|
| 56 |
+
"subject": "release v79"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"version": "v78",
|
| 60 |
+
"sha": "aad6561d63e6c",
|
| 61 |
+
"date": "2026-10-09",
|
| 62 |
+
"subject": "release v78: SDLC 71 with the D-7536 and D-7537 repairs"
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"version": "v77",
|
| 66 |
+
"sha": "e741e5ca604d4",
|
| 67 |
+
"date": "2026-10-05",
|
| 68 |
+
"subject": "release v77"
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"version": "v76",
|
| 72 |
+
"sha": "7990f03e02544",
|
| 73 |
+
"date": "2026-10-01",
|
| 74 |
+
"subject": "release v76"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"version": "v75",
|
| 78 |
+
"sha": "2c5146981ccfb",
|
| 79 |
+
"date": "2026-09-19",
|
| 80 |
+
"subject": "release v75"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"version": "v74",
|
| 84 |
+
"sha": "40dceba9262ec",
|
| 85 |
+
"date": "2026-09-14",
|
| 86 |
+
"subject": "release v74: waves 58+59 incl. the Amazon credential door (W59-T21/22/23) and sync producer (W59-T42); hotfix promote so"
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"version": "v73",
|
| 90 |
+
"sha": "bb94de5665e62",
|
| 91 |
+
"date": "2026-09-13",
|
| 92 |
+
"subject": "release v73"
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"version": "v71",
|
| 96 |
+
"sha": "96143dc6fb656",
|
| 97 |
+
"date": "2026-09-04",
|
| 98 |
+
"subject": "release v71 - wave 51, QA'd 24 PASS / 6 FAIL / 5 UNPROVEN"
|
| 99 |
+
},
|
| 100 |
+
{
|
| 101 |
+
"version": "v70",
|
| 102 |
+
"sha": "eedfb6a00163c",
|
| 103 |
+
"date": "2026-09-03",
|
| 104 |
+
"subject": "release v70 - wave 50 (grid analytics, the Sheet as a spreadsheet, view trash and audit)"
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"version": "v69",
|
| 108 |
+
"sha": "8598f56293a00",
|
| 109 |
+
"date": "2026-09-02",
|
| 110 |
+
"subject": "release v69: wave48-continuation validated + hotfixes v67/v68/d4e3e31 + go-live QA repairs (Supplier option limit, optio"
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"version": "v68",
|
| 114 |
+
"sha": "62b065a2610a4",
|
| 115 |
+
"date": "2026-09-01",
|
| 116 |
+
"subject": "release v68: unified holiday date offsets"
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"version": "v67",
|
| 120 |
+
"sha": "3b2241982cf73",
|
| 121 |
+
"date": "2026-09-01",
|
| 122 |
+
"subject": "release v67"
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"version": "v66",
|
| 126 |
+
"sha": "73c94f0767404",
|
| 127 |
+
"date": "2026-08-31",
|
| 128 |
+
"subject": "release v66"
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
"version": "v65",
|
| 132 |
+
"sha": "9f6cfedb8a43a",
|
| 133 |
+
"date": "2026-08-31",
|
| 134 |
+
"subject": "release v65"
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"version": "v64",
|
| 138 |
+
"sha": "bb8985c6e73e1",
|
| 139 |
+
"date": "2026-08-31",
|
| 140 |
+
"subject": "release v64"
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"version": "v63",
|
| 144 |
+
"sha": "ec75cb8f34129",
|
| 145 |
+
"date": "2026-08-31",
|
| 146 |
+
"subject": "release v63"
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"version": "v62",
|
| 150 |
+
"sha": "2c8f702c964fd",
|
| 151 |
+
"date": "2026-08-31",
|
| 152 |
+
"subject": "release v62"
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"version": "v61",
|
| 156 |
+
"sha": "bbc4c0760ab52",
|
| 157 |
+
"date": "2026-08-30",
|
| 158 |
+
"subject": "v61 - wave 46 hotfix: D-590 schema overlay reaches the grid read (all three assemblies), D-599 route to zone association"
|
| 159 |
+
},
|
| 160 |
+
{
|
| 161 |
+
"version": "v60",
|
| 162 |
+
"sha": "dd1b0620210e4",
|
| 163 |
+
"date": "2026-08-30",
|
| 164 |
+
"subject": "v60 - wave 46"
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"version": "v59",
|
| 168 |
+
"sha": "3cb73ce8021ed",
|
| 169 |
+
"date": "2026-08-29",
|
| 170 |
+
"subject": "release v59"
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"version": "v58",
|
| 174 |
+
"sha": "c526cede82f92",
|
| 175 |
+
"date": "2026-08-29",
|
| 176 |
+
"subject": "release v58"
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"version": "v57",
|
| 180 |
+
"sha": "4859c035e6a7d",
|
| 181 |
+
"date": "2026-08-29",
|
| 182 |
+
"subject": "release v57"
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"version": "v56",
|
| 186 |
+
"sha": "2970b729e8f04",
|
| 187 |
+
"date": "2026-08-28",
|
| 188 |
+
"subject": "release v56"
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"version": "v55",
|
| 192 |
+
"sha": "b766b904b8fa5",
|
| 193 |
+
"date": "2026-08-28",
|
| 194 |
+
"subject": "release v55"
|
| 195 |
+
},
|
| 196 |
+
{
|
| 197 |
+
"version": "v54",
|
| 198 |
+
"sha": "6a8e99a064d99",
|
| 199 |
+
"date": "2026-08-27",
|
| 200 |
+
"subject": "release v54"
|
| 201 |
+
},
|
| 202 |
+
{
|
| 203 |
+
"version": "v53",
|
| 204 |
+
"sha": "89ed2cb867da5",
|
| 205 |
+
"date": "2026-08-24",
|
| 206 |
+
"subject": "release v53"
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
"version": "v52",
|
| 210 |
+
"sha": "8263493762416",
|
| 211 |
+
"date": "2026-08-23",
|
| 212 |
+
"subject": "release v52"
|
| 213 |
+
},
|
| 214 |
+
{
|
| 215 |
+
"version": "v51",
|
| 216 |
+
"sha": "5740d20fa585e",
|
| 217 |
+
"date": "2026-08-23",
|
| 218 |
+
"subject": "release v51"
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
"version": "v50",
|
| 222 |
+
"sha": "8027c83d1da69",
|
| 223 |
+
"date": "2026-08-23",
|
| 224 |
+
"subject": "release v50"
|
| 225 |
+
},
|
| 226 |
+
{
|
| 227 |
+
"version": "v49",
|
| 228 |
+
"sha": "9a14c01a48a40",
|
| 229 |
+
"date": "2026-08-23",
|
| 230 |
+
"subject": "release v49"
|
| 231 |
+
},
|
| 232 |
+
{
|
| 233 |
+
"version": "v48",
|
| 234 |
+
"sha": "3c69fd93da7c4",
|
| 235 |
+
"date": "2026-08-23",
|
| 236 |
+
"subject": "release v48"
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"version": "v47",
|
| 240 |
+
"sha": "917f9a0eb5097",
|
| 241 |
+
"date": "2026-08-23",
|
| 242 |
+
"subject": "release v47"
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"version": "v46",
|
| 246 |
+
"sha": "a8dc531b5078e",
|
| 247 |
+
"date": "2026-08-23",
|
| 248 |
+
"subject": "release v46"
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"version": "v45",
|
| 252 |
+
"sha": "0b2b5010d86f0",
|
| 253 |
+
"date": "2026-08-23",
|
| 254 |
+
"subject": "release v45"
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"version": "v44",
|
| 258 |
+
"sha": "a7877aaf20ffd",
|
| 259 |
+
"date": "2026-08-23",
|
| 260 |
+
"subject": "Hotfix: allow renaming the active Map view"
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"version": "v43",
|
| 264 |
+
"sha": "e2ba4616fa0fd",
|
| 265 |
+
"date": "2026-08-23",
|
| 266 |
+
"subject": "release v43"
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"version": "v42",
|
| 270 |
+
"sha": "25b885e6373a7",
|
| 271 |
+
"date": "2026-08-23",
|
| 272 |
+
"subject": "release v42"
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"version": "v41",
|
| 276 |
+
"sha": "1d6adcc3c3ac5",
|
| 277 |
+
"date": "2026-08-22",
|
| 278 |
+
"subject": "v41 vertical route navigation"
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"version": "v40",
|
| 282 |
+
"sha": "1451837168f3c",
|
| 283 |
+
"date": "2026-08-22",
|
| 284 |
+
"subject": "release v40 route UI correction"
|
| 285 |
+
},
|
| 286 |
+
{
|
| 287 |
+
"version": "v39",
|
| 288 |
+
"sha": "a78bdf00e38e5",
|
| 289 |
+
"date": "2026-08-21",
|
| 290 |
+
"subject": "W39 clean stale migration targets"
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"version": "v38",
|
| 294 |
+
"sha": "a556631cf979d",
|
| 295 |
+
"date": "2026-08-21",
|
| 296 |
+
"subject": "W39 reversible Live store cutover"
|
| 297 |
+
},
|
| 298 |
+
{
|
| 299 |
+
"version": "v37",
|
| 300 |
+
"sha": "1ddd8cf82379b",
|
| 301 |
+
"date": "2026-08-21",
|
| 302 |
+
"subject": "W39: coalesce derived workspace reads"
|
| 303 |
+
},
|
| 304 |
+
{
|
| 305 |
+
"version": "v36",
|
| 306 |
+
"sha": "51db3b7fee2ce",
|
| 307 |
+
"date": "2026-08-21",
|
| 308 |
+
"subject": "W39 share-field oid repair"
|
| 309 |
+
},
|
| 310 |
+
{
|
| 311 |
+
"version": "v35",
|
| 312 |
+
"sha": "0f642129c8f77",
|
| 313 |
+
"date": "2026-08-21",
|
| 314 |
+
"subject": "W39 visual QA fixture"
|
| 315 |
+
},
|
| 316 |
+
{
|
| 317 |
+
"version": "v34",
|
| 318 |
+
"sha": "cb4caa188e05a",
|
| 319 |
+
"date": "2026-08-21",
|
| 320 |
+
"subject": "W39 Staging PostgreSQL egress controls"
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"version": "v33",
|
| 324 |
+
"sha": "2b115e573d6ab",
|
| 325 |
+
"date": "2026-08-21",
|
| 326 |
+
"subject": "W39: stop nonproduction Postgres background egress"
|
| 327 |
+
},
|
| 328 |
+
{
|
| 329 |
+
"version": "v32",
|
| 330 |
+
"sha": "b47b27ca7b2ae",
|
| 331 |
+
"date": "2026-08-21",
|
| 332 |
+
"subject": "W39: repair Staging Postgres projection"
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"version": "v31",
|
| 336 |
+
"sha": "8880a49a244ee",
|
| 337 |
+
"date": "2026-08-21",
|
| 338 |
+
"subject": "v31: Wave 39 staging candidate"
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"version": "v30",
|
| 342 |
+
"sha": "71776ca6c8dfa",
|
| 343 |
+
"date": "2026-08-20",
|
| 344 |
+
"subject": "v30: W39-T29 Member-owned View first-share hotfix"
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"version": "v29",
|
| 348 |
+
"sha": "55ced5018ff26",
|
| 349 |
+
"date": "2026-08-18",
|
| 350 |
+
"subject": "v29: the build staging has been running since 2026-08-18 (55ced50)."
|
| 351 |
+
},
|
| 352 |
+
{
|
| 353 |
+
"version": "v28",
|
| 354 |
+
"sha": "2793e4ba53dc1",
|
| 355 |
+
"date": "2026-08-18",
|
| 356 |
+
"subject": "release v28"
|
| 357 |
+
},
|
| 358 |
+
{
|
| 359 |
+
"version": "v27",
|
| 360 |
+
"sha": "c05a794837fc2",
|
| 361 |
+
"date": "2026-08-18",
|
| 362 |
+
"subject": "release v27"
|
| 363 |
+
},
|
| 364 |
+
{
|
| 365 |
+
"version": "v25",
|
| 366 |
+
"sha": "bc248295ed660",
|
| 367 |
+
"date": "2026-08-14",
|
| 368 |
+
"subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)"
|
| 369 |
+
},
|
| 370 |
+
{
|
| 371 |
+
"version": "v26",
|
| 372 |
+
"sha": "6a642e139cfaa",
|
| 373 |
+
"date": "2026-08-14",
|
| 374 |
+
"subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)"
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"version": "v24",
|
| 378 |
+
"sha": "514d8292ce734",
|
| 379 |
+
"date": "2026-08-09",
|
| 380 |
+
"subject": "release v24"
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"version": "v23",
|
| 384 |
+
"sha": "44cc28380fca6",
|
| 385 |
+
"date": "2026-08-09",
|
| 386 |
+
"subject": "release v23"
|
| 387 |
+
},
|
| 388 |
+
{
|
| 389 |
+
"version": "v22",
|
| 390 |
+
"sha": "f14e199ff32f4",
|
| 391 |
+
"date": "2026-08-09",
|
| 392 |
+
"subject": "release v22"
|
| 393 |
+
},
|
| 394 |
+
{
|
| 395 |
+
"version": "v21",
|
| 396 |
+
"sha": "5fdffce8b5586",
|
| 397 |
+
"date": "2026-08-09",
|
| 398 |
+
"subject": "release v21"
|
| 399 |
+
},
|
| 400 |
+
{
|
| 401 |
+
"version": "v20",
|
| 402 |
+
"sha": "442e4a1430a77",
|
| 403 |
+
"date": "2026-08-09",
|
| 404 |
+
"subject": "release v20"
|
| 405 |
+
},
|
| 406 |
+
{
|
| 407 |
+
"version": "v19",
|
| 408 |
+
"sha": "94cb4a4dfdfef",
|
| 409 |
+
"date": "2026-08-09",
|
| 410 |
+
"subject": "release v19"
|
| 411 |
+
},
|
| 412 |
+
{
|
| 413 |
+
"version": "v18",
|
| 414 |
+
"sha": "0106ef9498056",
|
| 415 |
+
"date": "2026-08-09",
|
| 416 |
+
"subject": "release v18"
|
| 417 |
+
},
|
| 418 |
+
{
|
| 419 |
+
"version": "v17",
|
| 420 |
+
"sha": "308240cc75e87",
|
| 421 |
+
"date": "2026-08-09",
|
| 422 |
+
"subject": "release v17"
|
| 423 |
+
},
|
| 424 |
+
{
|
| 425 |
+
"version": "v16",
|
| 426 |
+
"sha": "a804df54430f2",
|
| 427 |
+
"date": "2026-08-09",
|
| 428 |
+
"subject": "release v16"
|
| 429 |
+
},
|
| 430 |
+
{
|
| 431 |
+
"version": "v15",
|
| 432 |
+
"sha": "6b84d7264c81e",
|
| 433 |
+
"date": "2026-08-09",
|
| 434 |
+
"subject": "release v15"
|
| 435 |
+
},
|
| 436 |
+
{
|
| 437 |
+
"version": "v14",
|
| 438 |
+
"sha": "bee5eaf5b5cbd",
|
| 439 |
+
"date": "2026-08-07",
|
| 440 |
+
"subject": "release v14"
|
| 441 |
+
},
|
| 442 |
+
{
|
| 443 |
+
"version": "v13",
|
| 444 |
+
"sha": "314c0e368b1d2",
|
| 445 |
+
"date": "2026-08-06",
|
| 446 |
+
"subject": "release v13"
|
| 447 |
+
},
|
| 448 |
+
{
|
| 449 |
+
"version": "v12",
|
| 450 |
+
"sha": "4908f399785d2",
|
| 451 |
+
"date": "2026-08-06",
|
| 452 |
+
"subject": "release v12"
|
| 453 |
+
},
|
| 454 |
+
{
|
| 455 |
+
"version": "v11",
|
| 456 |
+
"sha": "312661712b980",
|
| 457 |
+
"date": "2026-08-05",
|
| 458 |
+
"subject": "release v11"
|
| 459 |
+
},
|
| 460 |
+
{
|
| 461 |
+
"version": "v10",
|
| 462 |
+
"sha": "43f8042874969",
|
| 463 |
+
"date": "2026-08-05",
|
| 464 |
+
"subject": "release v10"
|
| 465 |
+
},
|
| 466 |
+
{
|
| 467 |
+
"version": "v9",
|
| 468 |
+
"sha": "2e26f57396b19",
|
| 469 |
+
"date": "2026-08-05",
|
| 470 |
+
"subject": "release v9"
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"version": "v8",
|
| 474 |
+
"sha": "8073b3ecbac3d",
|
| 475 |
+
"date": "2026-08-05",
|
| 476 |
+
"subject": "release v8"
|
| 477 |
+
},
|
| 478 |
+
{
|
| 479 |
+
"version": "v7",
|
| 480 |
+
"sha": "b1c5a7e9b8486",
|
| 481 |
+
"date": "2026-08-05",
|
| 482 |
+
"subject": "release v7"
|
| 483 |
+
},
|
| 484 |
+
{
|
| 485 |
+
"version": "v6",
|
| 486 |
+
"sha": "a344020f114ac",
|
| 487 |
+
"date": "2026-08-05",
|
| 488 |
+
"subject": "release v6"
|
| 489 |
+
},
|
| 490 |
+
{
|
| 491 |
+
"version": "v5",
|
| 492 |
+
"sha": "c0b0fbb40a7b5",
|
| 493 |
+
"date": "2026-08-05",
|
| 494 |
+
"subject": "release v5"
|
| 495 |
+
},
|
| 496 |
+
{
|
| 497 |
+
"version": "v4",
|
| 498 |
+
"sha": "de5037ff1c8fd",
|
| 499 |
+
"date": "2026-08-05",
|
| 500 |
+
"subject": "release v4"
|
| 501 |
+
},
|
| 502 |
+
{
|
| 503 |
+
"version": "v3",
|
| 504 |
+
"sha": "6b9fa62d93e87",
|
| 505 |
+
"date": "2026-08-05",
|
| 506 |
+
"subject": "release v3"
|
| 507 |
+
},
|
| 508 |
+
{
|
| 509 |
+
"version": "v2",
|
| 510 |
+
"sha": "fd05861998586",
|
| 511 |
+
"date": "2026-08-04",
|
| 512 |
+
"subject": "release v2"
|
| 513 |
+
},
|
| 514 |
+
{
|
| 515 |
+
"version": "v1",
|
| 516 |
+
"sha": "5b4e2c4bc3f3c",
|
| 517 |
+
"date": "2026-08-04",
|
| 518 |
+
"subject": "release v1"
|
| 519 |
}
|
| 520 |
]
|
| 521 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
caef761dd5af969155c5c31f930259757738efe4 (caef761dd5af9)
|
api/amazon_relational.py
CHANGED
|
@@ -847,26 +847,38 @@ def _integer(value):
|
|
| 847 |
return None if value in (None, "") else int(value)
|
| 848 |
|
| 849 |
|
| 850 |
-
def
|
| 851 |
-
|
| 852 |
-
|
| 853 |
-
|
| 854 |
-
|
| 855 |
-
|
| 856 |
-
|
| 857 |
-
|
| 858 |
-
|
| 859 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 860 |
table = key.removeprefix(PREFIX)
|
| 861 |
-
prepared = {row[0] for row in cur.execute(
|
| 862 |
-
"SELECT column_name FROM information_schema.columns WHERE table_catalog = current_database() "
|
| 863 |
-
"AND table_schema = current_schema() AND table_name = ? AND column_name IN (?, ?, ?)",
|
| 864 |
-
[SOURCE[key], amazon_schema.IDENTITY_OCCURRENCE, amazon_schema.IDENTITY_PID,
|
| 865 |
-
amazon_schema.CONNECTION_ID]).fetchall()}
|
| 866 |
identity_names = names | prepared
|
| 867 |
identity = _identity_columns(table, identity_names)
|
| 868 |
-
fallback = sorted(set((*identity, *_fallback_columns(table))))
|
| 869 |
source = _quoted(SOURCE[key])
|
|
|
|
|
|
|
| 870 |
|
| 871 |
def value(name, owner=source):
|
| 872 |
return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
|
|
@@ -875,55 +887,62 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
|
|
| 875 |
def digest(values):
|
| 876 |
return f"sha256(to_json(list_value({', '.join(values)})))"
|
| 877 |
|
| 878 |
-
|
| 879 |
-
|
| 880 |
-
|
| 881 |
-
|
| 882 |
-
|
| 883 |
-
|
| 884 |
-
|
| 885 |
-
|
| 886 |
-
|
| 887 |
-
|
| 888 |
-
|
| 889 |
-
|
| 890 |
-
|
| 891 |
-
|
| 892 |
-
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
|
| 897 |
-
|
| 898 |
-
|
| 899 |
-
|
| 900 |
-
|
| 901 |
-
|
|
|
|
|
|
|
| 902 |
if amazon_schema.IDENTITY_PID in prepared:
|
| 903 |
ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}'
|
| 904 |
-
|
| 905 |
-
|
|
|
|
| 906 |
if legacy_owner:
|
| 907 |
literal = "'" + str(legacy_owner).replace("'", "''") + "'"
|
| 908 |
old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID})
|
| 909 |
-
|
| 910 |
-
old_complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in old_identity)
|
| 911 |
-
old_peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in old_identity)
|
| 912 |
-
old_partition = ", ".join(value(name, peer) for name in old_fallback)
|
| 913 |
-
old_occurrence = (
|
| 914 |
-
'(WITH "_amazon_legacy_identity" AS MATERIALIZED '
|
| 915 |
-
f'(SELECT {peer}.rowid AS "_identity_rowid", '
|
| 916 |
-
f'row_number() OVER (PARTITION BY {old_partition} ORDER BY {peer}.rowid) - 1 '
|
| 917 |
-
f'AS "_identity_occurrence" FROM {source} AS {peer} '
|
| 918 |
-
f'WHERE {peer}."_connection_id"={literal} AND NOT ({old_peer_complete})) '
|
| 919 |
-
'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_legacy_identity" '
|
| 920 |
-
f'WHERE "_identity_rowid" = {source}.rowid)')
|
| 921 |
-
old_missing = digest(["'__amazon_missing_identity__'", *(value(name) for name in old_fallback),
|
| 922 |
-
old_occurrence])
|
| 923 |
-
old_digest = (f"CASE WHEN {old_complete} THEN {digest([value(name) for name in old_identity])} "
|
| 924 |
-
f"ELSE {old_missing} END")
|
| 925 |
-
old_ident = f"greatest(1, CAST(('0x' || substr(({old_digest}), 1, 13)) AS BIGINT))"
|
| 926 |
ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 927 |
cols = {}
|
| 928 |
personal = set(amazon_store.personal_columns(table))
|
| 929 |
for name, dtype in columns:
|
|
@@ -942,10 +961,8 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
|
|
| 942 |
# group (count by parent) and a sort compile against them in SQL with no per-row Python.
|
| 943 |
for lookup in lookups(cur, key, names, allowed=allow_lookups):
|
| 944 |
cols[lookup["key"]] = (lookup["expr"], _text)
|
| 945 |
-
spec = {"table": SOURCE[key], "where": "", "
|
| 946 |
-
|
| 947 |
-
if table not in amazon_store.DERIVED_TABLES:
|
| 948 |
-
spec["page_key"] = f"{source}.rowid"
|
| 949 |
# D-2643: an empty table carries its code and reason to every door that reads this binding.
|
| 950 |
status = source_status(cur, key) if status is _UNRESOLVED else status
|
| 951 |
if status is not None:
|
|
@@ -953,6 +970,22 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
|
|
| 953 |
return spec
|
| 954 |
|
| 955 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 956 |
def page_source(cur, key, definition=None, *, allow_lookups=True):
|
| 957 |
"""Resolve one target's storage policy on the admitted cursor without provider discovery."""
|
| 958 |
columns = mirror_columns(cur, SOURCE[key], strict=True)
|
|
|
|
| 847 |
return None if value in (None, "") else int(value)
|
| 848 |
|
| 849 |
|
| 850 |
+
def _mirror_schemas(cur, tables):
|
| 851 |
+
rows = cur.execute(
|
| 852 |
+
'SELECT c.table_name, c.column_name, c.data_type, t.table_type '
|
| 853 |
+
'FROM information_schema.columns c JOIN information_schema.tables t '
|
| 854 |
+
'USING (table_catalog, table_schema, table_name) '
|
| 855 |
+
'WHERE c.table_catalog = current_database() AND c.table_schema = current_schema() '
|
| 856 |
+
'AND c.table_name IN (SELECT unnest(?::VARCHAR[])) ORDER BY c.table_name, c.ordinal_position',
|
| 857 |
+
[list(tables)]).fetchall()
|
| 858 |
+
schemas = {}
|
| 859 |
+
for table, name, kind, table_type in rows:
|
| 860 |
+
columns, _view = schemas.setdefault(table, ([], table_type == 'VIEW'))
|
| 861 |
+
columns.append((name, kind))
|
| 862 |
+
return schemas
|
| 863 |
+
|
| 864 |
+
|
| 865 |
+
def _missing_occurrence(source, partition, complete, predicate=''):
|
| 866 |
+
return ('(WITH "_amazon_missing_identity" AS MATERIALIZED '
|
| 867 |
+
f'(SELECT "_amazon_identity_peer".rowid AS "_identity_rowid", '
|
| 868 |
+
f'row_number() OVER (PARTITION BY {partition} ORDER BY "_amazon_identity_peer".rowid) - 1 '
|
| 869 |
+
f'AS "_identity_occurrence" FROM {source} AS "_amazon_identity_peer" '
|
| 870 |
+
f'WHERE {predicate}NOT ({complete})) '
|
| 871 |
+
'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_missing_identity" '
|
| 872 |
+
f'WHERE "_identity_rowid" = {source}.rowid)')
|
| 873 |
+
|
| 874 |
+
|
| 875 |
+
def _grid_identity(cur, key, names, prepared, is_view):
|
| 876 |
table = key.removeprefix(PREFIX)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 877 |
identity_names = names | prepared
|
| 878 |
identity = _identity_columns(table, identity_names)
|
|
|
|
| 879 |
source = _quoted(SOURCE[key])
|
| 880 |
+
peer = '"_amazon_identity_peer"'
|
| 881 |
+
ranks = []
|
| 882 |
|
| 883 |
def value(name, owner=source):
|
| 884 |
return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
|
|
|
|
| 887 |
def digest(values):
|
| 888 |
return f"sha256(to_json(list_value({', '.join(values)})))"
|
| 889 |
|
| 890 |
+
def identity_sql(keys, label, legacy=None):
|
| 891 |
+
if table in amazon_store.DERIVED_TABLES:
|
| 892 |
+
hashed = digest([value(name) for name in keys])
|
| 893 |
+
return f"greatest(1, CAST(('0x' || substr(({hashed}), 1, 13)) AS BIGINT))"
|
| 894 |
+
complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in keys)
|
| 895 |
+
fallback = sorted(set((*keys, *_fallback_columns(table))))
|
| 896 |
+
partition = ", ".join(value(name, peer) for name in fallback)
|
| 897 |
+
if is_view:
|
| 898 |
+
if legacy:
|
| 899 |
+
partition += f', ({peer}."_connection_id" = {legacy})'
|
| 900 |
+
ranks.append(f'row_number() OVER (PARTITION BY {partition} '
|
| 901 |
+
f'ORDER BY {peer}."_amazon_publication_order") - 1 AS {_quoted(label)}')
|
| 902 |
+
occurrence = f'CAST({source}.{_quoted(label)} AS VARCHAR)'
|
| 903 |
+
else:
|
| 904 |
+
peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in keys)
|
| 905 |
+
predicate = f'{peer}."_connection_id" = {legacy} AND ' if legacy else ''
|
| 906 |
+
occurrence = _missing_occurrence(source, partition, peer_complete, predicate)
|
| 907 |
+
if not legacy and amazon_schema.IDENTITY_OCCURRENCE in prepared:
|
| 908 |
+
occurrence = f'CAST({source}.{_quoted(amazon_schema.IDENTITY_OCCURRENCE)} AS VARCHAR)'
|
| 909 |
+
if is_view:
|
| 910 |
+
ranks.pop()
|
| 911 |
+
fallback_digest = digest(["'__amazon_missing_identity__'", *(value(name) for name in fallback), occurrence])
|
| 912 |
+
hashed = digest([value(name) for name in keys])
|
| 913 |
+
hashed = f'CASE WHEN {complete} THEN {hashed} ELSE {fallback_digest} END'
|
| 914 |
+
return f"greatest(1, CAST(('0x' || substr(({hashed}), 1, 13)) AS BIGINT))"
|
| 915 |
+
|
| 916 |
if amazon_schema.IDENTITY_PID in prepared:
|
| 917 |
ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}'
|
| 918 |
+
else:
|
| 919 |
+
ident = identity_sql(identity, '_amazon_publication_occurrence')
|
| 920 |
+
legacy_owner = amazon_store.legacy_identity_owner(cur, table) if amazon_schema.CONNECTION_ID in prepared else None
|
| 921 |
if legacy_owner:
|
| 922 |
literal = "'" + str(legacy_owner).replace("'", "''") + "'"
|
| 923 |
old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID})
|
| 924 |
+
old_ident = identity_sql(old_identity, '_amazon_publication_legacy_occurrence', literal)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 925 |
ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
|
| 926 |
+
spec = {'id': ident}
|
| 927 |
+
if ranks:
|
| 928 |
+
spec['from_sql'] = (f'(SELECT *, {", ".join(ranks)} FROM '
|
| 929 |
+
f'(SELECT *, row_number() OVER () - 1 AS "_amazon_publication_order" '
|
| 930 |
+
f'FROM {source}) AS {peer}) AS {source}')
|
| 931 |
+
if not is_view and table not in amazon_store.DERIVED_TABLES:
|
| 932 |
+
spec['page_key'] = f'{source}.rowid'
|
| 933 |
+
return spec
|
| 934 |
+
|
| 935 |
+
|
| 936 |
+
def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVED, schema=None):
|
| 937 |
+
"""Bind the canonical grid identity and fields to the admitted table or publication view."""
|
| 938 |
+
raw, is_view = schema if schema is not None else _mirror_schemas(cur, [SOURCE[key]]).get(SOURCE[key], ([], False))
|
| 939 |
+
columns = raw if columns is None else columns
|
| 940 |
+
columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
|
| 941 |
+
if not columns:
|
| 942 |
+
return missing_source(key, cur=cur)
|
| 943 |
+
names = {name for name, _dtype in columns}
|
| 944 |
+
prepared = {name for name, _kind in raw} & amazon_schema.MIRROR_INTERNAL_COLUMNS
|
| 945 |
+
table = key.removeprefix(PREFIX)
|
| 946 |
cols = {}
|
| 947 |
personal = set(amazon_store.personal_columns(table))
|
| 948 |
for name, dtype in columns:
|
|
|
|
| 961 |
# group (count by parent) and a sort compile against them in SQL with no per-row Python.
|
| 962 |
for lookup in lookups(cur, key, names, allowed=allow_lookups):
|
| 963 |
cols[lookup["key"]] = (lookup["expr"], _text)
|
| 964 |
+
spec = {"table": SOURCE[key], "where": "", "needs_excluded": False, "cols": cols,
|
| 965 |
+
**_grid_identity(cur, key, names, prepared, is_view)}
|
|
|
|
|
|
|
| 966 |
# D-2643: an empty table carries its code and reason to every door that reads this binding.
|
| 967 |
status = source_status(cur, key) if status is _UNRESOLVED else status
|
| 968 |
if status is not None:
|
|
|
|
| 970 |
return spec
|
| 971 |
|
| 972 |
|
| 973 |
+
def publication_sources(cur, keys):
|
| 974 |
+
"""Resolve completed raw publications once without request readiness or store discovery."""
|
| 975 |
+
keys = [key for key in keys if key in SOURCE]
|
| 976 |
+
schemas = _mirror_schemas(cur, [SOURCE[key] for key in keys])
|
| 977 |
+
result = {}
|
| 978 |
+
for key in keys:
|
| 979 |
+
schema = schemas.get(SOURCE[key])
|
| 980 |
+
if schema is None:
|
| 981 |
+
continue
|
| 982 |
+
spec = grid_source(cur, key, allow_lookups=False, status=None, schema=schema)
|
| 983 |
+
if amazon_schema.CONNECTION_ID in dict(schema[0]):
|
| 984 |
+
spec['cols'][amazon_schema.CONNECTION_ID] = ('"_connection_id"', _text)
|
| 985 |
+
result[key] = spec
|
| 986 |
+
return result
|
| 987 |
+
|
| 988 |
+
|
| 989 |
def page_source(cur, key, definition=None, *, allow_lookups=True):
|
| 990 |
"""Resolve one target's storage policy on the admitted cursor without provider discovery."""
|
| 991 |
columns = mirror_columns(cur, SOURCE[key], strict=True)
|
api/registry_source.py
CHANGED
|
@@ -526,6 +526,19 @@ class CanonicalStore(BatchStore):
|
|
| 526 |
key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 527 |
self.pool_cache[key] = (time.time(), rows)
|
| 528 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 529 |
def get_projection(self, name, drop=()):
|
| 530 |
if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
|
| 531 |
return self.get(name)
|
|
@@ -846,6 +859,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
|
|
| 846 |
# A caller's request snapshot carries the generation taken before its first read.
|
| 847 |
metadata = {'_scan': ids is None} if metadata is None else metadata
|
| 848 |
before = metadata.get('_generation') or generation(session, table_key)
|
|
|
|
| 849 |
# A whole-field read needs no schema pass: its one row assembly serves every visible field.
|
| 850 |
definition = metadata.get('_definition') or (None if needed is None else
|
| 851 |
canonical_metadata(session, table_key, metadata=metadata, cur=cur))
|
|
@@ -890,7 +904,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
|
|
| 890 |
cached = session.runtime.pool_cache.get(pool_key)
|
| 891 |
fresh = bool(cached) and time.time() - cached[0] < 900
|
| 892 |
# A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
|
| 893 |
-
paused = not fresh and routes_keychain.odoo_paused(
|
| 894 |
# One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
|
| 895 |
size = None if table_key == 'product_data' else BATCH_ROWS
|
| 896 |
if cached and (paused or fresh):
|
|
|
|
| 526 |
key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 527 |
self.pool_cache[key] = (time.time(), rows)
|
| 528 |
|
| 529 |
+
def revision(self, name):
|
| 530 |
+
before = self.metadata.get('_generation')
|
| 531 |
+
mapped = getattr(self._runtime, 'store_key', str)(name)
|
| 532 |
+
if before is not None and mapped in before[1]:
|
| 533 |
+
return copy.deepcopy(before[1][mapped])
|
| 534 |
+
return self._runtime.revision(name)
|
| 535 |
+
|
| 536 |
+
def odoo_paused(self):
|
| 537 |
+
slot = ('odoo_paused',)
|
| 538 |
+
if slot not in self.metadata:
|
| 539 |
+
self.metadata[slot] = self._runtime.odoo_paused()
|
| 540 |
+
return self.metadata[slot]
|
| 541 |
+
|
| 542 |
def get_projection(self, name, drop=()):
|
| 543 |
if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
|
| 544 |
return self.get(name)
|
|
|
|
| 859 |
# A caller's request snapshot carries the generation taken before its first read.
|
| 860 |
metadata = {'_scan': ids is None} if metadata is None else metadata
|
| 861 |
before = metadata.get('_generation') or generation(session, table_key)
|
| 862 |
+
metadata['_generation'] = before
|
| 863 |
# A whole-field read needs no schema pass: its one row assembly serves every visible field.
|
| 864 |
definition = metadata.get('_definition') or (None if needed is None else
|
| 865 |
canonical_metadata(session, table_key, metadata=metadata, cur=cur))
|
|
|
|
| 904 |
cached = session.runtime.pool_cache.get(pool_key)
|
| 905 |
fresh = bool(cached) and time.time() - cached[0] < 900
|
| 906 |
# A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
|
| 907 |
+
paused = not fresh and routes_keychain.odoo_paused(runtime)
|
| 908 |
# One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
|
| 909 |
size = None if table_key == 'product_data' else BATCH_ROWS
|
| 910 |
if cached and (paused or fresh):
|
api/routes_automation.py
CHANGED
|
@@ -2873,14 +2873,13 @@ def _received_projection(connector):
|
|
| 2873 |
from fastapi import HTTPException
|
| 2874 |
from core import user_tables
|
| 2875 |
import routes_odoo_tables
|
| 2876 |
-
|
| 2877 |
if connector == 'amazon':
|
| 2878 |
-
|
| 2879 |
-
|
| 2880 |
-
|
| 2881 |
-
|
| 2882 |
-
|
| 2883 |
-
for key in user_tables.all_defs(st=rt, schema_only=True):
|
| 2884 |
if not str(key).startswith(prefix):
|
| 2885 |
continue
|
| 2886 |
try:
|
|
|
|
| 2873 |
from fastapi import HTTPException
|
| 2874 |
from core import user_tables
|
| 2875 |
import routes_odoo_tables
|
| 2876 |
+
definitions = user_tables.all_defs(st=rt, schema_only=True)
|
| 2877 |
if connector == 'amazon':
|
| 2878 |
+
import amazon_relational
|
| 2879 |
+
return amazon_relational.publication_sources(cursor, definitions)
|
| 2880 |
+
result = {}
|
| 2881 |
+
prefix = 'ut_odoo_'
|
| 2882 |
+
for key in definitions:
|
|
|
|
| 2883 |
if not str(key).startswith(prefix):
|
| 2884 |
continue
|
| 2885 |
try:
|
api/routes_customers.py
CHANGED
|
@@ -138,7 +138,10 @@ def _assembly_revisions(runtime, names, uses_roster=False, roster_tenant=None, r
|
|
| 138 |
"""Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
|
| 139 |
from core import store, store_pg
|
| 140 |
names = tuple(sorted(set(names)))
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
| 142 |
users_stamp = None
|
| 143 |
|
| 144 |
def roster_stamp(revision):
|
|
|
|
| 138 |
"""Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
|
| 139 |
from core import store, store_pg
|
| 140 |
names = tuple(sorted(set(names)))
|
| 141 |
+
# The outer canonical generation check validates these borrowed pre-read revisions after hydration.
|
| 142 |
+
canonical = (getattr(runtime, '_canonical_read_only', False)
|
| 143 |
+
and getattr(runtime, 'metadata', {}).get('_generation') is not None)
|
| 144 |
+
handle = None if canonical else _assembly_pg_store(runtime)
|
| 145 |
users_stamp = None
|
| 146 |
|
| 147 |
def roster_stamp(revision):
|
api/routes_sheet.py
CHANGED
|
@@ -7084,23 +7084,20 @@ def _drill_tokens(session, databases):
|
|
| 7084 |
|
| 7085 |
@contextlib.contextmanager
|
| 7086 |
def _drill_mirror(session, database, window, needed, databases, sources=None):
|
| 7087 |
-
"""
|
| 7088 |
-
|
| 7089 |
-
`sources` is the caller's opening token pass. The lease key reuses it, so one request reads
|
| 7090 |
-
the tokens once before the build. The build-miss and end-of-request rechecks still read anew.
|
| 7091 |
-
"""
|
| 7092 |
from core import sheet_binding, perm_scope
|
| 7093 |
from core.request_timing import raise_if_resource_error
|
| 7094 |
import datetime
|
| 7095 |
|
| 7096 |
for key in sorted(databases):
|
| 7097 |
_require_caught_up(session, key)
|
|
|
|
| 7098 |
if _native_table_definition(session, database) is not None:
|
| 7099 |
# Native rows are a fresh scoped Data payload. Never cache their cursor/permissions
|
| 7100 |
# across requests; the request's ExitStack closes it after the page is produced.
|
| 7101 |
yield _mirror(session)(database, metric_window=window, needed=needed), None
|
|
|
|
| 7102 |
return
|
| 7103 |
-
sources = _drill_tokens(session, databases) if sources is None else dict(sources)
|
| 7104 |
token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
|
| 7105 |
# C1 also fingerprints derived wall sources, which can change without the primary table.
|
| 7106 |
# An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
|
|
@@ -7118,23 +7115,11 @@ def _drill_mirror(session, database, window, needed, databases, sources=None):
|
|
| 7118 |
getattr(session.runtime.tenant, "config", None)])
|
| 7119 |
key = (session.tenant, session.uname, database, token, reader,
|
| 7120 |
sheet_binding.fingerprint(window), datetime.date.today().isoformat())
|
| 7121 |
-
attrs = ("_sheet_drill_cursor", "_sheet_drill_isolated", "_sheet_mirrors",
|
| 7122 |
-
"_sheet_assemblies", "_sheet_stored_mirrors", "_sheet_amzn_cursor",
|
| 7123 |
-
"_sheet_drill_entries")
|
| 7124 |
-
absent = object()
|
| 7125 |
-
previous = {name: getattr(session, name, absent) for name in attrs}
|
| 7126 |
with _drill_lease(key if token else None) as lease:
|
| 7127 |
with _drill_mirror_lock():
|
| 7128 |
hit = _DRILL_MIRRORS.get(key) if token else None
|
| 7129 |
lease["entry"] = hit
|
| 7130 |
-
session
|
| 7131 |
-
session._sheet_drill_cursor = hit["owned"] if hit else None
|
| 7132 |
-
session._sheet_drill_entries = hit["mirrors"] if hit else {}
|
| 7133 |
-
session._sheet_mirrors = {}
|
| 7134 |
-
session._sheet_assemblies = {}
|
| 7135 |
-
session._sheet_stored_mirrors = {}
|
| 7136 |
-
session._sheet_amzn_cursor = session._sheet_drill_cursor
|
| 7137 |
-
try:
|
| 7138 |
want = set(needed) | (hit["needed"] if hit else set())
|
| 7139 |
if hit and want <= hit["needed"]:
|
| 7140 |
made = hit["made"]
|
|
@@ -7150,42 +7135,70 @@ def _drill_mirror(session, database, window, needed, databases, sources=None):
|
|
| 7150 |
owned = getattr(session, "_sheet_drill_cursor", None)
|
| 7151 |
hit = {"made": made, "needed": want, "owned": owned,
|
| 7152 |
"mirrors": session._sheet_drill_entries}
|
| 7153 |
-
#
|
| 7154 |
-
if token
|
| 7155 |
-
|
| 7156 |
-
_DRILL_MIRRORS[key] = hit
|
| 7157 |
-
_DRILL_MIRRORS.move_to_end(key)
|
| 7158 |
-
lease["entry"] = hit
|
| 7159 |
-
while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
|
| 7160 |
-
gone, evicted = _DRILL_MIRRORS.popitem(last=False)
|
| 7161 |
-
# A leased cursor closes when its holder leaves, never under it.
|
| 7162 |
-
held = _DRILL_LEASES.get(gone)
|
| 7163 |
-
if evicted["owned"] and not (held and held["entry"] is evicted):
|
| 7164 |
-
evicted["owned"][0].close()
|
| 7165 |
-
else:
|
| 7166 |
-
token = None
|
| 7167 |
# Related leaves may revisit the primary registry. Keep its table intact and widen
|
| 7168 |
# only within this cursor. Assemblies remain request-local and leave with the call.
|
| 7169 |
session._sheet_mirrors[(database, json.dumps(window, sort_keys=True, default=str))] = (
|
| 7170 |
want, made)
|
| 7171 |
-
|
| 7172 |
-
|
| 7173 |
-
|
| 7174 |
-
|
| 7175 |
-
|
| 7176 |
-
|
| 7177 |
-
|
| 7178 |
-
|
| 7179 |
-
|
| 7180 |
-
|
| 7181 |
-
|
| 7182 |
-
|
| 7183 |
-
|
| 7184 |
-
|
| 7185 |
-
|
| 7186 |
-
|
| 7187 |
-
|
| 7188 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7189 |
|
| 7190 |
|
| 7191 |
def _drill_page(session, database, record_ids, offset, limit, total):
|
|
@@ -7533,21 +7546,19 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
|
| 7533 |
binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
|
| 7534 |
if all(sources.values()) and wall_stamp else None
|
| 7535 |
|
| 7536 |
-
|
| 7537 |
-
|
| 7538 |
-
|
| 7539 |
-
|
| 7540 |
-
|
| 7541 |
-
|
| 7542 |
-
|
| 7543 |
-
|
| 7544 |
-
|
| 7545 |
-
|
| 7546 |
-
|
| 7547 |
-
|
| 7548 |
-
|
| 7549 |
-
if _drill_tokens(session, databases) != sources:
|
| 7550 |
-
raise err(409, "drill_population_changed", "These records changed while loading. Try again.")
|
| 7551 |
out["token"] = token
|
| 7552 |
return out
|
| 7553 |
|
|
|
|
| 7084 |
|
| 7085 |
@contextlib.contextmanager
|
| 7086 |
def _drill_mirror(session, database, window, needed, databases, sources=None):
|
| 7087 |
+
"""Lease a reader-scoped mirror and validate its source tokens after page hydration."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7088 |
from core import sheet_binding, perm_scope
|
| 7089 |
from core.request_timing import raise_if_resource_error
|
| 7090 |
import datetime
|
| 7091 |
|
| 7092 |
for key in sorted(databases):
|
| 7093 |
_require_caught_up(session, key)
|
| 7094 |
+
sources = _drill_tokens(session, databases) if sources is None else dict(sources)
|
| 7095 |
if _native_table_definition(session, database) is not None:
|
| 7096 |
# Native rows are a fresh scoped Data payload. Never cache their cursor/permissions
|
| 7097 |
# across requests; the request's ExitStack closes it after the page is produced.
|
| 7098 |
yield _mirror(session)(database, metric_window=window, needed=needed), None
|
| 7099 |
+
_require_drill_tokens(session, databases, sources)
|
| 7100 |
return
|
|
|
|
| 7101 |
token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
|
| 7102 |
# C1 also fingerprints derived wall sources, which can change without the primary table.
|
| 7103 |
# An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
|
|
|
|
| 7115 |
getattr(session.runtime.tenant, "config", None)])
|
| 7116 |
key = (session.tenant, session.uname, database, token, reader,
|
| 7117 |
sheet_binding.fingerprint(window), datetime.date.today().isoformat())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7118 |
with _drill_lease(key if token else None) as lease:
|
| 7119 |
with _drill_mirror_lock():
|
| 7120 |
hit = _DRILL_MIRRORS.get(key) if token else None
|
| 7121 |
lease["entry"] = hit
|
| 7122 |
+
with _drill_reader(session, hit, lease):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7123 |
want = set(needed) | (hit["needed"] if hit else set())
|
| 7124 |
if hit and want <= hit["needed"]:
|
| 7125 |
made = hit["made"]
|
|
|
|
| 7135 |
owned = getattr(session, "_sheet_drill_cursor", None)
|
| 7136 |
hit = {"made": made, "needed": want, "owned": owned,
|
| 7137 |
"mirrors": session._sheet_drill_entries}
|
| 7138 |
+
# Same-key readers wait for this lease until final validation succeeds.
|
| 7139 |
+
if token:
|
| 7140 |
+
_retain_drill_mirror(key, hit, lease)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7141 |
# Related leaves may revisit the primary registry. Keep its table intact and widen
|
| 7142 |
# only within this cursor. Assemblies remain request-local and leave with the call.
|
| 7143 |
session._sheet_mirrors[(database, json.dumps(window, sort_keys=True, default=str))] = (
|
| 7144 |
want, made)
|
| 7145 |
+
try:
|
| 7146 |
+
yield made, token
|
| 7147 |
+
_require_drill_tokens(session, databases, sources)
|
| 7148 |
+
except BaseException:
|
| 7149 |
+
with _DRILL_MIRROR_LOCK:
|
| 7150 |
+
if _DRILL_MIRRORS.get(key) is hit:
|
| 7151 |
+
_DRILL_MIRRORS.pop(key)
|
| 7152 |
+
raise
|
| 7153 |
+
|
| 7154 |
+
|
| 7155 |
+
@contextlib.contextmanager
|
| 7156 |
+
def _drill_reader(session, hit, lease):
|
| 7157 |
+
"""Restore request attributes and close cursors whose leases no longer retain them."""
|
| 7158 |
+
attrs = ("_sheet_drill_cursor", "_sheet_drill_isolated", "_sheet_mirrors",
|
| 7159 |
+
"_sheet_assemblies", "_sheet_stored_mirrors", "_sheet_amzn_cursor", "_sheet_drill_entries")
|
| 7160 |
+
absent = object()
|
| 7161 |
+
previous = {name: getattr(session, name, absent) for name in attrs}
|
| 7162 |
+
session._sheet_drill_isolated = True
|
| 7163 |
+
session._sheet_drill_cursor = hit['owned'] if hit else None
|
| 7164 |
+
session._sheet_drill_entries = hit['mirrors'] if hit else {}
|
| 7165 |
+
session._sheet_mirrors, session._sheet_assemblies, session._sheet_stored_mirrors = {}, {}, {}
|
| 7166 |
+
session._sheet_amzn_cursor = session._sheet_drill_cursor
|
| 7167 |
+
try:
|
| 7168 |
+
yield
|
| 7169 |
+
finally:
|
| 7170 |
+
owned = getattr(session, '_sheet_drill_cursor', None)
|
| 7171 |
+
entry = lease['entry']
|
| 7172 |
+
with _DRILL_MIRROR_LOCK:
|
| 7173 |
+
lease['entry'] = None
|
| 7174 |
+
kept = [item['owned'] for item in _DRILL_MIRRORS.values()]
|
| 7175 |
+
spent = [owned, entry['owned'] if entry else None]
|
| 7176 |
+
for index, cursor in enumerate(spent):
|
| 7177 |
+
if (cursor and not any(seen is cursor for seen in spent[:index])
|
| 7178 |
+
and not any(item is cursor for item in kept)):
|
| 7179 |
+
cursor[0].close()
|
| 7180 |
+
for name, value in previous.items():
|
| 7181 |
+
if value is absent:
|
| 7182 |
+
session.__dict__.pop(name, None)
|
| 7183 |
+
else:
|
| 7184 |
+
setattr(session, name, value)
|
| 7185 |
+
|
| 7186 |
+
|
| 7187 |
+
def _require_drill_tokens(session, databases, sources):
|
| 7188 |
+
if _drill_tokens(session, databases) != sources:
|
| 7189 |
+
raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
|
| 7190 |
+
|
| 7191 |
+
|
| 7192 |
+
def _retain_drill_mirror(key, hit, lease):
|
| 7193 |
+
with _DRILL_MIRROR_LOCK:
|
| 7194 |
+
_DRILL_MIRRORS[key] = hit
|
| 7195 |
+
_DRILL_MIRRORS.move_to_end(key)
|
| 7196 |
+
lease['entry'] = hit
|
| 7197 |
+
while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
|
| 7198 |
+
gone, evicted = _DRILL_MIRRORS.popitem(last=False)
|
| 7199 |
+
held = _DRILL_LEASES.get(gone)
|
| 7200 |
+
if evicted['owned'] and not (held and held['entry'] is evicted):
|
| 7201 |
+
evicted['owned'][0].close()
|
| 7202 |
|
| 7203 |
|
| 7204 |
def _drill_page(session, database, record_ids, offset, limit, total):
|
|
|
|
| 7546 |
binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
|
| 7547 |
if all(sources.values()) and wall_stamp else None
|
| 7548 |
|
| 7549 |
+
limits = list(limits) + list(spec.get("limits") or [])
|
| 7550 |
+
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
|
| 7551 |
+
"record_ids_scope": "page", "id_limit": None, "limits": limits,
|
| 7552 |
+
"population": population, "query": query}
|
| 7553 |
+
if scope.get("view_id"):
|
| 7554 |
+
out["view_id"] = scope["view_id"]
|
| 7555 |
+
else:
|
| 7556 |
+
out["filter"] = scope.get("filter")
|
| 7557 |
+
if summary is not None:
|
| 7558 |
+
out["summary"] = {"ok": True, **summary}
|
| 7559 |
+
else:
|
| 7560 |
+
out["page"] = _drill_page(session, database, record_ids, offset, limit, total)
|
| 7561 |
+
out["page"]["totalUnfiltered"] = population_total
|
|
|
|
|
|
|
| 7562 |
out["token"] = token
|
| 7563 |
return out
|
| 7564 |
|
platform/core/amazon_rivals.py
CHANGED
|
@@ -121,11 +121,11 @@ def _backend(rt):
|
|
| 121 |
# --------------------------------------------------------------------------- mirror reads
|
| 122 |
|
| 123 |
def weekly_period_sql():
|
| 124 |
-
"""
|
| 125 |
start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
|
| 126 |
end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
|
| 127 |
return (f"isfinite({start}) AND isfinite({end}) AND "
|
| 128 |
-
f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE))
|
| 129 |
|
| 130 |
|
| 131 |
def _week_sql():
|
|
|
|
| 121 |
# --------------------------------------------------------------------------- mirror reads
|
| 122 |
|
| 123 |
def weekly_period_sql():
|
| 124 |
+
"""Admit connector weeks with exclusive ends and legacy weeks with inclusive ends."""
|
| 125 |
start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
|
| 126 |
end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
|
| 127 |
return (f"isfinite({start}) AND isfinite({end}) AND "
|
| 128 |
+
f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) IN (6, 7)")
|
| 129 |
|
| 130 |
|
| 131 |
def _week_sql():
|
platform/core/connector_spec.py
CHANGED
|
@@ -3785,15 +3785,15 @@ AMAZON_ADS_REPORTS = ("campaigns", "targeting", "advertised_products", "search_t
|
|
| 3785 |
_ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
|
| 3786 |
_ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
|
| 3787 |
"advertised_products": "advertised products", "search_terms": "search terms"}
|
| 3788 |
-
#: `native` is the
|
| 3789 |
-
#: report
|
| 3790 |
_ADS_NATIVE = {
|
| 3791 |
-
"sp": {"campaigns": "
|
| 3792 |
-
"advertised_products": "
|
| 3793 |
-
"sb": {"campaigns": "
|
| 3794 |
-
"advertised_products": "
|
| 3795 |
-
"sd": {"campaigns": "
|
| 3796 |
-
"advertised_products": "
|
| 3797 |
}
|
| 3798 |
_ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
|
| 3799 |
"advertised_products": "asin", "search_terms": "search_term"}
|
|
@@ -3822,21 +3822,22 @@ AMAZON_ADS_TABLES = tuple(
|
|
| 3822 |
#: which verify_connectors holds equal to the `"amazon"` registration.
|
| 3823 |
AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
|
| 3824 |
|
| 3825 |
-
#: Days the first sync reaches back
|
| 3826 |
-
#:
|
| 3827 |
-
|
| 3828 |
-
AMAZON_ADS_BACKFILL_DAYS = {
|
| 3829 |
-
spec.key: (60 if spec.key.endswith("_search_terms")
|
| 3830 |
-
else {"sp": 95, "sb": 60, "sd": 65}[spec.key[len(_ADS):len(_ADS) + 2]])
|
| 3831 |
-
for spec in AMAZON_ADS_TABLES}
|
| 3832 |
|
| 3833 |
#: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
|
| 3834 |
AMAZON_ADS_THROUGHPUT = (
|
| 3835 |
-
"Amazon
|
| 3836 |
-
"
|
| 3837 |
-
"
|
| 3838 |
-
"
|
| 3839 |
-
"to raise the report limit
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3840 |
|
| 3841 |
#: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
|
| 3842 |
AMAZON_ADS_NOT_OFFERED = {
|
|
@@ -3859,9 +3860,10 @@ def amazon_ads_description(key):
|
|
| 3859 |
if unoffered:
|
| 3860 |
return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
|
| 3861 |
days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
|
| 3862 |
-
|
| 3863 |
-
|
| 3864 |
-
f"{
|
|
|
|
| 3865 |
|
| 3866 |
|
| 3867 |
def table_group(key):
|
|
|
|
| 3785 |
_ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
|
| 3786 |
_ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
|
| 3787 |
"advertised_products": "advertised products", "search_terms": "search terms"}
|
| 3788 |
+
#: `native` is the Reporting v1 report group that fills the table (`amazon_ads_schema.REPORT_GROUPS`).
|
| 3789 |
+
#: v1 has no report types. SD search terms has no group, because v1 does not offer it.
|
| 3790 |
_ADS_NATIVE = {
|
| 3791 |
+
"sp": {"campaigns": "campaigns", "targeting": "targeting",
|
| 3792 |
+
"advertised_products": "advertised_products", "search_terms": "search_terms"},
|
| 3793 |
+
"sb": {"campaigns": "campaigns", "targeting": "targeting",
|
| 3794 |
+
"advertised_products": "advertised_products_sb", "search_terms": "search_terms"},
|
| 3795 |
+
"sd": {"campaigns": "campaigns", "targeting": "targeting",
|
| 3796 |
+
"advertised_products": "advertised_products", "search_terms": "not_offered"},
|
| 3797 |
}
|
| 3798 |
_ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
|
| 3799 |
"advertised_products": "asin", "search_terms": "search_term"}
|
|
|
|
| 3822 |
#: which verify_connectors holds equal to the `"amazon"` registration.
|
| 3823 |
AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
|
| 3824 |
|
| 3825 |
+
#: Days the first sync reaches back: Reporting v1 keeps 24 months of daily data, and standing
|
| 3826 |
+
#: rule 1 takes all of it.
|
| 3827 |
+
AMAZON_ADS_BACKFILL_DAYS = {spec.key: 730 for spec in AMAZON_ADS_TABLES}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3828 |
|
| 3829 |
#: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
|
| 3830 |
AMAZON_ADS_THROUGHPUT = (
|
| 3831 |
+
"Amazon Ads runs one report at a time for each Loopable app login (its client ID) and about "
|
| 3832 |
+
"100 reports a day, shared by every advertiser on that login. Sponsored Products, Brands and "
|
| 3833 |
+
"Display need 5 reports for each refresh, and one report can take up to 3 hours, so one day of "
|
| 3834 |
+
"refresh may not finish in a day. Loopable runs the reports one at a time and keeps every day "
|
| 3835 |
+
"it already has. If the refresh falls behind, ask Amazon Ads support to raise the report limit "
|
| 3836 |
+
"for this client ID.")
|
| 3837 |
+
|
| 3838 |
+
#: On screen: the attribution window Amazon applies to Sponsored Products seller metrics.
|
| 3839 |
+
AMAZON_ADS_SP_ATTRIBUTION = ("Sponsored Products sales, orders and units count purchases within 7 "
|
| 3840 |
+
"days of a click, Amazon's default for sellers.")
|
| 3841 |
|
| 3842 |
#: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
|
| 3843 |
AMAZON_ADS_NOT_OFFERED = {
|
|
|
|
| 3860 |
if unoffered:
|
| 3861 |
return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
|
| 3862 |
days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
|
| 3863 |
+
attribution = AMAZON_ADS_SP_ATTRIBUTION + " " if spec.key.startswith(_ADS + "sp_") else ""
|
| 3864 |
+
return (f"{spec.label} by day, from Amazon Ads Reporting v1. {attribution}History is kept from "
|
| 3865 |
+
f"the day you connect, and the first sync reaches back {days} days, the longest Amazon "
|
| 3866 |
+
f"keeps. {AMAZON_ADS_THROUGHPUT}")
|
| 3867 |
|
| 3868 |
|
| 3869 |
def table_group(key):
|
platform/core/received_changes.py
CHANGED
|
@@ -406,6 +406,21 @@ def _move_baseline(rt, identity, expected, updated):
|
|
| 406 |
raise BaselineUnavailable("The observation reference was not saved.")
|
| 407 |
|
| 408 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
|
| 410 |
diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
|
| 411 |
"USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
|
|
@@ -417,28 +432,29 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
|
|
| 417 |
return
|
| 418 |
cursor, done = con.cursor(), 0
|
| 419 |
try:
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
|
|
|
| 442 |
finally:
|
| 443 |
cursor.close()
|
| 444 |
|
|
@@ -517,18 +533,31 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
|
|
| 517 |
deadline = (time.monotonic() + seconds, seconds)
|
| 518 |
con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
|
| 519 |
try:
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 532 |
finally:
|
| 533 |
con.close()
|
| 534 |
|
|
|
|
| 406 |
raise BaselineUnavailable("The observation reference was not saved.")
|
| 407 |
|
| 408 |
|
| 409 |
+
@contextmanager
|
| 410 |
+
def _query_watchdog(con, deadline):
|
| 411 |
+
"""Use the runtime's registered statement deadlines for publication queries."""
|
| 412 |
+
if deadline is None:
|
| 413 |
+
yield
|
| 414 |
+
return
|
| 415 |
+
from core import request_timing
|
| 416 |
+
parent = request_timing.CURRENT.get()
|
| 417 |
+
until = min(deadline[0], parent.deadline) if parent is not None and parent.deadline else deadline[0]
|
| 418 |
+
with request_timing.request_scope() as acc:
|
| 419 |
+
acc.deadline = until
|
| 420 |
+
with request_timing.duckdb_operation(con):
|
| 421 |
+
yield
|
| 422 |
+
|
| 423 |
+
|
| 424 |
def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
|
| 425 |
diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
|
| 426 |
"USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
|
|
|
|
| 432 |
return
|
| 433 |
cursor, done = con.cursor(), 0
|
| 434 |
try:
|
| 435 |
+
with _query_watchdog(cursor, deadline):
|
| 436 |
+
cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
|
| 437 |
+
+ " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
|
| 438 |
+
while batch := cursor.fetchmany(256):
|
| 439 |
+
for row_id, before_json, after_json in batch:
|
| 440 |
+
# Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
|
| 441 |
+
if deadline is not None and time.monotonic() >= deadline[0]:
|
| 442 |
+
print(f"[received-changes] {database}: journal deadline {deadline[1]:g} s passed "
|
| 443 |
+
f"after {done} of {count} rows; the rest are not journaled", flush=True)
|
| 444 |
+
return
|
| 445 |
+
before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
|
| 446 |
+
changes = {}
|
| 447 |
+
for key, (_sql, formatter) in spec["cols"].items():
|
| 448 |
+
left, right = before.get(key), after.get(key)
|
| 449 |
+
if formatter:
|
| 450 |
+
left = formatter(left) if left is not None else None
|
| 451 |
+
right = formatter(right) if right is not None else None
|
| 452 |
+
if left != right:
|
| 453 |
+
changes[key] = {"before": left, "after": right}
|
| 454 |
+
operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
|
| 455 |
+
publish(rt, source=source, database=database, row_id=row_id, changes=changes,
|
| 456 |
+
receipt_id=snapshot, operation=operation)
|
| 457 |
+
done += 1
|
| 458 |
finally:
|
| 459 |
cursor.close()
|
| 460 |
|
|
|
|
| 533 |
deadline = (time.monotonic() + seconds, seconds)
|
| 534 |
con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
|
| 535 |
try:
|
| 536 |
+
with _query_watchdog(con, deadline):
|
| 537 |
+
for name, files in paths.items():
|
| 538 |
+
con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
|
| 539 |
+
started = time.monotonic()
|
| 540 |
+
specs = resolver(rt, con)
|
| 541 |
+
resolution_seconds = time.monotonic() - started
|
| 542 |
+
snapshots_started = time.monotonic()
|
| 543 |
+
completed = deferred = skipped = 0
|
| 544 |
+
for database, spec in specs.items():
|
| 545 |
+
# A skipped database keeps its stored baseline for the next publication.
|
| 546 |
+
if time.monotonic() >= deadline[0]:
|
| 547 |
+
skipped += 1
|
| 548 |
+
print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
|
| 549 |
+
continue
|
| 550 |
+
try:
|
| 551 |
+
_snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
|
| 552 |
+
completed += 1
|
| 553 |
+
except Exception as exc:
|
| 554 |
+
deferred += 1
|
| 555 |
+
_LOG.warning("Received-change projection deferred for %s: %s",
|
| 556 |
+
database, type(exc).__name__)
|
| 557 |
+
print(f'[received-changes] publication connector={connector} tables={len(specs)} '
|
| 558 |
+
f'completed={completed} deferred={deferred} skipped={skipped} '
|
| 559 |
+
f'resolution={resolution_seconds:.3f}s snapshots={time.monotonic() - snapshots_started:.3f}s',
|
| 560 |
+
flush=True)
|
| 561 |
finally:
|
| 562 |
con.close()
|
| 563 |
|
platform/core/sheet_binding.py
CHANGED
|
@@ -58,7 +58,7 @@ def database_change_token(st, database):
|
|
| 58 |
key(connector_sync.JOBS_KEY), database,
|
| 59 |
database, key(connector_sync.STATE_KEY))).fetchone()
|
| 60 |
token, definition = row[0], row[1] or {}
|
| 61 |
-
if store_pg._ut_rows(key(user_tables.STORE_KEY), handle.tenant_slug):
|
| 62 |
from core import user_table_rows
|
| 63 |
from psycopg import sql
|
| 64 |
definition = user_table_rows.definition_view(user_table_rows.read_definition(
|
|
|
|
| 58 |
key(connector_sync.JOBS_KEY), database,
|
| 59 |
database, key(connector_sync.STATE_KEY))).fetchone()
|
| 60 |
token, definition = row[0], row[1] or {}
|
| 61 |
+
if database.startswith('ut_') and store_pg._ut_rows(key(user_tables.STORE_KEY), handle.tenant_slug):
|
| 62 |
from core import user_table_rows
|
| 63 |
from psycopg import sql
|
| 64 |
definition = user_table_rows.definition_view(user_table_rows.read_definition(
|
platform/harness/amazon_ads_client.py
CHANGED
|
@@ -4,7 +4,7 @@
|
|
| 4 |
read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
|
| 5 |
refusal lives here, in the request builder, before any credential read, token exchange or socket:
|
| 6 |
|
| 7 |
-
reports create,
|
| 8 |
streams create and list subscriptions for the dedicated V1 destination
|
| 9 |
named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
|
| 10 |
`bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
|
|
@@ -17,8 +17,11 @@ harness module never imports `modules` at load). A missing approval source refus
|
|
| 17 |
made until Royal's Direct Advertiser credential arrives (R7, R17).
|
| 18 |
|
| 19 |
NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
|
| 20 |
-
stored with the credential
|
|
|
|
|
|
|
| 21 |
"""
|
|
|
|
| 22 |
import importlib
|
| 23 |
import json
|
| 24 |
import logging
|
|
@@ -31,6 +34,7 @@ import urllib.parse
|
|
| 31 |
import urllib.request
|
| 32 |
from contextlib import contextmanager
|
| 33 |
from datetime import date, datetime, timedelta, timezone
|
|
|
|
| 34 |
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
| 35 |
|
| 36 |
KEY = "amazon_ads"
|
|
@@ -46,22 +50,28 @@ DEFAULT_REGION = "na"
|
|
| 46 |
LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
|
| 47 |
CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
|
| 48 |
|
| 49 |
-
#: Reporting v1 limits (
|
| 50 |
-
#: up to 3 hours per report, at most
|
| 51 |
REPORT_CONCURRENCY = 1
|
| 52 |
REPORTS_PER_DAY = 100
|
| 53 |
REPORT_MAX_WAIT_SECONDS = 3 * 3600
|
| 54 |
-
MAX_DAYS_PER_REQUEST =
|
| 55 |
-
REPORT_POLL_SECONDS =
|
|
|
|
|
|
|
| 56 |
_TIMEOUT = 60
|
| 57 |
|
| 58 |
-
#: The only non-write calls: Reporting v1
|
| 59 |
-
#:
|
| 60 |
REPORT_OPERATIONS = {
|
| 61 |
-
"createReport": ("POST", "
|
| 62 |
-
"
|
| 63 |
-
"
|
| 64 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
# Subscription management never admits an advertising write.
|
| 67 |
STREAM_OPERATIONS = {
|
|
@@ -208,7 +218,15 @@ class AdsWriteRefused(AdsError):
|
|
| 208 |
f"the named writes: bid, budget, negative keyword and scheduled change.")
|
| 209 |
|
| 210 |
|
| 211 |
-
class
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
"""Amazon does not offer this report to this advertiser."""
|
| 213 |
|
| 214 |
|
|
@@ -225,25 +243,10 @@ def fixture_mode():
|
|
| 225 |
return os.environ.get(FIXTURE_ENV) == "1"
|
| 226 |
|
| 227 |
|
| 228 |
-
def _template(path):
|
| 229 |
-
return tuple(path.strip("/").split("/"))
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
def _matches(template, path):
|
| 233 |
-
parts = str(path or "").strip("/").split("/")
|
| 234 |
-
if len(parts) != len(template):
|
| 235 |
-
return False
|
| 236 |
-
return all((bool(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part)))
|
| 237 |
-
if seg.startswith("{") and seg.endswith("}") else part == seg
|
| 238 |
-
for part, seg in zip(parts, template))
|
| 239 |
-
|
| 240 |
-
|
| 241 |
def report_operation(method, path):
|
| 242 |
-
"""The Reporting v1 operation name for one call, or None."""
|
| 243 |
-
for name, (verb,
|
| 244 |
-
|
| 245 |
-
return name
|
| 246 |
-
return None
|
| 247 |
|
| 248 |
|
| 249 |
class _Prepared:
|
|
@@ -503,12 +506,17 @@ def _send(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 503 |
if not _admitted(prepared):
|
| 504 |
raise AdsWriteRefused("the request did not come from the request builder")
|
| 505 |
if fixture_mode():
|
| 506 |
-
|
|
|
|
| 507 |
return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 508 |
|
| 509 |
|
| 510 |
# --------------------------------------------------------------------------- live transport
|
| 511 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 512 |
def credential(tenant_key, rt=None, connection_id=None):
|
| 513 |
"""The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
|
| 514 |
try:
|
|
@@ -538,11 +546,50 @@ def _token(fields):
|
|
| 538 |
raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
|
| 539 |
|
| 540 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 541 |
def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
|
| 542 |
fields = credential(tenant_key, rt, connection_id=connection_id)
|
| 543 |
if fields is None:
|
| 544 |
-
raise AdsError(
|
| 545 |
-
"Connect Amazon Ads under Amazon Ads Setup.")
|
| 546 |
if prepared.operation in STREAM_OPERATIONS and (
|
| 547 |
str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
|
| 548 |
or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
|
|
@@ -551,13 +598,7 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 551 |
url = f"{host}/{prepared.path}"
|
| 552 |
if prepared.query:
|
| 553 |
url += "?" + urllib.parse.urlencode(prepared.query)
|
| 554 |
-
headers =
|
| 555 |
-
"Amazon-Advertising-API-ClientId": fields["client_id"],
|
| 556 |
-
"Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
|
| 557 |
-
"Content-Type": "application/json", "Accept": "application/json"}
|
| 558 |
-
if prepared.operation in STREAM_OPERATIONS:
|
| 559 |
-
headers["Content-Type"] = STREAM_CONTENT_TYPE
|
| 560 |
-
headers["Accept"] = STREAM_CONTENT_TYPE
|
| 561 |
data = json.dumps(prepared.body).encode() if prepared.body is not None else None
|
| 562 |
req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
|
| 563 |
try:
|
|
@@ -566,12 +607,10 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 566 |
except urllib.error.HTTPError as exc:
|
| 567 |
try:
|
| 568 |
detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
raise AdsReportUnavailable(f"Amazon does not offer this report: {message}") from None
|
| 574 |
-
raise AdsError(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}: {message}") from None
|
| 575 |
except Exception as exc: # noqa: BLE001
|
| 576 |
raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
|
| 577 |
|
|
@@ -580,89 +619,276 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
|
|
| 580 |
|
| 581 |
_REPORT_LOCKS = {}
|
| 582 |
_REPORT_LOCKS_GUARD = threading.Lock()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 583 |
|
| 584 |
|
| 585 |
-
def _report_lock(
|
|
|
|
| 586 |
with _REPORT_LOCKS_GUARD:
|
| 587 |
-
return _REPORT_LOCKS.setdefault(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 589 |
|
| 590 |
-
def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep,
|
| 591 |
-
now=time.monotonic, max_wait=None, report_id=None):
|
| 592 |
-
"""Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).
|
| 593 |
|
| 594 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 595 |
"""
|
| 596 |
from harness import amazon_ads_schema as schema
|
| 597 |
-
|
| 598 |
if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
|
| 599 |
-
raise ValueError("a report window is at most
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
|
| 604 |
-
|
| 605 |
-
with _report_lock(tenant_key, profile_id):
|
| 606 |
if report_id is None:
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}")
|
| 626 |
-
elapsed = now() - started
|
| 627 |
-
if elapsed > REPORT_MAX_WAIT_SECONDS:
|
| 628 |
-
raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
|
| 629 |
-
if limit is not None and elapsed >= limit:
|
| 630 |
-
raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
|
| 631 |
-
pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
|
| 632 |
-
sleep(pause if limit is None else min(pause, limit - elapsed))
|
| 633 |
-
rows, token = [], None
|
| 634 |
-
rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id)
|
| 635 |
-
while True:
|
| 636 |
-
page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None,
|
| 637 |
-
rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 638 |
-
rows.extend(page.get("rows") or [])
|
| 639 |
-
token = page.get("nextToken")
|
| 640 |
-
if not token:
|
| 641 |
-
return report_id, rows
|
| 642 |
|
| 643 |
|
| 644 |
# --------------------------------------------------------------------------- recording fixture
|
| 645 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 646 |
class _Fixture:
|
| 647 |
-
"""The recording fixture `AMAZON_ADS_FIXTURE=1` routes to
|
| 648 |
-
with deterministic rows, two report polls, and paged rows."""
|
| 649 |
|
| 650 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 651 |
|
| 652 |
def __init__(self):
|
| 653 |
self.reset()
|
| 654 |
|
| 655 |
def reset(self):
|
| 656 |
-
self.calls, self.reports = [], {}
|
| 657 |
-
self.subscriptions = {}
|
| 658 |
self.in_flight = self.max_in_flight = 0
|
| 659 |
-
self.drop_days, self.unsupported = set(), set()
|
|
|
|
| 660 |
self.responder = None
|
| 661 |
|
| 662 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
| 663 |
self.calls.append({"method": prepared.method, "path": prepared.path,
|
| 664 |
"operation": prepared.operation, "body": prepared.body,
|
| 665 |
-
"query": dict(prepared.query), "tenant": tenant_key,
|
| 666 |
"named": dict(prepared.named) if prepared.named else None})
|
| 667 |
if self.responder is not None:
|
| 668 |
return self.responder(prepared)
|
|
@@ -678,88 +904,174 @@ class _Fixture:
|
|
| 678 |
if prepared.named is not None:
|
| 679 |
items = next(iter(prepared.body.values()))
|
| 680 |
return {"success": [{"index": i} for i in range(len(items))], "error": []}
|
|
|
|
|
|
|
| 681 |
if prepared.operation == "createReport":
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 688 |
report = self.reports[report_id]
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 703 |
self.in_flight -= 1
|
| 704 |
-
|
| 705 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 706 |
|
| 707 |
|
| 708 |
def fixture_rows(body, *, drop_days=()):
|
| 709 |
-
"""Deterministic
|
| 710 |
-
|
| 711 |
-
start, end =
|
| 712 |
-
|
| 713 |
-
out
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
"promotedAsin": f"B00FX0{c}{g}0", "searchTerm": f"search {c}{g}",
|
| 730 |
-
"impressions": 100 * n, "clicks": 3 * n, "cost": round(1.25 * n, 2),
|
| 731 |
-
"sales": round(9.5 * n, 2), "sales14d": round(9.5 * n, 2),
|
| 732 |
-
"purchases": n, "purchases14d": n, "unitsSold": n, "unitsSoldClicks14d": n,
|
| 733 |
-
}
|
| 734 |
-
out.append({k: row.get(k) for k in columns})
|
| 735 |
-
day += timedelta(days=1)
|
| 736 |
return out
|
| 737 |
|
| 738 |
|
| 739 |
FIXTURE = _Fixture()
|
| 740 |
|
| 741 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 742 |
@contextmanager
|
| 743 |
def recording_fixture():
|
| 744 |
-
"""Route every call to a fresh recording fixture for the block. -> the fixture."""
|
| 745 |
previous = os.environ.get(FIXTURE_ENV)
|
| 746 |
os.environ[FIXTURE_ENV] = "1"
|
| 747 |
FIXTURE.reset()
|
|
|
|
| 748 |
try:
|
| 749 |
yield FIXTURE
|
| 750 |
finally:
|
| 751 |
FIXTURE.reset()
|
|
|
|
| 752 |
if previous is None:
|
| 753 |
os.environ.pop(FIXTURE_ENV, None)
|
| 754 |
else:
|
| 755 |
os.environ[FIXTURE_ENV] = previous
|
| 756 |
|
| 757 |
|
| 758 |
-
__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "
|
| 759 |
-
"STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
|
| 760 |
"FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
|
| 761 |
"SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
|
| 762 |
-
"MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsReportUnavailable",
|
| 763 |
-
"AdsReportPending", "fixture_mode", "build_request", "report_operation",
|
| 764 |
-
"send_named_write", "request", "credential", "run_report", "FIXTURE",
|
| 765 |
-
"recording_fixture", "fixture_rows"]
|
|
|
|
| 4 |
read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
|
| 5 |
refusal lives here, in the request builder, before any credential read, token exchange or socket:
|
| 6 |
|
| 7 |
+
reports Reporting v1: create, retrieve, the advertiser account query, and the report files
|
| 8 |
streams create and list subscriptions for the dedicated V1 destination
|
| 9 |
named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
|
| 10 |
`bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
|
|
|
|
| 17 |
made until Royal's Direct Advertiser credential arrives (R7, R17).
|
| 18 |
|
| 19 |
NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
|
| 20 |
+
stored with the credential. v1 names the account by `advertiserAccountId`, which the client looks
|
| 21 |
+
up once per connection from `query/advertiserAccounts`. Every path, header and field comes from
|
| 22 |
+
Amazon's saved contracts in `AutoWiki/research/amazon-ads-v1-schema/`.
|
| 23 |
"""
|
| 24 |
+
import gzip
|
| 25 |
import importlib
|
| 26 |
import json
|
| 27 |
import logging
|
|
|
|
| 34 |
import urllib.request
|
| 35 |
from contextlib import contextmanager
|
| 36 |
from datetime import date, datetime, timedelta, timezone
|
| 37 |
+
from typing import NamedTuple
|
| 38 |
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
| 39 |
|
| 40 |
KEY = "amazon_ads"
|
|
|
|
| 50 |
LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
|
| 51 |
CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
|
| 52 |
|
| 53 |
+
#: Reporting v1 limits (limits.md, dates.md): one report in flight per client ID, about 100 a day,
|
| 54 |
+
#: up to 3 hours per report, at most 120 days per request.
|
| 55 |
REPORT_CONCURRENCY = 1
|
| 56 |
REPORTS_PER_DAY = 100
|
| 57 |
REPORT_MAX_WAIT_SECONDS = 3 * 3600
|
| 58 |
+
MAX_DAYS_PER_REQUEST = 120
|
| 59 |
+
REPORT_POLL_SECONDS = 60.0
|
| 60 |
+
#: Amazon's error text reaches the sync report verbatim, up to this many characters.
|
| 61 |
+
ERROR_TEXT_LIMIT = 600
|
| 62 |
_TIMEOUT = 60
|
| 63 |
|
| 64 |
+
#: The only non-write calls: the Reporting v1 operations, each a POST to one literal path from the
|
| 65 |
+
#: Reports and AdvertiserAccounts contracts. Every other `adsApi` path refuses, a GET included.
|
| 66 |
REPORT_OPERATIONS = {
|
| 67 |
+
"createReport": ("POST", "adsApi/v1/create/reports"),
|
| 68 |
+
"retrieveReport": ("POST", "adsApi/v1/retrieve/reports"),
|
| 69 |
+
"queryAdvertiserAccounts": ("POST", "adsApi/v1/query/advertiserAccounts"),
|
| 70 |
}
|
| 71 |
+
#: The contract names `Amazon-Ads-ClientId`. The limits page names the older header. v1 sends both.
|
| 72 |
+
V1_CLIENT_HEADERS = ("Amazon-Ads-ClientId", "Amazon-Advertising-API-ClientId")
|
| 73 |
+
#: A report file is a presigned S3 URL: https, on this host suffix, GET with no Ads header.
|
| 74 |
+
REPORT_FILE_HOST_SUFFIX = ".amazonaws.com"
|
| 75 |
|
| 76 |
# Subscription management never admits an advertising write.
|
| 77 |
STREAM_OPERATIONS = {
|
|
|
|
| 218 |
f"the named writes: bid, budget, negative keyword and scheduled change.")
|
| 219 |
|
| 220 |
|
| 221 |
+
class AdsRejected(AdsError):
|
| 222 |
+
"""Amazon answered an error. `errors` keeps each entry's code, message and fieldLocation."""
|
| 223 |
+
|
| 224 |
+
def __init__(self, message, errors=()):
|
| 225 |
+
super().__init__(message)
|
| 226 |
+
self.errors = list(errors)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
class AdsReportUnavailable(AdsRejected):
|
| 230 |
"""Amazon does not offer this report to this advertiser."""
|
| 231 |
|
| 232 |
|
|
|
|
| 243 |
return os.environ.get(FIXTURE_ENV) == "1"
|
| 244 |
|
| 245 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
def report_operation(method, path):
|
| 247 |
+
"""The Reporting v1 operation name for one call, or None. A path matches only its literal."""
|
| 248 |
+
return next((name for name, (verb, endpoint) in REPORT_OPERATIONS.items()
|
| 249 |
+
if str(method).upper() == verb and str(path or "").strip("/") == endpoint), None)
|
|
|
|
|
|
|
| 250 |
|
| 251 |
|
| 252 |
class _Prepared:
|
|
|
|
| 506 |
if not _admitted(prepared):
|
| 507 |
raise AdsWriteRefused("the request did not come from the request builder")
|
| 508 |
if fixture_mode():
|
| 509 |
+
headers = _headers(prepared, FIXTURE.credential(tenant_key), FIXTURE_TOKEN, profile_id)
|
| 510 |
+
return FIXTURE.answer(prepared, tenant_key, profile_id, headers)
|
| 511 |
return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 512 |
|
| 513 |
|
| 514 |
# --------------------------------------------------------------------------- live transport
|
| 515 |
|
| 516 |
+
_NO_CREDENTIAL = ("No Amazon Ads credential is stored for this tenant. "
|
| 517 |
+
"Connect Amazon Ads under Amazon Ads Setup.")
|
| 518 |
+
|
| 519 |
+
|
| 520 |
def credential(tenant_key, rt=None, connection_id=None):
|
| 521 |
"""The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
|
| 522 |
try:
|
|
|
|
| 546 |
raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
|
| 547 |
|
| 548 |
|
| 549 |
+
def _headers(prepared, fields, token, profile_id=None):
|
| 550 |
+
"""The headers one admitted call sends. A v1 call names its account in the body, not a scope."""
|
| 551 |
+
if prepared.operation in REPORT_OPERATIONS:
|
| 552 |
+
return {"Authorization": f"Bearer {token}", **dict.fromkeys(V1_CLIENT_HEADERS, fields["client_id"]),
|
| 553 |
+
"Content-Type": "application/json", "Accept": "application/json"}
|
| 554 |
+
headers = {"Authorization": f"Bearer {token}",
|
| 555 |
+
"Amazon-Advertising-API-ClientId": fields["client_id"],
|
| 556 |
+
"Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
|
| 557 |
+
"Content-Type": "application/json", "Accept": "application/json"}
|
| 558 |
+
if prepared.operation in STREAM_OPERATIONS:
|
| 559 |
+
headers["Content-Type"] = STREAM_CONTENT_TYPE
|
| 560 |
+
headers["Accept"] = STREAM_CONTENT_TYPE
|
| 561 |
+
return headers
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
def _errors(detail):
|
| 565 |
+
"""Amazon's error entries from an error body, each with its code, message and fieldLocation."""
|
| 566 |
+
if not isinstance(detail, dict):
|
| 567 |
+
return []
|
| 568 |
+
entries = detail["errors"] if isinstance(detail.get("errors"), list) else [detail]
|
| 569 |
+
return [{"code": str(e.get("code") or ""), "message": str(e.get("details") or e.get("message") or ""),
|
| 570 |
+
"fieldLocation": str(e.get("fieldLocation") or "")}
|
| 571 |
+
for e in entries if isinstance(e, dict) and (e.get("code") or e.get("details") or e.get("message"))]
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
def _error_text(errors):
|
| 575 |
+
"""Amazon's errors as one line a person reads, verbatim, cut to `ERROR_TEXT_LIMIT`."""
|
| 576 |
+
return " ".join(" ".join(filter(None, (e["code"] + ":" if e["code"] else "", e["message"],
|
| 577 |
+
f"(field {e['fieldLocation']})" if e["fieldLocation"] else "")))
|
| 578 |
+
for e in errors)[:ERROR_TEXT_LIMIT]
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _rejection(prefix, errors, *, may_be_unoffered=True):
|
| 582 |
+
"""The exception for Amazon's errors: not offered when Amazon says so, else rejected."""
|
| 583 |
+
text = _error_text(errors)
|
| 584 |
+
if may_be_unoffered and re.search(r"not (?:supported|offered|available)", text, re.I):
|
| 585 |
+
return AdsReportUnavailable(f"Amazon does not offer this report: {text}", errors)
|
| 586 |
+
return AdsRejected(f"{prefix}: {text}" if text else prefix, errors)
|
| 587 |
+
|
| 588 |
+
|
| 589 |
def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
|
| 590 |
fields = credential(tenant_key, rt, connection_id=connection_id)
|
| 591 |
if fields is None:
|
| 592 |
+
raise AdsError(_NO_CREDENTIAL)
|
|
|
|
| 593 |
if prepared.operation in STREAM_OPERATIONS and (
|
| 594 |
str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
|
| 595 |
or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
|
|
|
|
| 598 |
url = f"{host}/{prepared.path}"
|
| 599 |
if prepared.query:
|
| 600 |
url += "?" + urllib.parse.urlencode(prepared.query)
|
| 601 |
+
headers = _headers(prepared, fields, _token(fields), profile_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 602 |
data = json.dumps(prepared.body).encode() if prepared.body is not None else None
|
| 603 |
req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
|
| 604 |
try:
|
|
|
|
| 607 |
except urllib.error.HTTPError as exc:
|
| 608 |
try:
|
| 609 |
detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
|
| 610 |
+
except Exception: # noqa: BLE001 an unreadable body still raises its status
|
| 611 |
+
detail = {}
|
| 612 |
+
raise _rejection(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}", _errors(detail),
|
| 613 |
+
may_be_unoffered=exc.code == 400) from None
|
|
|
|
|
|
|
| 614 |
except Exception as exc: # noqa: BLE001
|
| 615 |
raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
|
| 616 |
|
|
|
|
| 619 |
|
| 620 |
_REPORT_LOCKS = {}
|
| 621 |
_REPORT_LOCKS_GUARD = threading.Lock()
|
| 622 |
+
#: Process caches: the advertiser account per connection, the optional fields Amazon refused per
|
| 623 |
+
#: (client id, group) with Amazon's text, and the report file URLs a retrieve returned.
|
| 624 |
+
_ACCOUNTS = {}
|
| 625 |
+
_REFUSED = {}
|
| 626 |
+
_ISSUED_FILES = set()
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
class GroupReport(NamedTuple):
|
| 630 |
+
"""One finished group report: its id, flat rows per member table, and notes a person reads."""
|
| 631 |
+
report_id: str
|
| 632 |
+
rows: dict
|
| 633 |
+
notes: tuple
|
| 634 |
|
| 635 |
|
| 636 |
+
def _report_lock(client_id):
|
| 637 |
+
"""One lock per client ID: Amazon runs one report at a time for each app login."""
|
| 638 |
with _REPORT_LOCKS_GUARD:
|
| 639 |
+
return _REPORT_LOCKS.setdefault(str(client_id), threading.Lock())
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def _report_credential(tenant_key, rt, connection_id):
|
| 643 |
+
"""The credential a report runs under: its client id keys the lock, its profile names the account."""
|
| 644 |
+
if fixture_mode():
|
| 645 |
+
return FIXTURE.credential(tenant_key)
|
| 646 |
+
fields = credential(tenant_key, rt, connection_id=connection_id)
|
| 647 |
+
if fields is None:
|
| 648 |
+
raise AdsError(_NO_CREDENTIAL)
|
| 649 |
+
return fields
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
def _lists_profile(account, profile_id):
|
| 653 |
+
return (isinstance(account, dict) and bool(account.get("advertiserAccountId"))
|
| 654 |
+
and any(isinstance(alt, dict) and str(alt.get("profileId")) == str(profile_id)
|
| 655 |
+
for alt in account.get("alternateIds") or ()))
|
| 656 |
+
|
| 657 |
|
| 658 |
+
def _advertiser_account(tenant_key, *, rt, profile_id, connection_id):
|
| 659 |
+
"""The v1 `advertiserAccountId` whose alternate ids carry the stored profile id, cached per connection."""
|
| 660 |
+
key, token = (tenant_key, connection_id or "", str(profile_id)), None
|
| 661 |
+
while key not in _ACCOUNTS:
|
| 662 |
+
page = request(tenant_key, "POST", REPORT_OPERATIONS["queryAdvertiserAccounts"][1],
|
| 663 |
+
body={"maxResults": 100, **({"nextToken": token} if token else {})},
|
| 664 |
+
rt=rt, profile_id=profile_id, connection_id=connection_id)
|
| 665 |
+
match = next((a for a in page.get("advertiserAccounts") or () if _lists_profile(a, profile_id)), None)
|
| 666 |
+
token = page.get("nextToken")
|
| 667 |
+
if match is not None:
|
| 668 |
+
_ACCOUNTS[key] = str(match["advertiserAccountId"])
|
| 669 |
+
elif not token:
|
| 670 |
+
raise AdsError(f"No Amazon Ads advertiser account lists profile {profile_id}. "
|
| 671 |
+
f"Check the profile ID on this Amazon Ads connection.")
|
| 672 |
+
return _ACCOUNTS[key]
|
| 673 |
|
|
|
|
|
|
|
|
|
|
| 674 |
|
| 675 |
+
def _create_body(group, start, end, account, fields):
|
| 676 |
+
"""A CreateReportRequest for one group and window, filtered to the group's ad products."""
|
| 677 |
+
from harness import amazon_ads_schema as schema
|
| 678 |
+
return {"accessRequestedAccounts": [{"advertiserAccountId": account}],
|
| 679 |
+
"reports": [{"format": "GZIP_JSON",
|
| 680 |
+
"periods": [{"datePeriod": {"startDate": start.isoformat(), "endDate": end.isoformat()}}],
|
| 681 |
+
"query": {"fields": list(fields),
|
| 682 |
+
"filter": {"on": {"field": schema.AD_PRODUCT_FIELD, "comparisonOperator": "IN",
|
| 683 |
+
"not": False, "values": list(group.tables)}}}}]}
|
| 684 |
+
|
| 685 |
+
|
| 686 |
+
def _first_report(reply, operation):
|
| 687 |
+
"""The one report a 207 multi-status answer carries, or its errors raised verbatim."""
|
| 688 |
+
errors = [e for entry in reply.get("error") or () if isinstance(entry, dict)
|
| 689 |
+
for e in _errors({"errors": entry.get("errors") or []})]
|
| 690 |
+
if errors or reply.get("error"):
|
| 691 |
+
raise _rejection(f"Amazon Ads refused {operation}", errors)
|
| 692 |
+
for entry in reply.get("success") or ():
|
| 693 |
+
report = entry.get("report") if isinstance(entry, dict) else None
|
| 694 |
+
if isinstance(report, dict) and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", str(report.get("reportId") or "")):
|
| 695 |
+
return report
|
| 696 |
+
raise AdsError(f"Amazon Ads returned no report on {operation}")
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
def _named_fields(exc, fields):
|
| 700 |
+
"""The requested fields an error names, by id in its text or by index in its field location."""
|
| 701 |
+
text = " ".join([str(exc)] + [e["fieldLocation"] for e in exc.errors])
|
| 702 |
+
named = {f for f in fields if re.search(rf"(?<![\w.]){re.escape(f)}(?!\w)", text)}
|
| 703 |
+
named.update(fields[int(i)] for i in re.findall(r"fields\[(\d+)\]", text) if int(i) < len(fields))
|
| 704 |
+
return named
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
def _create(tenant_key, group, start, end, account, refused, opts):
|
| 708 |
+
"""Create one group report. An error naming only optional fields drops them and creates again."""
|
| 709 |
+
while True:
|
| 710 |
+
fields = group.fields(drop=refused)
|
| 711 |
+
try:
|
| 712 |
+
reply = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1],
|
| 713 |
+
body=_create_body(group, start, end, account, fields), **opts)
|
| 714 |
+
return _first_report(reply, "createReport")
|
| 715 |
+
except AdsRejected as exc:
|
| 716 |
+
named = _named_fields(exc, fields)
|
| 717 |
+
if not named or named & set(group.core):
|
| 718 |
+
raise
|
| 719 |
+
refused.update(dict.fromkeys(sorted(named), _error_text(exc.errors) or str(exc)))
|
| 720 |
+
|
| 721 |
+
|
| 722 |
+
def _await(tenant_key, report_id, opts, *, sleep, now, limit):
|
| 723 |
+
"""Poll retrieve about once a minute until the report completes. -> its report file URLs."""
|
| 724 |
+
started = now()
|
| 725 |
+
while True:
|
| 726 |
+
report = _first_report(request(tenant_key, "POST", REPORT_OPERATIONS["retrieveReport"][1],
|
| 727 |
+
body={"reportIds": [report_id]}, **opts), "retrieveReport")
|
| 728 |
+
state = str(report.get("status") or "").upper()
|
| 729 |
+
if state == "COMPLETED":
|
| 730 |
+
urls = [str(part.get("url") or "") for part in report.get("completedReportParts") or ()
|
| 731 |
+
if isinstance(part, dict)]
|
| 732 |
+
_ISSUED_FILES.update(urls)
|
| 733 |
+
return urls
|
| 734 |
+
if state == "FAILED":
|
| 735 |
+
reason = " ".join(filter(None, (str(report.get("failureCode") or ""),
|
| 736 |
+
str(report.get("failureReason") or ""))))[:ERROR_TEXT_LIMIT] or state
|
| 737 |
+
if re.search(r"not (?:supported|offered|available)", reason, re.I):
|
| 738 |
+
raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
|
| 739 |
+
raise AdsError(f"Amazon Ads report {report_id} ended FAILED: {reason}")
|
| 740 |
+
elapsed = now() - started
|
| 741 |
+
if elapsed > REPORT_MAX_WAIT_SECONDS:
|
| 742 |
+
raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
|
| 743 |
+
if limit is not None and elapsed >= limit:
|
| 744 |
+
raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
|
| 745 |
+
pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
|
| 746 |
+
sleep(pause if limit is None else min(pause, limit - elapsed))
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
def _flat(row, prefix=""):
|
| 750 |
+
"""A nested v1 row -> flat dotted field ids. A flat row stays as it is."""
|
| 751 |
+
out = {}
|
| 752 |
+
for key, value in row.items():
|
| 753 |
+
if isinstance(value, dict):
|
| 754 |
+
out.update(_flat(value, f"{prefix}{key}."))
|
| 755 |
+
else:
|
| 756 |
+
out[f"{prefix}{key}"] = value
|
| 757 |
+
return out
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
def _part_rows(raw):
|
| 761 |
+
"""One report file -> flat rows. GZIP or plain, a JSON array or JSON lines."""
|
| 762 |
+
try:
|
| 763 |
+
text = (gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw).decode("utf-8", "replace").strip()
|
| 764 |
+
if not text:
|
| 765 |
+
return []
|
| 766 |
+
try:
|
| 767 |
+
data = json.loads(text)
|
| 768 |
+
except ValueError:
|
| 769 |
+
data = [json.loads(line) for line in text.splitlines() if line.strip()]
|
| 770 |
+
except (OSError, EOFError, ValueError) as exc:
|
| 771 |
+
raise AdsError(f"Amazon Ads sent a report file Loopable cannot read ({type(exc).__name__})") from None
|
| 772 |
+
return [_flat(row) for row in (data if isinstance(data, list) else [data]) if isinstance(row, dict)]
|
| 773 |
+
|
| 774 |
+
|
| 775 |
+
def _read_file(url):
|
| 776 |
+
"""GET one report file with no Ads header. Only an https S3 URL a retrieve returned is admitted."""
|
| 777 |
+
parts = urllib.parse.urlsplit(str(url or ""))
|
| 778 |
+
host = parts.hostname or ""
|
| 779 |
+
if parts.scheme != "https" or not host.endswith(REPORT_FILE_HOST_SUFFIX) or url not in _ISSUED_FILES:
|
| 780 |
+
raise AdsWriteRefused(f"{parts.scheme or 'no scheme'} {_safe_path(host)} is not a report file Amazon returned")
|
| 781 |
+
_ISSUED_FILES.discard(url)
|
| 782 |
+
headers = {}
|
| 783 |
+
if fixture_mode():
|
| 784 |
+
return _part_rows(FIXTURE.download(url, headers))
|
| 785 |
+
try:
|
| 786 |
+
with urllib.request.urlopen(urllib.request.Request(url, method="GET", headers=headers),
|
| 787 |
+
timeout=_TIMEOUT) as r:
|
| 788 |
+
raw = r.read()
|
| 789 |
+
except Exception as exc: # noqa: BLE001 the URL carries a signature: never echo it
|
| 790 |
+
raise AdsError(f"Amazon Ads report file could not be read ({type(exc).__name__})") from None
|
| 791 |
+
return _part_rows(raw)
|
| 792 |
+
|
| 793 |
+
|
| 794 |
+
def _split(group, rows):
|
| 795 |
+
"""Flat rows -> {member table: rows} by `adProduct.value`, and a note for rows no member takes."""
|
| 796 |
+
from harness import amazon_ads_schema as schema
|
| 797 |
+
out, tables, stray = {table: [] for table in group.members}, group.tables, 0
|
| 798 |
+
for row in rows:
|
| 799 |
+
table = tables.get(str(row.get(schema.AD_PRODUCT_FIELD) or "").upper())
|
| 800 |
+
if table is None or not row.get(schema.vendor_name("date")):
|
| 801 |
+
stray += 1
|
| 802 |
+
else:
|
| 803 |
+
out[table].append(row)
|
| 804 |
+
note = (f"Amazon Ads sent {stray} rows with no date or with another ad product. Loopable left them out.",)
|
| 805 |
+
return out, note if stray else ()
|
| 806 |
+
|
| 807 |
+
|
| 808 |
+
def run_group_report(tenant_key, group, start, end, *, rt=None, profile_id=None, connection_id=None,
|
| 809 |
+
sleep=time.sleep, now=time.monotonic, max_wait=None, report_id=None):
|
| 810 |
+
"""Create one Reporting v1 group report, wait for it, and read every file. -> GroupReport.
|
| 811 |
+
|
| 812 |
+
`max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without a create.
|
| 813 |
"""
|
| 814 |
from harness import amazon_ads_schema as schema
|
| 815 |
+
group = schema.GROUPS[group] if isinstance(group, str) else group
|
| 816 |
if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
|
| 817 |
+
raise ValueError(f"a report window is at most {MAX_DAYS_PER_REQUEST} days")
|
| 818 |
+
fields = _report_credential(tenant_key, rt, connection_id)
|
| 819 |
+
opts = {"rt": rt, "profile_id": str(profile_id or fields["profile_id"]), "connection_id": connection_id}
|
| 820 |
+
refused = _REFUSED.setdefault((str(fields["client_id"]), group.name), {})
|
| 821 |
limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
|
| 822 |
+
with _report_lock(fields["client_id"]):
|
|
|
|
| 823 |
if report_id is None:
|
| 824 |
+
account = _advertiser_account(tenant_key, **opts)
|
| 825 |
+
report_id = _create(tenant_key, group, start, end, account, refused, opts)["reportId"]
|
| 826 |
+
urls = _await(tenant_key, str(report_id), opts, sleep=sleep, now=now, limit=limit)
|
| 827 |
+
rows = [row for url in urls for row in _read_file(url)]
|
| 828 |
+
split, stray = _split(group, rows)
|
| 829 |
+
notes = tuple(f"Amazon Ads refused the field {field} on this report, so Loopable reads it without that "
|
| 830 |
+
f"field. Amazon said: {text}" for field, text in refused.items())
|
| 831 |
+
return GroupReport(str(report_id), split, notes + stray)
|
| 832 |
+
|
| 833 |
+
|
| 834 |
+
def run_report(tenant_key, table, start, end, **options):
|
| 835 |
+
"""One table's rows from its group report. -> (report_id, rows). The store reads `run_group_report`."""
|
| 836 |
+
from harness import amazon_ads_schema as schema
|
| 837 |
+
group = schema.group_of(table)
|
| 838 |
+
if group is None:
|
| 839 |
+
raise AdsReportUnavailable(f"Amazon Reporting v1 offers no report for {schema.short_name(table)}")
|
| 840 |
+
result = run_group_report(tenant_key, group, start, end, **options)
|
| 841 |
+
return result.report_id, result.rows[schema.short_name(table)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 842 |
|
| 843 |
|
| 844 |
# --------------------------------------------------------------------------- recording fixture
|
| 845 |
|
| 846 |
+
FIXTURE_TOKEN = "fixture-access-token"
|
| 847 |
+
FIXTURE_CREDENTIAL = {"client_id": "amzn1.application-oa2-client.fixture", "client_secret": "fixture",
|
| 848 |
+
"refresh_token": "fixture", "profile_id": "1000000001", "region": "na"}
|
| 849 |
+
FIXTURE_FILE_HOST = "https://fixture-ads-reports.s3.amazonaws.com"
|
| 850 |
+
FIXTURE_CREATED = "2026-10-20T09:00:00Z"
|
| 851 |
+
_FIXTURE_PREFIX = {"SPONSORED_PRODUCTS": ("71", "SP"), "SPONSORED_BRANDS": ("72", "SB"),
|
| 852 |
+
"SPONSORED_DISPLAY": ("73", "SD")}
|
| 853 |
+
#: Per campaign and day: the metric totals every finer row splits evenly.
|
| 854 |
+
_FIXTURE_TOTALS = {"metric.impressions": 400, "metric.clicks": 12, "metric.totalCost": 5.0,
|
| 855 |
+
"metric.sales": 38.0, "metric.purchases": 4, "metric.unitsSold": 4}
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
def _multi_status(report=None, errors=None):
|
| 859 |
+
"""A ReportMultiStatusResponse with one success or one error entry at index 0."""
|
| 860 |
+
return {"success": [{"index": 0, "report": report}] if report else [],
|
| 861 |
+
"error": [{"index": 0, "errors": errors}] if errors else []}
|
| 862 |
+
|
| 863 |
+
|
| 864 |
class _Fixture:
|
| 865 |
+
"""The recording fixture `AMAZON_ADS_FIXTURE=1` routes to, in the saved contracts' v1 shapes.
|
|
|
|
| 866 |
|
| 867 |
+
It records method, path, operation, headers and body of every call, report file downloads
|
| 868 |
+
included. Toggles: `drop_days`, `unsupported` (group names that end FAILED), `refuse_fields`
|
| 869 |
+
(a create naming one answers 400), `nested` (files carry nested JSON lines), `responder`.
|
| 870 |
+
"""
|
| 871 |
+
|
| 872 |
+
FILES = 2
|
| 873 |
|
| 874 |
def __init__(self):
|
| 875 |
self.reset()
|
| 876 |
|
| 877 |
def reset(self):
|
| 878 |
+
self.calls, self.reports, self.subscriptions, self.credentials = [], {}, {}, {}
|
|
|
|
| 879 |
self.in_flight = self.max_in_flight = 0
|
| 880 |
+
self.drop_days, self.unsupported, self.refuse_fields = set(), set(), set()
|
| 881 |
+
self.nested = False
|
| 882 |
self.responder = None
|
| 883 |
|
| 884 |
+
def credential(self, tenant_key):
|
| 885 |
+
"""The fixture credential for one tenant. `credentials[tenant]` overrides its fields."""
|
| 886 |
+
return {**FIXTURE_CREDENTIAL, **self.credentials.get(tenant_key, {})}
|
| 887 |
+
|
| 888 |
+
def answer(self, prepared, tenant_key, profile_id, headers=None):
|
| 889 |
self.calls.append({"method": prepared.method, "path": prepared.path,
|
| 890 |
"operation": prepared.operation, "body": prepared.body,
|
| 891 |
+
"query": dict(prepared.query), "tenant": tenant_key, "headers": dict(headers or {}),
|
| 892 |
"named": dict(prepared.named) if prepared.named else None})
|
| 893 |
if self.responder is not None:
|
| 894 |
return self.responder(prepared)
|
|
|
|
| 904 |
if prepared.named is not None:
|
| 905 |
items = next(iter(prepared.body.values()))
|
| 906 |
return {"success": [{"index": i} for i in range(len(items))], "error": []}
|
| 907 |
+
if prepared.operation == "queryAdvertiserAccounts":
|
| 908 |
+
return self._accounts(prepared.body, profile_id)
|
| 909 |
if prepared.operation == "createReport":
|
| 910 |
+
return self._create(prepared.body)
|
| 911 |
+
return self._retrieve(prepared.body["reportIds"][0])
|
| 912 |
+
|
| 913 |
+
def _accounts(self, body, profile_id):
|
| 914 |
+
"""Two pages: an unrelated account, then the account that lists the caller's profile."""
|
| 915 |
+
if not body.get("nextToken"):
|
| 916 |
+
return {"advertiserAccounts": [{"advertiserAccountId": "amzn1.ads-account.g.fxother",
|
| 917 |
+
"alternateIds": [{"profileId": "999", "countryCode": "CA"}]}],
|
| 918 |
+
"nextToken": "fixture-page-2"}
|
| 919 |
+
return {"advertiserAccounts": [{"advertiserAccountId": f"amzn1.ads-account.g.fx{profile_id}",
|
| 920 |
+
"alternateIds": [{"profileId": str(profile_id), "countryCode": "US",
|
| 921 |
+
"entityId": "ENTITYFX"}]}]}
|
| 922 |
+
|
| 923 |
+
def _create(self, body):
|
| 924 |
+
if not (body.get("accessRequestedAccounts") or [{}])[0].get("advertiserAccountId"):
|
| 925 |
+
return _multi_status(errors=[{"code": "400001", "message": "An advertiser account is required.",
|
| 926 |
+
"fieldLocation": "accessRequestedAccounts"}])
|
| 927 |
+
refused = [f for f in body["reports"][0]["query"]["fields"] if f in self.refuse_fields]
|
| 928 |
+
if refused:
|
| 929 |
+
raise _rejection("Amazon Ads answered HTTP 400 on createReport",
|
| 930 |
+
[{"code": "400004", "message": f"Field {refused[0]} is invalid for this request.",
|
| 931 |
+
"fieldLocation": ""}], may_be_unoffered=False)
|
| 932 |
+
report_id = f"fx{len(self.reports) + 1:05d}"
|
| 933 |
+
self.in_flight += 1
|
| 934 |
+
self.max_in_flight = max(self.max_in_flight, self.in_flight)
|
| 935 |
+
self.reports[report_id] = {"body": body, "polls": 0, "files": None, "done": False}
|
| 936 |
+
return _multi_status(self._record(report_id, "PENDING"))
|
| 937 |
+
|
| 938 |
+
def _record(self, report_id, status, **extra):
|
| 939 |
+
"""A contract `Report`: every required key, the request's format, periods and query echoed."""
|
| 940 |
+
request = self.reports[report_id]["body"]["reports"][0]
|
| 941 |
+
return {"reportId": report_id, "status": status, "format": request["format"],
|
| 942 |
+
"periods": request["periods"], "query": request["query"],
|
| 943 |
+
"creationDateTime": FIXTURE_CREATED, "lastUpdatedDateTime": FIXTURE_CREATED, **extra}
|
| 944 |
+
|
| 945 |
+
def _retrieve(self, report_id):
|
| 946 |
+
from harness import amazon_ads_schema as schema
|
| 947 |
report = self.reports[report_id]
|
| 948 |
+
report["polls"] += 1
|
| 949 |
+
query = report["body"]["reports"][0]["query"]
|
| 950 |
+
group = schema.group_for(query["fields"], query["filter"]["on"]["values"])
|
| 951 |
+
if group is not None and group.name in self.unsupported:
|
| 952 |
+
self._finish(report)
|
| 953 |
+
return _multi_status(self._record(report_id, "FAILED", failureCode="REPORT_NOT_SUPPORTED",
|
| 954 |
+
failureReason="This report is not supported for this advertiser."))
|
| 955 |
+
if report["polls"] < 2:
|
| 956 |
+
return _multi_status(self._record(report_id, "PROCESSING"))
|
| 957 |
+
if report["files"] is None:
|
| 958 |
+
rows = fixture_rows(report["body"], drop_days=self.drop_days)
|
| 959 |
+
report["files"] = {f"{FIXTURE_FILE_HOST}/{report_id}/part-{i}.json.gz?X-Amz-Signature=fx{report_id}{i}":
|
| 960 |
+
_fixture_file(rows[i::self.FILES], self.nested) for i in range(self.FILES)}
|
| 961 |
+
return _multi_status(self._record(report_id, "COMPLETED", completedDateTime=FIXTURE_CREATED, completedReportParts=[
|
| 962 |
+
{"url": url, "sizeInBytes": len(raw), "urlExpirationDateTime": "2026-12-31T00:00:00Z"}
|
| 963 |
+
for url, raw in report["files"].items()]))
|
| 964 |
+
|
| 965 |
+
def _finish(self, report):
|
| 966 |
+
if not report["done"]:
|
| 967 |
+
report["done"] = True
|
| 968 |
self.in_flight -= 1
|
| 969 |
+
|
| 970 |
+
def download(self, url, headers):
|
| 971 |
+
"""Serve one report file and record the GET with the headers the client sent."""
|
| 972 |
+
self.calls.append({"method": "GET", "path": url, "operation": "downloadReportPart", "body": None,
|
| 973 |
+
"query": {}, "tenant": None, "headers": dict(headers), "named": None})
|
| 974 |
+
report = self.reports[urllib.parse.urlsplit(url).path.split("/")[1]]
|
| 975 |
+
raw = report["files"][url]
|
| 976 |
+
report.setdefault("read", set()).add(url)
|
| 977 |
+
if report["read"] == set(report["files"]):
|
| 978 |
+
self._finish(report)
|
| 979 |
+
return raw
|
| 980 |
+
|
| 981 |
+
|
| 982 |
+
def _fixture_file(rows, nested):
|
| 983 |
+
"""GZIP JSON: an array of flat rows, or nested objects one per line."""
|
| 984 |
+
def nest(row):
|
| 985 |
+
out = {}
|
| 986 |
+
for field, value in row.items():
|
| 987 |
+
head, _, tail = field.partition(".")
|
| 988 |
+
out.setdefault(head, {})[tail] = value
|
| 989 |
+
return out
|
| 990 |
+
text = "\n".join(json.dumps(nest(r)) for r in rows) if nested else json.dumps(rows)
|
| 991 |
+
return gzip.compress(text.encode("utf-8"))
|
| 992 |
+
|
| 993 |
+
|
| 994 |
+
def _fixture_query(body):
|
| 995 |
+
report = body["reports"][0]
|
| 996 |
+
period = report["periods"][0]["datePeriod"]
|
| 997 |
+
return (report["query"]["fields"], report["query"]["filter"]["on"]["values"],
|
| 998 |
+
date.fromisoformat(period["startDate"]), date.fromisoformat(period["endDate"]))
|
| 999 |
+
|
| 1000 |
+
|
| 1001 |
+
def _fixture_days(start, end, drop_days):
|
| 1002 |
+
return [start + timedelta(days=i) for i in range((end - start).days + 1)
|
| 1003 |
+
if (start + timedelta(days=i)).isoformat() not in drop_days]
|
| 1004 |
+
|
| 1005 |
+
|
| 1006 |
+
def fixture_totals(body, *, drop_days=()):
|
| 1007 |
+
"""The fixture's own totals for one create body. -> {(ad product, date, campaign id): {field: total}}."""
|
| 1008 |
+
_fields, products, start, end = _fixture_query(body)
|
| 1009 |
+
out = {}
|
| 1010 |
+
for day in _fixture_days(start, end, drop_days):
|
| 1011 |
+
for index, product in enumerate(products):
|
| 1012 |
+
for c in (1, 2):
|
| 1013 |
+
n = day.toordinal() % 7 + c + index
|
| 1014 |
+
out[(product, day.isoformat(), f"{_FIXTURE_PREFIX[product][0]}10{c}")] = {
|
| 1015 |
+
field: total * n for field, total in _FIXTURE_TOTALS.items()}
|
| 1016 |
+
return out
|
| 1017 |
|
| 1018 |
|
| 1019 |
def fixture_rows(body, *, drop_days=()):
|
| 1020 |
+
"""Deterministic v1 rows for one create body: one row per distinct value of every requested
|
| 1021 |
+
dimension and day, only the requested fields and ad products. Two ad groups split a campaign."""
|
| 1022 |
+
fields, _products, start, end = _fixture_query(body)
|
| 1023 |
+
groups = (1, 2) if "adGroup.id" in fields else (0,)
|
| 1024 |
+
out = []
|
| 1025 |
+
for (product, day, campaign), totals in fixture_totals(body, drop_days=drop_days).items():
|
| 1026 |
+
prefix, short = _FIXTURE_PREFIX[product]
|
| 1027 |
+
c = campaign[-1]
|
| 1028 |
+
for g in groups:
|
| 1029 |
+
row = {"date.value": day, "adProduct.value": product, "campaign.id": campaign,
|
| 1030 |
+
"campaign.name": f"{short} campaign {c}", "campaign.deliveryStatus": "DELIVERING",
|
| 1031 |
+
"campaign.budgetAmount": 25 * int(c), "budgetCurrency.value": "USD",
|
| 1032 |
+
"adGroup.id": f"{prefix}20{c}{g}", "adGroup.name": f"Ad group {c}.{g}",
|
| 1033 |
+
"adGroupTarget.id": f"{prefix}30{c}{g}", "target.value": f"term {c}{g}",
|
| 1034 |
+
"target.matchType": "EXACT", "adGroupTarget.bid": str(0.5 + g / 10),
|
| 1035 |
+
"ad.id": f"{prefix}40{c}{g}", "advertisedProduct.id": f"B0FX{prefix}{c}{g}00",
|
| 1036 |
+
"advertisedProductMarketplace.value": "ATVPDKIKX0DER", "advertisedProductSku.value": f"SKU-{c}{g}",
|
| 1037 |
+
"searchTerm.value": f"search {c}{g}",
|
| 1038 |
+
**{f: (t // len(groups) if isinstance(t, int) else t / len(groups)) for f, t in totals.items()}}
|
| 1039 |
+
out.append({f: row[f] for f in fields})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1040 |
return out
|
| 1041 |
|
| 1042 |
|
| 1043 |
FIXTURE = _Fixture()
|
| 1044 |
|
| 1045 |
|
| 1046 |
+
def _reset_caches():
|
| 1047 |
+
_ACCOUNTS.clear()
|
| 1048 |
+
_REFUSED.clear()
|
| 1049 |
+
_ISSUED_FILES.clear()
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
@contextmanager
|
| 1053 |
def recording_fixture():
|
| 1054 |
+
"""Route every call to a fresh recording fixture and fresh report caches for the block. -> the fixture."""
|
| 1055 |
previous = os.environ.get(FIXTURE_ENV)
|
| 1056 |
os.environ[FIXTURE_ENV] = "1"
|
| 1057 |
FIXTURE.reset()
|
| 1058 |
+
_reset_caches()
|
| 1059 |
try:
|
| 1060 |
yield FIXTURE
|
| 1061 |
finally:
|
| 1062 |
FIXTURE.reset()
|
| 1063 |
+
_reset_caches()
|
| 1064 |
if previous is None:
|
| 1065 |
os.environ.pop(FIXTURE_ENV, None)
|
| 1066 |
else:
|
| 1067 |
os.environ[FIXTURE_ENV] = previous
|
| 1068 |
|
| 1069 |
|
| 1070 |
+
__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "V1_CLIENT_HEADERS",
|
| 1071 |
+
"NAMED_WRITES", "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
|
| 1072 |
"FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
|
| 1073 |
"SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
|
| 1074 |
+
"MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsRejected", "AdsReportUnavailable",
|
| 1075 |
+
"AdsReportPending", "GroupReport", "fixture_mode", "build_request", "report_operation",
|
| 1076 |
+
"send_named_write", "request", "credential", "run_group_report", "run_report", "FIXTURE",
|
| 1077 |
+
"recording_fixture", "fixture_rows", "fixture_totals"]
|
platform/harness/amazon_ads_schema.py
CHANGED
|
@@ -1,16 +1,17 @@
|
|
| 1 |
"""amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
|
| 2 |
|
| 3 |
-
One entry per ad type and report: the
|
| 4 |
-
|
| 5 |
-
|
| 6 |
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
"""
|
| 11 |
import hashlib
|
| 12 |
import sys
|
| 13 |
from pathlib import Path
|
|
|
|
| 14 |
|
| 15 |
_HERE = Path(__file__).resolve().parent
|
| 16 |
if str(_HERE.parent) not in sys.path:
|
|
@@ -22,8 +23,6 @@ AD_TYPES = _cs.AMAZON_ADS_AD_TYPES
|
|
| 22 |
REPORTS = _cs.AMAZON_ADS_REPORTS
|
| 23 |
DATABASE_PREFIX = "ut_amzn_"
|
| 24 |
MIRROR_PREFIX = "amzn_ads_"
|
| 25 |
-
#: Amazon accepts at most 31 days per report request.
|
| 26 |
-
MAX_DAYS_PER_REQUEST = 31
|
| 27 |
|
| 28 |
#: The C6 columns every Ads table carries, in this order.
|
| 29 |
C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
|
|
@@ -54,22 +53,76 @@ _EXTRA = {
|
|
| 54 |
}
|
| 55 |
_TAIL = ("profile_id", "currency")
|
| 56 |
|
| 57 |
-
#: Our column ->
|
| 58 |
_VENDOR = {
|
| 59 |
-
"campaign_id": "
|
| 60 |
-
"campaign_status": "
|
| 61 |
-
"ad_group_id": "
|
| 62 |
-
"targeting": "
|
| 63 |
-
"ad_id": "
|
| 64 |
-
"search_term": "searchTerm", "
|
| 65 |
-
"
|
| 66 |
-
"units": "unitsSold", "currency": "
|
| 67 |
-
}
|
| 68 |
-
_VENDOR_BY_TYPE = {
|
| 69 |
-
"sp": {"sales": "sales14d", "orders": "purchases14d", "units": "unitsSoldClicks14d"},
|
| 70 |
-
"sd": {"target_id": "targetingId", "targeting": "targetingExpression",
|
| 71 |
-
"advertised_sku": "promotedSku", "asin": "promotedAsin"},
|
| 72 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
|
| 75 |
def _short(ad_type, report):
|
|
@@ -89,25 +142,24 @@ def coltype(name):
|
|
| 89 |
return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
|
| 90 |
|
| 91 |
|
| 92 |
-
def vendor_name(
|
| 93 |
-
"""The
|
| 94 |
-
|
| 95 |
-
return None
|
| 96 |
-
return _VENDOR_BY_TYPE.get(ad_type, {}).get(column) or _VENDOR.get(column)
|
| 97 |
|
| 98 |
|
| 99 |
TABLE_SPECS = {}
|
| 100 |
for _spec in _cs.AMAZON_ADS_TABLES:
|
| 101 |
_ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
|
|
|
|
| 102 |
TABLE_SPECS[_short(_ad_type, _report)] = {
|
| 103 |
"key": _spec.key, "source": _spec.source, "label": _spec.label,
|
| 104 |
-
"ad_type": _ad_type, "report": _report, "
|
| 105 |
"columns": _columns(_report), "row_identity": tuple(_spec.identity),
|
| 106 |
"backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
|
| 107 |
"offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
|
| 108 |
"not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
|
| 109 |
}
|
| 110 |
-
del _spec, _ad_type, _report
|
| 111 |
|
| 112 |
|
| 113 |
def short_name(table):
|
|
@@ -127,11 +179,16 @@ def columns_for(table):
|
|
| 127 |
return TABLE_SPECS[short_name(table)]["columns"]
|
| 128 |
|
| 129 |
|
| 130 |
-
def
|
| 131 |
-
"""The
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
|
| 137 |
def _cell(name, value):
|
|
@@ -149,11 +206,11 @@ def _cell(name, value):
|
|
| 149 |
|
| 150 |
|
| 151 |
def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
|
| 152 |
-
"""One
|
| 153 |
spec = TABLE_SPECS[short_name(table)]
|
| 154 |
out = {}
|
| 155 |
for column in spec["columns"]:
|
| 156 |
-
source = vendor_name(
|
| 157 |
value = vendor_row.get(source) if source else None
|
| 158 |
if column == "profile_id":
|
| 159 |
value = profile_id
|
|
@@ -213,7 +270,8 @@ def stream_coltype(name):
|
|
| 213 |
return "DOUBLE"
|
| 214 |
return "VARCHAR"
|
| 215 |
|
| 216 |
-
__all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "
|
|
|
|
| 217 |
"ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
|
| 218 |
-
"
|
| 219 |
"STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
|
|
|
|
| 1 |
"""amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
|
| 2 |
|
| 3 |
+
One entry per ad type and report: the columns we store, the Reporting v1 field each one is read
|
| 4 |
+
from, the row identity and the backfill window. The table keys, labels and backfill days come from
|
| 5 |
+
`core.connector_spec`, so the two files cannot disagree on a name.
|
| 6 |
|
| 7 |
+
Field ids come from Amazon's v1 field library, saved at
|
| 8 |
+
`AutoWiki/research/amazon-ads-v1-schema/reporting_v1_fields.json`. v1 has no report types: one
|
| 9 |
+
report per `REPORT_GROUPS` row covers several ad types, and its rows split by `adProduct.value`.
|
| 10 |
"""
|
| 11 |
import hashlib
|
| 12 |
import sys
|
| 13 |
from pathlib import Path
|
| 14 |
+
from typing import NamedTuple
|
| 15 |
|
| 16 |
_HERE = Path(__file__).resolve().parent
|
| 17 |
if str(_HERE.parent) not in sys.path:
|
|
|
|
| 23 |
REPORTS = _cs.AMAZON_ADS_REPORTS
|
| 24 |
DATABASE_PREFIX = "ut_amzn_"
|
| 25 |
MIRROR_PREFIX = "amzn_ads_"
|
|
|
|
|
|
|
| 26 |
|
| 27 |
#: The C6 columns every Ads table carries, in this order.
|
| 28 |
C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
|
|
|
|
| 53 |
}
|
| 54 |
_TAIL = ("profile_id", "currency")
|
| 55 |
|
| 56 |
+
#: Our column -> its Reporting v1 field id, one map for every ad type. The store stamps profile_id.
|
| 57 |
_VENDOR = {
|
| 58 |
+
"date": "date.value", "campaign_id": "campaign.id", "campaign_name": "campaign.name",
|
| 59 |
+
"campaign_status": "campaign.deliveryStatus", "campaign_budget_amount": "campaign.budgetAmount",
|
| 60 |
+
"ad_group_id": "adGroup.id", "ad_group_name": "adGroup.name", "target_id": "adGroupTarget.id",
|
| 61 |
+
"targeting": "target.value", "match_type": "target.matchType", "bid": "adGroupTarget.bid",
|
| 62 |
+
"ad_id": "ad.id", "advertised_sku": "advertisedProductSku.value", "asin": "advertisedProduct.id",
|
| 63 |
+
"search_term": "searchTerm.value", "impressions": "metric.impressions", "clicks": "metric.clicks",
|
| 64 |
+
"cost": "metric.totalCost", "sales": "metric.sales", "orders": "metric.purchases",
|
| 65 |
+
"units": "metric.unitsSold", "currency": "budgetCurrency.value",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
}
|
| 67 |
+
#: The v1 field that names each row's ad type, and its value per ad type.
|
| 68 |
+
AD_PRODUCT_FIELD = "adProduct.value"
|
| 69 |
+
AD_PRODUCTS = {"sp": "SPONSORED_PRODUCTS", "sb": "SPONSORED_BRANDS", "sd": "SPONSORED_DISPLAY"}
|
| 70 |
+
#: The name the v1 field library lists each ad type under.
|
| 71 |
+
LIBRARY_AD_PRODUCTS = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
|
| 72 |
+
|
| 73 |
+
METRICS = ("metric.impressions", "metric.clicks", "metric.totalCost", "metric.sales",
|
| 74 |
+
"metric.purchases", "metric.unitsSold")
|
| 75 |
+
_HEAD = ("date.value", AD_PRODUCT_FIELD, "campaign.id", "campaign.name")
|
| 76 |
+
_AD_GROUP = ("adGroup.id", "adGroup.name")
|
| 77 |
+
_TAIL_FIELDS = ("budgetCurrency.value",) + METRICS
|
| 78 |
+
#: ⛔ Not `targeting.value` or `targetingMatchType.value`: the library lists them for no ad product.
|
| 79 |
+
#: The live probe of 2026-11-18 (`probe-2026-11-18.json`) proved these names, absent from the library.
|
| 80 |
+
_TARGET_TEXT = ("target.value", "target.matchType")
|
| 81 |
+
#: Fields a live Reporting v1 create accepted that the saved field library does not list.
|
| 82 |
+
PROBE_PROVEN = frozenset(_TARGET_TEXT)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class ReportGroup(NamedTuple):
|
| 86 |
+
"""One Reporting v1 report and the member tables its rows split into by `adProduct.value`.
|
| 87 |
+
|
| 88 |
+
`optional` fields list no ad product in Amazon's library, or only a live probe proved them:
|
| 89 |
+
a create that refuses one drops it.
|
| 90 |
+
"""
|
| 91 |
+
name: str
|
| 92 |
+
members: tuple
|
| 93 |
+
core: tuple
|
| 94 |
+
optional: tuple = ()
|
| 95 |
+
|
| 96 |
+
@property
|
| 97 |
+
def tables(self):
|
| 98 |
+
"""`adProduct.value` -> the member table its rows land in."""
|
| 99 |
+
return {AD_PRODUCTS[table.split("_", 1)[0]]: table for table in self.members}
|
| 100 |
+
|
| 101 |
+
def fields(self, drop=()):
|
| 102 |
+
"""The fields one create asks for, without the optional fields Amazon refused."""
|
| 103 |
+
return tuple(f for f in self.core + self.optional if f not in drop)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
#: ⛔ Every requested dimension is a group-by and sets the row grain. A group asks only for its
|
| 107 |
+
#: tables' identity dimensions, attributes of those, metrics, the ad product and required partners.
|
| 108 |
+
#: Every core field lists every ad product of its group in the library: SB lacks the SKU field and
|
| 109 |
+
#: the marketplace field `advertisedProduct.id` requires, so SB advertised products is its own report.
|
| 110 |
+
REPORT_GROUPS = (
|
| 111 |
+
ReportGroup("campaigns", ("sp_campaigns", "sb_campaigns", "sd_campaigns"),
|
| 112 |
+
_HEAD + ("campaign.deliveryStatus", "campaign.budgetAmount") + _TAIL_FIELDS),
|
| 113 |
+
ReportGroup("targeting", ("sp_targeting", "sb_targeting", "sd_targeting"),
|
| 114 |
+
_HEAD + _AD_GROUP + ("adGroupTarget.id",) + _TAIL_FIELDS,
|
| 115 |
+
_TARGET_TEXT + ("adGroupTarget.bid",)),
|
| 116 |
+
ReportGroup("advertised_products", ("sp_advertised_products", "sd_advertised_products"),
|
| 117 |
+
_HEAD + _AD_GROUP + ("ad.id", "advertisedProduct.id", "advertisedProductMarketplace.value",
|
| 118 |
+
"advertisedProductSku.value") + _TAIL_FIELDS),
|
| 119 |
+
ReportGroup("advertised_products_sb", ("sb_advertised_products",),
|
| 120 |
+
_HEAD + _AD_GROUP + ("ad.id",) + _TAIL_FIELDS),
|
| 121 |
+
ReportGroup("search_terms", ("sp_search_terms", "sb_search_terms"),
|
| 122 |
+
_HEAD + _AD_GROUP + ("adGroupTarget.id", "searchTerm.value") + _TAIL_FIELDS, _TARGET_TEXT),
|
| 123 |
+
)
|
| 124 |
+
GROUPS = {group.name: group for group in REPORT_GROUPS}
|
| 125 |
+
_GROUP_OF = {table: group for group in REPORT_GROUPS for table in group.members}
|
| 126 |
|
| 127 |
|
| 128 |
def _short(ad_type, report):
|
|
|
|
| 142 |
return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
|
| 143 |
|
| 144 |
|
| 145 |
+
def vendor_name(column):
|
| 146 |
+
"""The v1 field one of our columns is read from, or None when the vendor has none."""
|
| 147 |
+
return _VENDOR.get(column)
|
|
|
|
|
|
|
| 148 |
|
| 149 |
|
| 150 |
TABLE_SPECS = {}
|
| 151 |
for _spec in _cs.AMAZON_ADS_TABLES:
|
| 152 |
_ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
|
| 153 |
+
_group = _GROUP_OF.get(_short(_ad_type, _report))
|
| 154 |
TABLE_SPECS[_short(_ad_type, _report)] = {
|
| 155 |
"key": _spec.key, "source": _spec.source, "label": _spec.label,
|
| 156 |
+
"ad_type": _ad_type, "report": _report, "group": _group.name if _group else None,
|
| 157 |
"columns": _columns(_report), "row_identity": tuple(_spec.identity),
|
| 158 |
"backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
|
| 159 |
"offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
|
| 160 |
"not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
|
| 161 |
}
|
| 162 |
+
del _spec, _ad_type, _report, _group
|
| 163 |
|
| 164 |
|
| 165 |
def short_name(table):
|
|
|
|
| 179 |
return TABLE_SPECS[short_name(table)]["columns"]
|
| 180 |
|
| 181 |
|
| 182 |
+
def group_of(table):
|
| 183 |
+
"""The report group that fills one Ads table, or None for a table v1 does not offer."""
|
| 184 |
+
return _GROUP_OF.get(short_name(table))
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def group_for(fields, ad_products):
|
| 188 |
+
"""The report group a create request asks for, by its fields and ad product filter, or None."""
|
| 189 |
+
fields, ad_products = set(fields), set(ad_products)
|
| 190 |
+
return next((g for g in REPORT_GROUPS if set(g.tables) == ad_products
|
| 191 |
+
and set(g.core) <= fields <= set(g.core + g.optional)), None)
|
| 192 |
|
| 193 |
|
| 194 |
def _cell(name, value):
|
|
|
|
| 206 |
|
| 207 |
|
| 208 |
def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
|
| 209 |
+
"""One flat v1 report row (dotted field ids) -> one stored row, with its `_row_key` and report origin."""
|
| 210 |
spec = TABLE_SPECS[short_name(table)]
|
| 211 |
out = {}
|
| 212 |
for column in spec["columns"]:
|
| 213 |
+
source = vendor_name(column)
|
| 214 |
value = vendor_row.get(source) if source else None
|
| 215 |
if column == "profile_id":
|
| 216 |
value = profile_id
|
|
|
|
| 270 |
return "DOUBLE"
|
| 271 |
return "VARCHAR"
|
| 272 |
|
| 273 |
+
__all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "AD_PRODUCT_FIELD", "AD_PRODUCTS",
|
| 274 |
+
"LIBRARY_AD_PRODUCTS", "METRICS", "ReportGroup", "REPORT_GROUPS", "GROUPS",
|
| 275 |
"ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
|
| 276 |
+
"group_of", "group_for", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
|
| 277 |
"STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
|
platform/harness/amazon_ads_store.py
CHANGED
|
@@ -10,8 +10,8 @@ back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_B
|
|
| 10 |
A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
|
| 11 |
No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
|
| 12 |
|
| 13 |
-
⛔ STANDING RULE 1. Every report is read whole
|
| 14 |
-
report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
|
| 15 |
`amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
|
| 16 |
|
| 17 |
Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
|
|
@@ -178,7 +178,7 @@ def _day_marker(con, key):
|
|
| 178 |
|
| 179 |
|
| 180 |
def windows(con, table, connect_day, today, connection_id=None):
|
| 181 |
-
"""The report windows one sync asks for, oldest first, each at most
|
| 182 |
table = schema.short_name(table)
|
| 183 |
spec = schema.TABLE_SPECS[table]
|
| 184 |
cadence = amazon_client.CADENCE["ads_" + table]
|
|
@@ -237,13 +237,13 @@ def _upsert(con, table, rows):
|
|
| 237 |
cols = _columns_for(table)
|
| 238 |
by_key = {r["_row_key"]: r for r in rows}
|
| 239 |
keys = list(by_key)
|
|
|
|
| 240 |
for i in range(0, len(keys), INSERT_BATCH_ROWS):
|
| 241 |
part = keys[i:i + INSERT_BATCH_ROWS]
|
| 242 |
con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
[[by_key[k].get(c) for c in cols] for k in part])
|
| 247 |
return len(by_key)
|
| 248 |
|
| 249 |
|
|
@@ -389,16 +389,158 @@ def _land(con, table, rows, report_id, start, end, profile, connection_id, label
|
|
| 389 |
|
| 390 |
|
| 391 |
def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
|
| 392 |
-
"""
|
| 393 |
options = {"rt": rt, "profile_id": profile_id, **extra}
|
| 394 |
if connection_id:
|
| 395 |
options["connection_id"] = connection_id
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 402 |
|
| 403 |
|
| 404 |
def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
|
|
@@ -409,6 +551,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 409 |
through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
|
| 410 |
`max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
|
| 411 |
pending ledger, and the next pass reads it before it creates any other report.
|
|
|
|
| 412 |
"""
|
| 413 |
key = amazon_store._tenant_key(tenant_key)
|
| 414 |
report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
|
|
@@ -425,10 +568,9 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 425 |
own = con is None
|
| 426 |
if own:
|
| 427 |
con, _path = amazon_store._connect_tenant_mirror(key, rt)
|
| 428 |
-
bounded = max_seconds is not None
|
| 429 |
started = _clock()
|
| 430 |
-
deadline =
|
| 431 |
-
|
| 432 |
try:
|
| 433 |
datastore._ensure_sync_state(con)
|
| 434 |
now = _utc_now(now)
|
|
@@ -437,164 +579,24 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 437 |
def stamp():
|
| 438 |
return now + timedelta(seconds=max(0.0, _clock() - started))
|
| 439 |
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
synced = []
|
| 444 |
-
checkpointed = []
|
| 445 |
-
for cid, label, profile, legacy in connections:
|
| 446 |
connect_day = _connect_day(con, today, cid, legacy=legacy)
|
| 447 |
report.setdefault("connectDay", connect_day.isoformat())
|
| 448 |
entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
|
| 449 |
report["connections"].append(entry)
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
pending = _read_pending(con, table, cid)
|
| 455 |
-
if pending is None:
|
| 456 |
-
continue
|
| 457 |
-
spec = schema.TABLE_SPECS[table]
|
| 458 |
-
_ensure_table(con, table)
|
| 459 |
-
if table_name(table) not in checkpointed:
|
| 460 |
-
checkpointed.append(table_name(table))
|
| 461 |
-
age = (stamp() - pending["created"]).total_seconds()
|
| 462 |
-
expired = (f"Amazon Ads report {pending['report_id']} did not finish in 3 hours")
|
| 463 |
-
if age >= client.REPORT_MAX_WAIT_SECONDS:
|
| 464 |
-
_clear_pending(con, table, cid)
|
| 465 |
-
report["problems"].append(f"{label}: {spec['label']}: {expired}")
|
| 466 |
-
resumed[table] = ("failed", expired, 0, None)
|
| 467 |
-
continue
|
| 468 |
-
limit = client.REPORT_MAX_WAIT_SECONDS - age
|
| 469 |
-
if bounded:
|
| 470 |
-
limit = min(limit, max(0.0, deadline - _clock()))
|
| 471 |
-
try:
|
| 472 |
-
report_id, rows = _run_report(key, table, pending["start"], pending["end"], rt, profile, cid,
|
| 473 |
-
max_wait=limit, report_id=pending["report_id"], now=_clock)
|
| 474 |
-
except client.AdsReportUnavailable as exc:
|
| 475 |
-
_clear_pending(con, table, cid)
|
| 476 |
-
resumed[table] = ("not_offered", _not_offered_reason(spec, exc), 0, None)
|
| 477 |
-
continue
|
| 478 |
-
except client.AdsReportPending:
|
| 479 |
-
if (stamp() - pending["created"]).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS:
|
| 480 |
-
_clear_pending(con, table, cid)
|
| 481 |
-
report["problems"].append(f"{label}: {spec['label']}: {expired}")
|
| 482 |
-
resumed[table] = ("failed", expired, 0, None)
|
| 483 |
-
else:
|
| 484 |
-
blocked = True
|
| 485 |
-
wait(label, spec, PREPARING_REASON)
|
| 486 |
-
resumed[table] = ("deferred", PREPARING_REASON, 0, None)
|
| 487 |
-
continue
|
| 488 |
-
except client.AdsError as exc:
|
| 489 |
-
_clear_pending(con, table, cid)
|
| 490 |
-
reason = str(exc)[:200]
|
| 491 |
-
report["problems"].append(f"{label}: {spec['label']}: {reason}")
|
| 492 |
-
resumed[table] = ("failed", reason, 0, None)
|
| 493 |
-
continue
|
| 494 |
-
_clear_pending(con, table, cid)
|
| 495 |
-
pulled = _land(con, table, rows, report_id, pending["start"], pending["end"], profile, cid, label)
|
| 496 |
-
resumed[table] = ("done", "", pulled, (pending["start"], pending["end"]))
|
| 497 |
-
requested = [schema.short_name(t) for t in (tables or TABLES)]
|
| 498 |
for table in list(dict.fromkeys(requested + list(resumed))):
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
prior = resumed.get(table)
|
| 504 |
-
landed = None
|
| 505 |
-
if prior is not None:
|
| 506 |
-
state, reason, pulled, landed = prior
|
| 507 |
-
if state != "done" or table not in requested:
|
| 508 |
-
pass
|
| 509 |
-
elif not spec["offered"]:
|
| 510 |
-
state, reason = "not_offered", spec["not_offered_reason"]
|
| 511 |
-
elif not _cadence_due(con, table, now, cid):
|
| 512 |
-
state = "skipped"
|
| 513 |
-
elif blocked:
|
| 514 |
-
state, reason = "deferred", PREPARING_REASON
|
| 515 |
-
wait(label, spec, reason)
|
| 516 |
-
else:
|
| 517 |
-
for start, end in windows(con, table, connect_day, today, cid):
|
| 518 |
-
if landed and landed[0] <= start and end <= landed[1]:
|
| 519 |
-
continue # The resumed report just read these days.
|
| 520 |
-
if used >= client.REPORTS_PER_DAY:
|
| 521 |
-
state = "deferred"
|
| 522 |
-
reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
|
| 523 |
-
f"reports are used, so days from {start.isoformat()} load on the next sync.")
|
| 524 |
-
report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit")
|
| 525 |
-
waits.append(report["problems"][-1])
|
| 526 |
-
break
|
| 527 |
-
if bounded and _clock() >= deadline:
|
| 528 |
-
state, reason = "deferred", TIME_LIMIT_REASON
|
| 529 |
-
wait(label, spec, reason)
|
| 530 |
-
break
|
| 531 |
-
used += 1
|
| 532 |
-
_spend_budget(con, today, used, cid)
|
| 533 |
-
created = stamp()
|
| 534 |
-
try:
|
| 535 |
-
if bounded:
|
| 536 |
-
report_id, rows = _run_report(key, table, start, end, rt, profile, cid,
|
| 537 |
-
max_wait=max(0.0, deadline - _clock()), now=_clock)
|
| 538 |
-
else:
|
| 539 |
-
report_id, rows = _run_report(key, table, start, end, rt, profile, cid)
|
| 540 |
-
except client.AdsReportUnavailable as exc:
|
| 541 |
-
state = "not_offered"
|
| 542 |
-
reason = _not_offered_reason(spec, exc)
|
| 543 |
-
break
|
| 544 |
-
except client.AdsError as exc:
|
| 545 |
-
if (bounded and isinstance(exc, client.AdsReportPending) and exc.report_id
|
| 546 |
-
and (stamp() - created).total_seconds() < client.REPORT_MAX_WAIT_SECONDS):
|
| 547 |
-
_write_pending(con, table, cid, exc.report_id, start, end, created)
|
| 548 |
-
if table_name(table) not in checkpointed:
|
| 549 |
-
checkpointed.append(table_name(table))
|
| 550 |
-
blocked = True
|
| 551 |
-
state, reason = "deferred", PREPARING_REASON
|
| 552 |
-
wait(label, spec, reason)
|
| 553 |
-
break
|
| 554 |
-
state = "failed"
|
| 555 |
-
reason = str(exc)[:200]
|
| 556 |
-
report["problems"].append(f"{label}: {spec['label']}: {reason}")
|
| 557 |
-
break
|
| 558 |
-
pulled += _land(con, table, rows, report_id, start, end, profile, cid, label)
|
| 559 |
-
if table_name(table) not in checkpointed:
|
| 560 |
-
checkpointed.append(table_name(table))
|
| 561 |
-
if state == "done":
|
| 562 |
-
_record_cadence(con, table, now, cid)
|
| 563 |
-
if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
|
| 564 |
-
report["partial"] = True
|
| 565 |
-
if reason:
|
| 566 |
-
_record_limit(con, table, reason, cid, state=state)
|
| 567 |
-
shown = f"{label}: {reason}" if cid else reason
|
| 568 |
-
report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
|
| 569 |
-
elif state == "done":
|
| 570 |
-
_clear_limit(con, table, cid)
|
| 571 |
-
total = _count(con, table)
|
| 572 |
-
entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
|
| 573 |
-
if state != "skipped":
|
| 574 |
-
amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid),
|
| 575 |
-
"done" if state == "done" else "partial", _count(con, table, cid), state)
|
| 576 |
-
previous = report["tables"].get(table, {})
|
| 577 |
-
severity = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
|
| 578 |
-
aggregate = max((state, previous.get("state", state)), key=severity.get)
|
| 579 |
-
report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled,
|
| 580 |
-
"in_mirror": total}
|
| 581 |
-
if (pulled or origin_changed) and table_name(table) not in synced:
|
| 582 |
synced.append(table_name(table))
|
| 583 |
-
|
| 584 |
-
for table, state in report["tables"].items():
|
| 585 |
-
if state["state"] == "skipped":
|
| 586 |
-
continue
|
| 587 |
-
amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
|
| 588 |
-
"done" if state["state"] == "done" else "partial",
|
| 589 |
-
state["in_mirror"], state["state"])
|
| 590 |
-
if table in report["limits"]:
|
| 591 |
-
_record_limit(con, table, report["limits"][table], state=state["state"])
|
| 592 |
-
else:
|
| 593 |
-
_clear_limit(con, table)
|
| 594 |
-
report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
|
| 595 |
-
for cid, *_ in connections for table in TABLES)
|
| 596 |
-
if report["partial"]:
|
| 597 |
-
report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
|
| 598 |
if own and rt is not None and (synced or checkpointed):
|
| 599 |
_persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
|
| 600 |
finally:
|
|
@@ -604,10 +606,75 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
|
|
| 604 |
return report
|
| 605 |
|
| 606 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 607 |
def _not_offered_reason(spec, exc):
|
| 608 |
return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
|
| 609 |
-
f"({str(exc)[:
|
| 610 |
-
f"contact Amazon Ads support
|
| 611 |
|
| 612 |
|
| 613 |
def _persist(tenant_key, rt, con, synced, report, log):
|
|
|
|
| 10 |
A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
|
| 11 |
No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
|
| 12 |
|
| 13 |
+
⛔ STANDING RULE 1. Every report is read whole, every file of it. The limits that cannot be removed
|
| 14 |
+
(one report at a time per client ID, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
|
| 15 |
`amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
|
| 16 |
|
| 17 |
Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
|
|
|
|
| 178 |
|
| 179 |
|
| 180 |
def windows(con, table, connect_day, today, connection_id=None):
|
| 181 |
+
"""The report windows one sync asks for, oldest first, each at most 120 days (`client.MAX_DAYS_PER_REQUEST`)."""
|
| 182 |
table = schema.short_name(table)
|
| 183 |
spec = schema.TABLE_SPECS[table]
|
| 184 |
cadence = amazon_client.CADENCE["ads_" + table]
|
|
|
|
| 237 |
cols = _columns_for(table)
|
| 238 |
by_key = {r["_row_key"]: r for r in rows}
|
| 239 |
keys = list(by_key)
|
| 240 |
+
one = f'({", ".join("?" for _ in cols)})'
|
| 241 |
for i in range(0, len(keys), INSERT_BATCH_ROWS):
|
| 242 |
part = keys[i:i + INSERT_BATCH_ROWS]
|
| 243 |
con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
|
| 244 |
+
# One multi-row INSERT per batch: DuckDB's executemany is about 5 times slower at 24 months a pass.
|
| 245 |
+
con.execute(f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
|
| 246 |
+
f'VALUES {", ".join(one for _ in part)}', [by_key[k].get(c) for k in part for c in cols])
|
|
|
|
| 247 |
return len(by_key)
|
| 248 |
|
| 249 |
|
|
|
|
| 389 |
|
| 390 |
|
| 391 |
def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
|
| 392 |
+
"""Run the group report that fills `table` under this connection's own credential. -> GroupReport."""
|
| 393 |
options = {"rt": rt, "profile_id": profile_id, **extra}
|
| 394 |
if connection_id:
|
| 395 |
options["connection_id"] = connection_id
|
| 396 |
+
return client.run_group_report(key, schema.group_of(table), start, end, **options)
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
class _Pass:
|
| 400 |
+
"""One connection's sync pass: its budget, clock, and what its group reports already covered.
|
| 401 |
+
|
| 402 |
+
`eligible` holds the tables a group report may land into this pass: requested, offered, due and
|
| 403 |
+
not yet finished. `covered` holds each table's spans landed this pass, `pulled` the rows a
|
| 404 |
+
sibling's report landed, `outcome` a group's failure, `notes` Amazon's text per table.
|
| 405 |
+
"""
|
| 406 |
+
|
| 407 |
+
def __init__(self, con, key, rt, connection, report, *, today, used, deadline, stamp, checkpointed):
|
| 408 |
+
self.con, self.key, self.rt, self.report = con, key, rt, report
|
| 409 |
+
self.cid, self.label, self.profile = connection[:3]
|
| 410 |
+
self.today, self.used, self.deadline, self.stamp = today, used, deadline, stamp
|
| 411 |
+
self.checkpointed = checkpointed
|
| 412 |
+
self.eligible, self.covered, self.pulled, self.outcome, self.notes = set(), {}, {}, {}, {}
|
| 413 |
+
self.blocked = False
|
| 414 |
+
|
| 415 |
+
def wait(self, spec, reason):
|
| 416 |
+
self.report["waits"].append(f"{self.label}: {spec['label']}: {reason}")
|
| 417 |
+
|
| 418 |
+
def checkpoint(self, table):
|
| 419 |
+
if table_name(table) not in self.checkpointed:
|
| 420 |
+
self.checkpointed.append(table_name(table))
|
| 421 |
+
|
| 422 |
+
def covers(self, table, start, end):
|
| 423 |
+
"""True when reports this pass already read every day from `start` to `end` for `table`."""
|
| 424 |
+
spans = self.covered.get(table, ())
|
| 425 |
+
return all(any(lo <= start + timedelta(days=i) <= hi for lo, hi in spans)
|
| 426 |
+
for i in range((end - start).days + 1))
|
| 427 |
+
|
| 428 |
+
def bounds(self):
|
| 429 |
+
"""The poll limit for one new report: none, or the time left in this bounded pass."""
|
| 430 |
+
return {} if self.deadline is None else {"max_wait": max(0.0, self.deadline - _clock()), "now": _clock}
|
| 431 |
+
|
| 432 |
+
def fail(self, table, exc):
|
| 433 |
+
"""Record a group report's failure for every member. -> (state, reason) for `table`."""
|
| 434 |
+
self.outcome[schema.group_of(table).name] = exc
|
| 435 |
+
return self.failure(table, exc)
|
| 436 |
+
|
| 437 |
+
def failure(self, table, exc):
|
| 438 |
+
spec = schema.TABLE_SPECS[table]
|
| 439 |
+
if isinstance(exc, client.AdsReportUnavailable):
|
| 440 |
+
return "not_offered", _not_offered_reason(spec, exc)
|
| 441 |
+
reason = str(exc)[:client.ERROR_TEXT_LIMIT]
|
| 442 |
+
self.report["problems"].append(f"{self.label}: {spec['label']}: {reason}")
|
| 443 |
+
return "failed", reason
|
| 444 |
+
|
| 445 |
+
def land(self, table, result, start, end):
|
| 446 |
+
"""Land one group report into `table` and each eligible sibling, covering the span. -> rows into `table`."""
|
| 447 |
+
own = 0
|
| 448 |
+
for member in schema.group_of(table).members:
|
| 449 |
+
if member != table and member not in self.eligible:
|
| 450 |
+
continue
|
| 451 |
+
_ensure_table(self.con, member)
|
| 452 |
+
rows = _land(self.con, member, result.rows.get(member, []), result.report_id, start, end,
|
| 453 |
+
self.profile, self.cid, self.label)
|
| 454 |
+
self.covered.setdefault(member, []).append((start, end))
|
| 455 |
+
self.notes[member] = list(dict.fromkeys(self.notes.get(member, []) + list(result.notes)))
|
| 456 |
+
self.checkpoint(member)
|
| 457 |
+
if member == table:
|
| 458 |
+
own = rows
|
| 459 |
+
else:
|
| 460 |
+
self.pulled[member] = self.pulled.get(member, 0) + rows
|
| 461 |
+
for note in result.notes:
|
| 462 |
+
if f"{self.label}: {note}" not in self.report["problems"]:
|
| 463 |
+
self.report["problems"].append(f"{self.label}: {note}")
|
| 464 |
+
return own
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
def _resume(p):
|
| 468 |
+
"""Read every report this connection left preparing before it creates another. -> {table: outcome}."""
|
| 469 |
+
resumed = {}
|
| 470 |
+
for table in TABLES:
|
| 471 |
+
pending = _read_pending(p.con, table, p.cid)
|
| 472 |
+
if pending is None:
|
| 473 |
+
continue
|
| 474 |
+
spec = schema.TABLE_SPECS[table]
|
| 475 |
+
_ensure_table(p.con, table)
|
| 476 |
+
p.checkpoint(table)
|
| 477 |
+
expired = f"Amazon Ads report {pending['report_id']} did not finish in 3 hours"
|
| 478 |
+
age = (p.stamp() - pending["created"]).total_seconds()
|
| 479 |
+
if age >= client.REPORT_MAX_WAIT_SECONDS:
|
| 480 |
+
_clear_pending(p.con, table, p.cid)
|
| 481 |
+
resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
|
| 482 |
+
continue
|
| 483 |
+
limit = client.REPORT_MAX_WAIT_SECONDS - age
|
| 484 |
+
if p.deadline is not None:
|
| 485 |
+
limit = min(limit, max(0.0, p.deadline - _clock()))
|
| 486 |
+
try:
|
| 487 |
+
result = _run_report(p.key, table, pending["start"], pending["end"], p.rt, p.profile, p.cid,
|
| 488 |
+
max_wait=limit, report_id=pending["report_id"], now=_clock)
|
| 489 |
+
except client.AdsReportPending:
|
| 490 |
+
if (p.stamp() - pending["created"]).total_seconds() < client.REPORT_MAX_WAIT_SECONDS:
|
| 491 |
+
p.blocked = True
|
| 492 |
+
p.wait(spec, PREPARING_REASON)
|
| 493 |
+
resumed[table] = ("deferred", PREPARING_REASON, 0)
|
| 494 |
+
continue
|
| 495 |
+
_clear_pending(p.con, table, p.cid)
|
| 496 |
+
resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
|
| 497 |
+
continue
|
| 498 |
+
except client.AdsError as exc:
|
| 499 |
+
_clear_pending(p.con, table, p.cid)
|
| 500 |
+
resumed[table] = (*p.fail(table, exc), 0)
|
| 501 |
+
continue
|
| 502 |
+
_clear_pending(p.con, table, p.cid)
|
| 503 |
+
resumed[table] = ("done", "", p.land(table, result, pending["start"], pending["end"]))
|
| 504 |
+
return resumed
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
def _run_windows(p, table, connect_day):
|
| 508 |
+
"""Run the group report for each window of `table` no report covered yet. -> (state, reason, rows)."""
|
| 509 |
+
spec = schema.TABLE_SPECS[table]
|
| 510 |
+
failed = p.outcome.get(schema.group_of(table).name)
|
| 511 |
+
if failed is not None:
|
| 512 |
+
return (*p.failure(table, failed), 0)
|
| 513 |
+
pulled = 0
|
| 514 |
+
for start, end in windows(p.con, table, connect_day, p.today, p.cid):
|
| 515 |
+
if p.covers(table, start, end):
|
| 516 |
+
continue
|
| 517 |
+
if p.used >= client.REPORTS_PER_DAY:
|
| 518 |
+
reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
|
| 519 |
+
f"reports are used, so days from {start.isoformat()} load on the next sync.")
|
| 520 |
+
p.report["problems"].append(f"{p.label}: {spec['label']}: waiting for tomorrow's report limit")
|
| 521 |
+
p.report["waits"].append(p.report["problems"][-1])
|
| 522 |
+
return "deferred", reason, pulled
|
| 523 |
+
if p.deadline is not None and _clock() >= p.deadline:
|
| 524 |
+
p.wait(spec, TIME_LIMIT_REASON)
|
| 525 |
+
return "deferred", TIME_LIMIT_REASON, pulled
|
| 526 |
+
p.used += 1
|
| 527 |
+
_spend_budget(p.con, p.today, p.used, p.cid)
|
| 528 |
+
created = p.stamp()
|
| 529 |
+
try:
|
| 530 |
+
result = _run_report(p.key, table, start, end, p.rt, p.profile, p.cid, **p.bounds())
|
| 531 |
+
except client.AdsReportPending as exc:
|
| 532 |
+
if (p.deadline is None or not exc.report_id
|
| 533 |
+
or (p.stamp() - created).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS):
|
| 534 |
+
return (*p.failure(table, exc), pulled)
|
| 535 |
+
_write_pending(p.con, table, p.cid, exc.report_id, start, end, created)
|
| 536 |
+
p.checkpoint(table)
|
| 537 |
+
p.blocked = True
|
| 538 |
+
p.wait(spec, PREPARING_REASON)
|
| 539 |
+
return "deferred", PREPARING_REASON, pulled
|
| 540 |
+
except client.AdsError as exc:
|
| 541 |
+
return (*p.fail(table, exc), pulled)
|
| 542 |
+
pulled += p.land(table, result, start, end)
|
| 543 |
+
return "done", "", pulled
|
| 544 |
|
| 545 |
|
| 546 |
def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
|
|
|
|
| 551 |
through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
|
| 552 |
`max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
|
| 553 |
pending ledger, and the next pass reads it before it creates any other report.
|
| 554 |
+
One group report fills every requested member table, so a sibling's covered days spend no report.
|
| 555 |
"""
|
| 556 |
key = amazon_store._tenant_key(tenant_key)
|
| 557 |
report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
|
|
|
|
| 568 |
own = con is None
|
| 569 |
if own:
|
| 570 |
con, _path = amazon_store._connect_tenant_mirror(key, rt)
|
|
|
|
| 571 |
started = _clock()
|
| 572 |
+
deadline = None if max_seconds is None else started + float(max_seconds)
|
| 573 |
+
synced, checkpointed = [], []
|
| 574 |
try:
|
| 575 |
datastore._ensure_sync_state(con)
|
| 576 |
now = _utc_now(now)
|
|
|
|
| 579 |
def stamp():
|
| 580 |
return now + timedelta(seconds=max(0.0, _clock() - started))
|
| 581 |
|
| 582 |
+
requested = [schema.short_name(t) for t in (tables or TABLES)]
|
| 583 |
+
for connection in connections:
|
| 584 |
+
cid, label, _profile, legacy = connection
|
|
|
|
|
|
|
|
|
|
| 585 |
connect_day = _connect_day(con, today, cid, legacy=legacy)
|
| 586 |
report.setdefault("connectDay", connect_day.isoformat())
|
| 587 |
entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
|
| 588 |
report["connections"].append(entry)
|
| 589 |
+
p = _Pass(con, key, rt, connection, report, today=today, used=_budget(con, today, cid, legacy=legacy),
|
| 590 |
+
deadline=deadline, stamp=stamp, checkpointed=checkpointed)
|
| 591 |
+
p.eligible = {t for t in requested if schema.TABLE_SPECS[t]["offered"] and _cadence_due(con, t, now, cid)}
|
| 592 |
+
resumed = _resume(p)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 593 |
for table in list(dict.fromkeys(requested + list(resumed))):
|
| 594 |
+
landed = _sync_table(p, table, entry, connect_day, now, legacy, table in requested,
|
| 595 |
+
resumed.get(table), log)
|
| 596 |
+
p.eligible.discard(table)
|
| 597 |
+
if landed and table_name(table) not in synced:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 598 |
synced.append(table_name(table))
|
| 599 |
+
_record_tables(con, report, connections)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 600 |
if own and rt is not None and (synced or checkpointed):
|
| 601 |
_persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
|
| 602 |
finally:
|
|
|
|
| 606 |
return report
|
| 607 |
|
| 608 |
|
| 609 |
+
_SEVERITY = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
|
| 610 |
+
|
| 611 |
+
|
| 612 |
+
def _sync_table(p, table, entry, connect_day, now, legacy, requested, prior, log):
|
| 613 |
+
"""One table's turn in one connection's pass: run or skip it, record its state. -> True if it changed."""
|
| 614 |
+
con, cid, label, report = p.con, p.cid, p.label, p.report
|
| 615 |
+
spec = schema.TABLE_SPECS[table]
|
| 616 |
+
_ensure_table(con, table)
|
| 617 |
+
origin_changed = _refresh_origin(con, table, cid, label, legacy)
|
| 618 |
+
state, reason, pulled = prior or ("done", "", 0)
|
| 619 |
+
pulled += p.pulled.pop(table, 0)
|
| 620 |
+
if state != "done" or not requested:
|
| 621 |
+
pass
|
| 622 |
+
elif not spec["offered"]:
|
| 623 |
+
state, reason = "not_offered", spec["not_offered_reason"]
|
| 624 |
+
elif not _cadence_due(con, table, now, cid):
|
| 625 |
+
state = "skipped"
|
| 626 |
+
elif p.blocked:
|
| 627 |
+
state, reason = "deferred", PREPARING_REASON
|
| 628 |
+
p.wait(spec, reason)
|
| 629 |
+
else:
|
| 630 |
+
state, reason, rows = _run_windows(p, table, connect_day)
|
| 631 |
+
pulled += rows
|
| 632 |
+
if state == "done":
|
| 633 |
+
_record_cadence(con, table, now, cid)
|
| 634 |
+
if state == "done" and not reason and p.notes.get(table):
|
| 635 |
+
reason = " ".join(p.notes[table])
|
| 636 |
+
if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
|
| 637 |
+
report["partial"] = True
|
| 638 |
+
if reason:
|
| 639 |
+
_record_limit(con, table, reason, cid, state=state)
|
| 640 |
+
shown = f"{label}: {reason}" if cid else reason
|
| 641 |
+
report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
|
| 642 |
+
elif state == "done":
|
| 643 |
+
_clear_limit(con, table, cid)
|
| 644 |
+
total = _count(con, table)
|
| 645 |
+
entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
|
| 646 |
+
if state != "skipped":
|
| 647 |
+
amazon_store._record_state(con, _state_key(amazon_store.entity_key(_marker_table(table)), cid),
|
| 648 |
+
"done" if state == "done" else "partial", _count(con, table, cid), state)
|
| 649 |
+
previous = report["tables"].get(table, {})
|
| 650 |
+
report["tables"][table] = {"state": max((state, previous.get("state", state)), key=_SEVERITY.get),
|
| 651 |
+
"pulled": previous.get("pulled", 0) + pulled, "in_mirror": total}
|
| 652 |
+
log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
|
| 653 |
+
return bool(pulled or origin_changed)
|
| 654 |
+
|
| 655 |
+
|
| 656 |
+
def _record_tables(con, report, connections):
|
| 657 |
+
"""Write each table's aggregate state and limit across connections, and the pass's pending count."""
|
| 658 |
+
for table, state in report["tables"].items():
|
| 659 |
+
if state["state"] == "skipped":
|
| 660 |
+
continue
|
| 661 |
+
amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
|
| 662 |
+
"done" if state["state"] == "done" else "partial",
|
| 663 |
+
state["in_mirror"], state["state"])
|
| 664 |
+
if table in report["limits"]:
|
| 665 |
+
_record_limit(con, table, report["limits"][table], state=state["state"])
|
| 666 |
+
else:
|
| 667 |
+
_clear_limit(con, table)
|
| 668 |
+
report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
|
| 669 |
+
for cid, *_ in connections for table in TABLES)
|
| 670 |
+
if report["partial"]:
|
| 671 |
+
report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
|
| 672 |
+
|
| 673 |
+
|
| 674 |
def _not_offered_reason(spec, exc):
|
| 675 |
return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
|
| 676 |
+
f"({str(exc)[:client.ERROR_TEXT_LIMIT]}). The table keeps the days it has. To ask for it, "
|
| 677 |
+
f"contact Amazon Ads support and quote Amazon's message.")
|
| 678 |
|
| 679 |
|
| 680 |
def _persist(tenant_key, rt, con, synced, report, log):
|
platform/harness/amazon_client.py
CHANGED
|
@@ -1127,7 +1127,7 @@ for _tables, _passes, _daily, _intraday in (
|
|
| 1127 |
from harness import amazon_ads_schema as _ads_schema
|
| 1128 |
|
| 1129 |
CADENCE.update({
|
| 1130 |
-
"ads_" + table: {"perDay": 4, "lookbackDays":
|
| 1131 |
for table in _ads_schema.TABLE_SPECS
|
| 1132 |
})
|
| 1133 |
del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
|
|
|
|
| 1127 |
from harness import amazon_ads_schema as _ads_schema
|
| 1128 |
|
| 1129 |
CADENCE.update({
|
| 1130 |
+
"ads_" + table: {"perDay": 4, "lookbackDays": 60, "intradayLookbackDays": 5, "period": "day"}
|
| 1131 |
for table in _ads_schema.TABLE_SPECS
|
| 1132 |
})
|
| 1133 |
del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
|
platform/harness/canonical_source.py
CHANGED
|
@@ -6,6 +6,7 @@ import logging
|
|
| 6 |
import re
|
| 7 |
import uuid
|
| 8 |
from contextlib import contextmanager
|
|
|
|
| 9 |
|
| 10 |
from core import odoo as _odoo
|
| 11 |
|
|
@@ -53,6 +54,17 @@ class Rows:
|
|
| 53 |
return bool(len(self))
|
| 54 |
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
class Source:
|
| 57 |
"""Own bounded source operands and release every temporary snapshot on exit."""
|
| 58 |
|
|
@@ -119,6 +131,30 @@ class Source:
|
|
| 119 |
['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
|
| 120 |
return self.cache[key]
|
| 121 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
def sku_revenue(self, groups):
|
| 123 |
products = self.products()
|
| 124 |
code = code_sql('p')
|
|
@@ -171,7 +207,7 @@ class Source:
|
|
| 171 |
|
| 172 |
def categories(self, keep_blank_code=False):
|
| 173 |
rows = self.products()
|
| 174 |
-
cats = self.
|
| 175 |
blank = '' if keep_blank_code else "AND json_extract_string(p.body, '/default_code') != ''"
|
| 176 |
return {code: category for code, category in self.cur.execute(f"""SELECT
|
| 177 |
{code_sql('p')} code,
|
|
@@ -206,7 +242,7 @@ class Source:
|
|
| 206 |
['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
|
| 207 |
products = self.search_read('product.product', [('default_code', '!=', False)],
|
| 208 |
['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
|
| 209 |
-
cats = self.
|
| 210 |
query = f"""WITH raw AS (SELECT p.ordinal,
|
| 211 |
trim(json_extract_string(p.body, '/default_code')) sku,
|
| 212 |
json_extract_string(p.body, '/name') AS product_name,
|
|
@@ -231,17 +267,18 @@ class Source:
|
|
| 231 |
def tiers(self, prods):
|
| 232 |
from core import periods
|
| 233 |
from modules import products
|
| 234 |
-
|
| 235 |
today = periods.today().isoformat()
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
'|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)],
|
| 240 |
['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
|
| 241 |
-
'fixed_price', 'packaging_id', 'packaging_price'])
|
|
|
|
|
|
|
|
|
|
| 242 |
normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
|
| 243 |
canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
|
| 244 |
-
prods = products.active_products() if prods is None else prods
|
| 245 |
declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
|
| 246 |
# Two equality joins and an anti-join: an OR join with a correlated NOT EXISTS is a nested loop
|
| 247 |
# (234 s for the whole catalogue on the staging mirror).
|
|
@@ -250,7 +287,7 @@ class Source:
|
|
| 250 |
json_extract_string(p.body, '/product_tmpl_id/0') tmpl
|
| 251 |
FROM "{prods.table}" p),
|
| 252 |
rr AS (
|
| 253 |
-
SELECT r.
|
| 254 |
json_extract_string(r.body, '/product_id/0') rpid,
|
| 255 |
json_extract_string(r.body, '/product_tmpl_id/0') rtmpl,
|
| 256 |
json_extract_string(r.body, '/pricelist_id/0') plid,
|
|
@@ -289,10 +326,11 @@ class Source:
|
|
| 289 |
entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
|
| 290 |
if name in declared and n > products.PRICELIST_TIER_DEPTH:
|
| 291 |
entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
|
| 292 |
-
return out,
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
|
|
|
| 296 |
|
| 297 |
def search_read(self, model, domain=None, fields=None, **kw):
|
| 298 |
domain = self.domain(model, domain)
|
|
@@ -615,6 +653,9 @@ class _Query:
|
|
| 615 |
expr = expr[0]
|
| 616 |
if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
|
| 617 |
raise _Unmirrored(f'name search on {path}')
|
|
|
|
|
|
|
|
|
|
| 618 |
if kind == 'bool':
|
| 619 |
expr = f'coalesce({expr}, false)'
|
| 620 |
value = [_plain(v) for v in value] if isinstance(value, (list, tuple, set)) else _plain(value)
|
|
@@ -721,6 +762,11 @@ class MirrorSource(Source):
|
|
| 721 |
"""The FROM operand for `model`: its mirror table, or an empty typed one while unsynced."""
|
| 722 |
if model not in self._relations:
|
| 723 |
target = _MIRROR_MODELS[model]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 724 |
if not target.startswith('(') and not self.cur.execute(
|
| 725 |
'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1',
|
| 726 |
[target]).fetchone():
|
|
@@ -750,6 +796,24 @@ class MirrorSource(Source):
|
|
| 750 |
# Live `_code_category` keys every `default_code != False` product, a blank '' code included.
|
| 751 |
return Source.categories(self, keep_blank_code)
|
| 752 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 753 |
def servable(self, model, names):
|
| 754 |
"""The `names` the mirror declares for `model`; the rest are logged once and dropped."""
|
| 755 |
kept, dropped = [], []
|
|
@@ -922,9 +986,10 @@ class MirrorSource(Source):
|
|
| 922 |
coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
|
| 923 |
FROM {prods.from_sql()} WHERE {pw}),
|
| 924 |
quant AS (SELECT b.product_id id, sum(coalesce(b.quantity, 0)) qty
|
| 925 |
-
FROM {quants.from_sql()} WHERE {qw} GROUP BY 1),
|
| 926 |
sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
|
| 927 |
-
sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw}
|
|
|
|
| 928 |
SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
|
| 929 |
bool_or(p.storable) storable, sum(coalesce(q.qty, 0)) on_hand,
|
| 930 |
sum(coalesce(q.qty, 0) * p.cost) inv_value, sum(coalesce(s.qty, 0) * p.cost) cogs_ltm,
|
|
@@ -962,6 +1027,7 @@ class MirrorSource(Source):
|
|
| 962 |
"SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
|
| 963 |
"JOIN stock_location l ON l.id = q.location_id "
|
| 964 |
"WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
|
|
|
|
| 965 |
"GROUP BY 1) f ON f.product_id = p.id "
|
| 966 |
"WHERE p.id IN (SELECT unnest($2::BIGINT[]))",
|
| 967 |
[view[1], ids])
|
|
|
|
| 6 |
import re
|
| 7 |
import uuid
|
| 8 |
from contextlib import contextmanager
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
|
| 11 |
from core import odoo as _odoo
|
| 12 |
|
|
|
|
| 54 |
return bool(len(self))
|
| 55 |
|
| 56 |
|
| 57 |
+
@dataclass(frozen=True)
|
| 58 |
+
class RowIds:
|
| 59 |
+
"""A SQL-backed selection of related identities from one source snapshot."""
|
| 60 |
+
rows: Rows
|
| 61 |
+
path: str
|
| 62 |
+
|
| 63 |
+
def sql(self):
|
| 64 |
+
return (f'SELECT DISTINCT try_cast(json_extract_string(body, \'{self.path}\') AS BIGINT) id '
|
| 65 |
+
f'FROM "{self.rows.table}" WHERE id IS NOT NULL')
|
| 66 |
+
|
| 67 |
+
|
| 68 |
class Source:
|
| 69 |
"""Own bounded source operands and release every temporary snapshot on exit."""
|
| 70 |
|
|
|
|
| 131 |
['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
|
| 132 |
return self.cache[key]
|
| 133 |
|
| 134 |
+
def related_read(self, model, selections, fields, domain=()):
|
| 135 |
+
"""Page related source reads by selected identities without retaining the population in Python."""
|
| 136 |
+
def records():
|
| 137 |
+
for field, ids, extra in selections:
|
| 138 |
+
last = -1
|
| 139 |
+
while True:
|
| 140 |
+
page = [row[0] for row in self.cur.execute(
|
| 141 |
+
f'SELECT id FROM ({ids.sql()}) selected WHERE id > ? ORDER BY id LIMIT ?',
|
| 142 |
+
[last, BATCH]).fetchall()]
|
| 143 |
+
if not page:
|
| 144 |
+
break
|
| 145 |
+
selected = self.search_read(model, [*domain, *extra, (field, 'in', page)], fields)
|
| 146 |
+
try:
|
| 147 |
+
yield from selected
|
| 148 |
+
finally:
|
| 149 |
+
self.cur.execute(f'DROP TABLE "{selected.table}"')
|
| 150 |
+
self.tables.remove(selected.table)
|
| 151 |
+
last = page[-1]
|
| 152 |
+
return self.snapshot(records())
|
| 153 |
+
|
| 154 |
+
def category_rows(self, products):
|
| 155 |
+
return self.related_read('product.category', [('id', RowIds(products, '/categ_id/0'), [])],
|
| 156 |
+
['id', 'complete_name'])
|
| 157 |
+
|
| 158 |
def sku_revenue(self, groups):
|
| 159 |
products = self.products()
|
| 160 |
code = code_sql('p')
|
|
|
|
| 207 |
|
| 208 |
def categories(self, keep_blank_code=False):
|
| 209 |
rows = self.products()
|
| 210 |
+
cats = self.category_rows(rows)
|
| 211 |
blank = '' if keep_blank_code else "AND json_extract_string(p.body, '/default_code') != ''"
|
| 212 |
return {code: category for code, category in self.cur.execute(f"""SELECT
|
| 213 |
{code_sql('p')} code,
|
|
|
|
| 242 |
['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
|
| 243 |
products = self.search_read('product.product', [('default_code', '!=', False)],
|
| 244 |
['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
|
| 245 |
+
cats = self.category_rows(products)
|
| 246 |
query = f"""WITH raw AS (SELECT p.ordinal,
|
| 247 |
trim(json_extract_string(p.body, '/default_code')) sku,
|
| 248 |
json_extract_string(p.body, '/name') AS product_name,
|
|
|
|
| 267 |
def tiers(self, prods):
|
| 268 |
from core import periods
|
| 269 |
from modules import products
|
| 270 |
+
prods = products.active_products() if prods is None else prods
|
| 271 |
today = periods.today().isoformat()
|
| 272 |
+
selections = [('product_id', RowIds(prods, '/id'), [('applied_on', '=', '0_product_variant')]),
|
| 273 |
+
('product_tmpl_id', RowIds(prods, '/product_tmpl_id/0'), [('applied_on', '=', '1_product')])]
|
| 274 |
+
rules = self.related_read('product.pricelist.item', selections,
|
|
|
|
| 275 |
['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
|
| 276 |
+
'fixed_price', 'packaging_id', 'packaging_price'], [('compute_price', '=', 'fixed'),
|
| 277 |
+
'|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 278 |
+
'|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)])
|
| 279 |
+
lists = self.related_read('product.pricelist', [('id', RowIds(rules, '/pricelist_id/0'), [])], ['id', 'name'])
|
| 280 |
normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
|
| 281 |
canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
|
|
|
|
| 282 |
declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
|
| 283 |
# Two equality joins and an anti-join: an OR join with a correlated NOT EXISTS is a nested loop
|
| 284 |
# (234 s for the whole catalogue on the staging mirror).
|
|
|
|
| 287 |
json_extract_string(p.body, '/product_tmpl_id/0') tmpl
|
| 288 |
FROM "{prods.table}" p),
|
| 289 |
rr AS (
|
| 290 |
+
SELECT try_cast(json_extract_string(r.body, '/id') AS BIGINT) rule_order, json_extract_string(r.body, '/applied_on') applied_on,
|
| 291 |
json_extract_string(r.body, '/product_id/0') rpid,
|
| 292 |
json_extract_string(r.body, '/product_tmpl_id/0') rtmpl,
|
| 293 |
json_extract_string(r.body, '/pricelist_id/0') plid,
|
|
|
|
| 326 |
entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
|
| 327 |
if name in declared and n > products.PRICELIST_TIER_DEPTH:
|
| 328 |
entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
|
| 329 |
+
return out, self.tier_report()
|
| 330 |
+
|
| 331 |
+
def tier_report(self):
|
| 332 |
+
return {'bounded': True, 'rules_total': self.search_count('product.pricelist.item', []),
|
| 333 |
+
'rules_not_fixed': self.search_count('product.pricelist.item', [('compute_price', '!=', 'fixed')])}
|
| 334 |
|
| 335 |
def search_read(self, model, domain=None, fields=None, **kw):
|
| 336 |
domain = self.domain(model, domain)
|
|
|
|
| 653 |
expr = expr[0]
|
| 654 |
if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
|
| 655 |
raise _Unmirrored(f'name search on {path}')
|
| 656 |
+
if op == 'in' and isinstance(value, RowIds):
|
| 657 |
+
cond = f'{expr} IN ({value.sql()})'
|
| 658 |
+
return f'coalesce(({guard} AND {cond}), false)' if guard else f'coalesce(({cond}), false)'
|
| 659 |
if kind == 'bool':
|
| 660 |
expr = f'coalesce({expr}, false)'
|
| 661 |
value = [_plain(v) for v in value] if isinstance(value, (list, tuple, set)) else _plain(value)
|
|
|
|
| 762 |
"""The FROM operand for `model`: its mirror table, or an empty typed one while unsynced."""
|
| 763 |
if model not in self._relations:
|
| 764 |
target = _MIRROR_MODELS[model]
|
| 765 |
+
if model == 'product.category' and self.kind == 'product':
|
| 766 |
+
products = self.products()
|
| 767 |
+
target = ("(SELECT DISTINCT try_cast(json_extract_string(body, '/categ_id/0') AS BIGINT) AS id, "
|
| 768 |
+
"json_extract_string(body, '/categ_id/1') AS complete_name "
|
| 769 |
+
f'FROM "{products.table}" WHERE id IS NOT NULL)')
|
| 770 |
if not target.startswith('(') and not self.cur.execute(
|
| 771 |
'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1',
|
| 772 |
[target]).fetchone():
|
|
|
|
| 796 |
# Live `_code_category` keys every `default_code != False` product, a blank '' code included.
|
| 797 |
return Source.categories(self, keep_blank_code)
|
| 798 |
|
| 799 |
+
def related_read(self, model, selections, fields, domain=()):
|
| 800 |
+
branches = [['&'] * len(extra) + [*extra, (field, 'in', ids)]
|
| 801 |
+
for field, ids, extra in selections]
|
| 802 |
+
scoped = ['|'] * (len(branches) - 1) + [term for branch in branches for term in branch]
|
| 803 |
+
return self.search_read(model, [*domain, *scoped], fields)
|
| 804 |
+
|
| 805 |
+
def tier_report(self):
|
| 806 |
+
owner = self.parent or self
|
| 807 |
+
if 'tier_report' not in owner.cache:
|
| 808 |
+
query = _Query(self, 'product.pricelist.item')
|
| 809 |
+
price = query.field('compute_price')[0]
|
| 810 |
+
where = query.where([])
|
| 811 |
+
total, other = self.cur.execute(
|
| 812 |
+
f"SELECT count(*), count(*) FILTER (WHERE {price} IS DISTINCT FROM 'fixed') "
|
| 813 |
+
f'FROM {query.from_sql()} WHERE {where}', query.params).fetchone()
|
| 814 |
+
owner.cache['tier_report'] = {'bounded': True, 'rules_total': total, 'rules_not_fixed': other}
|
| 815 |
+
return dict(owner.cache['tier_report'])
|
| 816 |
+
|
| 817 |
def servable(self, model, names):
|
| 818 |
"""The `names` the mirror declares for `model`; the rest are logged once and dropped."""
|
| 819 |
kept, dropped = [], []
|
|
|
|
| 986 |
coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
|
| 987 |
FROM {prods.from_sql()} WHERE {pw}),
|
| 988 |
quant AS (SELECT b.product_id id, sum(coalesce(b.quantity, 0)) qty
|
| 989 |
+
FROM {quants.from_sql()} WHERE {qw} AND b.product_id IN (SELECT id FROM prods) GROUP BY 1),
|
| 990 |
sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
|
| 991 |
+
sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw}
|
| 992 |
+
AND b.product_id IN (SELECT id FROM prods) GROUP BY 1)
|
| 993 |
SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
|
| 994 |
bool_or(p.storable) storable, sum(coalesce(q.qty, 0)) on_hand,
|
| 995 |
sum(coalesce(q.qty, 0) * p.cost) inv_value, sum(coalesce(s.qty, 0) * p.cost) cogs_ltm,
|
|
|
|
| 1027 |
"SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
|
| 1028 |
"JOIN stock_location l ON l.id = q.location_id "
|
| 1029 |
"WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
|
| 1030 |
+
"AND q.product_id IN (SELECT unnest($2::BIGINT[])) "
|
| 1031 |
"GROUP BY 1) f ON f.product_id = p.id "
|
| 1032 |
"WHERE p.id IN (SELECT unnest($2::BIGINT[]))",
|
| 1033 |
[view[1], ids])
|
platform/modules/competitor_watch.py
CHANGED
|
@@ -26,6 +26,8 @@ import re
|
|
| 26 |
from pathlib import Path
|
| 27 |
from urllib.parse import unquote_plus
|
| 28 |
|
|
|
|
|
|
|
| 29 |
TRIGGERS = {
|
| 30 |
"new_asin": "New ASIN in top 3",
|
| 31 |
"lost_spot": "Lost spot",
|
|
@@ -438,14 +440,6 @@ def _columns(cur, table):
|
|
| 438 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 439 |
|
| 440 |
|
| 441 |
-
def _weekly_period_sql():
|
| 442 |
-
"""Only finite dates or timestamps with an inclusive seven-day span define a report week."""
|
| 443 |
-
start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
|
| 444 |
-
end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
|
| 445 |
-
return (f"isfinite({start}) AND isfinite({end}) AND "
|
| 446 |
-
f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) = 6")
|
| 447 |
-
|
| 448 |
-
|
| 449 |
def _ba_columns(cur):
|
| 450 |
"""Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
|
| 451 |
cols = _columns(cur, BA_TABLE) if cur is not None else set()
|
|
|
|
| 26 |
from pathlib import Path
|
| 27 |
from urllib.parse import unquote_plus
|
| 28 |
|
| 29 |
+
from core.amazon_rivals import weekly_period_sql as _weekly_period_sql
|
| 30 |
+
|
| 31 |
TRIGGERS = {
|
| 32 |
"new_asin": "New ASIN in top 3",
|
| 33 |
"lost_spot": "Lost spot",
|
|
|
|
| 440 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 441 |
|
| 442 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 443 |
def _ba_columns(cur):
|
| 444 |
"""Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
|
| 445 |
cols = _columns(cur, BA_TABLE) if cur is not None else set()
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
8c86af445a651c7ecbd3085c9d8b5afa5671a13f052ac92abb0ba0d5a7811991
|