Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +148 -23
- api/providers.py +335 -0
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "8ca7046",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
8ca7046
|
api/automation_engine.py
CHANGED
|
@@ -37,6 +37,8 @@ from urllib.parse import urljoin, urlparse
|
|
| 37 |
|
| 38 |
import requests
|
| 39 |
|
|
|
|
|
|
|
| 40 |
try: # the parser the extraction half needs
|
| 41 |
from bs4 import BeautifulSoup
|
| 42 |
except Exception: # pragma: no cover β deploy lag; see mailbox β‘
|
|
@@ -1042,6 +1044,8 @@ IG_FIELD_DESCRIPTIONS = {
|
|
| 1042 |
"avg_engagement": "Average engagement rate reported for the profile.",
|
| 1043 |
"avg_likes_12": "Average likes across the 12 most recent captured posts.",
|
| 1044 |
"avg_plays_12": "Average plays across the 12 most recent captured posts.",
|
|
|
|
|
|
|
| 1045 |
"bio": "Biography shown on the Instagram profile.",
|
| 1046 |
"bio_hashtags": "Hashtags listed in the profile biography.",
|
| 1047 |
"business_category": "Instagram's business category for the account.",
|
|
@@ -2261,6 +2265,14 @@ POST_FIELDS = [
|
|
| 2261 |
# Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named
|
| 2262 |
# and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10
|
| 2263 |
# (folded into a single "Views"), not because we are reading the wrong key.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2264 |
field_def("plays", "Plays", "int"),
|
| 2265 |
field_def("likes", "Likes", "int"),
|
| 2266 |
field_def("comments", "Comments", "int"),
|
|
@@ -2328,9 +2340,9 @@ POST_SNAPSHOT_FIELDS = [
|
|
| 2328 |
field_def("comments", "Comments", "int"),
|
| 2329 |
# Plays move independently of likes on video, so it is its own series rather than a thing to
|
| 2330 |
# derive. Paid rung only; blank means not read.
|
| 2331 |
-
#
|
| 2332 |
-
#
|
| 2333 |
-
|
| 2334 |
field_def("plays", "Plays", "int"),
|
| 2335 |
field_def("source_payload", "Source data", "json"),
|
| 2336 |
field_def("post_link", "Post", "link",
|
|
@@ -3137,6 +3149,79 @@ def bd_scrape(dataset_id, urls, wait=None, deferred=None):
|
|
| 3137 |
f"or collect {sid} from the control panel")
|
| 3138 |
|
| 3139 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3140 |
#: ββ 2026-08-07 β HOW LONG THE CORPUS RUNG MAY WAIT. A corpus query is a scan over 620M rows
|
| 3141 |
#: and takes MINUTES, not seconds (Β§2d MEASURED ~4 min for a narrow one). It is a FALLBACK behind
|
| 3142 |
#: a scrape that already failed, so the budget is generous β but bounded, because an enrich step
|
|
@@ -3354,6 +3439,50 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False,
|
|
| 3354 |
elif metric_notes:
|
| 3355 |
note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
|
| 3356 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3357 |
comments = []
|
| 3358 |
if comment_metrics and posts:
|
| 3359 |
# This is the cost boundary: do not make the full comment request unless the stored
|
|
@@ -3873,12 +4002,13 @@ PRESET_PROFILE_FIELDS = [
|
|
| 3873 |
# (`MAX_POSTS_PER_PULL`), so a window wider than that would silently average fewer posts than its own
|
| 3874 |
# label claims. `sortBy` is mandatory alongside a `limit`, and this is why β "the last 12"
|
| 3875 |
# has to name what makes one post later than another.
|
| 3876 |
-
#
|
| 3877 |
-
#
|
| 3878 |
-
#
|
| 3879 |
-
|
| 3880 |
-
|
| 3881 |
-
|
|
|
|
| 3882 |
field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup",
|
| 3883 |
rollup={"link": "posts_link", "field": "plays", "fn": "average",
|
| 3884 |
"limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
|
|
@@ -4021,14 +4151,11 @@ def _ig_schema_contract(rt, profile_tables=None):
|
|
| 4021 |
else _ig_profile_tables(rt)))
|
| 4022 |
backlinks, wanted, drops = [], {}, {
|
| 4023 |
IG_SNAPSHOTS_TABLE: {"post_hashtags"},
|
| 4024 |
-
#
|
| 4025 |
-
#
|
| 4026 |
-
#
|
| 4027 |
-
#
|
| 4028 |
-
#
|
| 4029 |
-
# deleting the `field_def` alone would only have changed tables created after today.
|
| 4030 |
-
IG_POSTS_TABLE: {"views"},
|
| 4031 |
-
IG_POST_SNAPSHOTS_TABLE: {"views"},
|
| 4032 |
}
|
| 4033 |
for table_key in profile_keys:
|
| 4034 |
table = tables.get(table_key) or {}
|
|
@@ -4040,9 +4167,7 @@ def _ig_schema_contract(rt, profile_tables=None):
|
|
| 4040 |
wanted[table_key] = _profile_schema_for(bound)
|
| 4041 |
# `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
|
| 4042 |
# to disagree, so it is retired by key just like the old nested Posts JSON.
|
| 4043 |
-
|
| 4044 |
-
# survive on every profile table already in production and render as a permanent blank.
|
| 4045 |
-
drops[table_key] = {"posts", "post_hashtags", "tracked", "avg_views_12"}
|
| 4046 |
wanted.update({
|
| 4047 |
IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
|
| 4048 |
IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
|
|
@@ -5411,7 +5536,7 @@ def capture_rows(res, pulled):
|
|
| 5411 |
ident[k] = _s(p.get(k), n)
|
| 5412 |
if p.get("source_payload"):
|
| 5413 |
ident["source_payload"] = str(p["source_payload"])
|
| 5414 |
-
metrics = {k: p.get(k) for k in ("likes", "comments", "plays")}
|
| 5415 |
# ββ 2026-08-07 β THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
|
| 5416 |
#
|
| 5417 |
# This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
|
|
@@ -5442,7 +5567,7 @@ def capture_rows(res, pulled):
|
|
| 5442 |
"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5443 |
"pulled_at": pulled,
|
| 5444 |
"likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
|
| 5445 |
-
"plays": _s(metrics["plays"]),
|
| 5446 |
"source_payload": str(p.get("source_payload") or "")})
|
| 5447 |
for comment in p.get("embedded_comments") or []:
|
| 5448 |
# The embedded shape often names only the commenter. Its parent Post is the
|
|
@@ -5497,7 +5622,7 @@ PRESET_FROM_PROFILE = {
|
|
| 5497 |
PRESET_WRITTEN_ELSEWHERE = (
|
| 5498 |
"platform", "handle", "enriched_at",
|
| 5499 |
"posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link",
|
| 5500 |
-
"avg_plays_12", "avg_likes_12", "avg_comments_12",
|
| 5501 |
"posts_captured", "profile_reads", "post_measurements_captured", "comments_captured",
|
| 5502 |
)
|
| 5503 |
|
|
|
|
| 37 |
|
| 38 |
import requests
|
| 39 |
|
| 40 |
+
import providers
|
| 41 |
+
|
| 42 |
try: # the parser the extraction half needs
|
| 43 |
from bs4 import BeautifulSoup
|
| 44 |
except Exception: # pragma: no cover β deploy lag; see mailbox β‘
|
|
|
|
| 1044 |
"avg_engagement": "Average engagement rate reported for the profile.",
|
| 1045 |
"avg_likes_12": "Average likes across the 12 most recent captured posts.",
|
| 1046 |
"avg_plays_12": "Average plays across the 12 most recent captured posts.",
|
| 1047 |
+
"avg_views_12": "Average views across the 12 most recent captured posts.",
|
| 1048 |
+
"views": "View count for the post, as Instagram displays it.",
|
| 1049 |
"bio": "Biography shown on the Instagram profile.",
|
| 1050 |
"bio_hashtags": "Hashtags listed in the profile biography.",
|
| 1051 |
"business_category": "Instagram's business category for the account.",
|
|
|
|
| 2265 |
# Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named
|
| 2266 |
# and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10
|
| 2267 |
# (folded into a single "Views"), not because we are reading the wrong key.
|
| 2268 |
+
# ββ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED.
|
| 2269 |
+
# The retired column was fed by Bright Data's account-grain `views`. This one is fed by the
|
| 2270 |
+
# `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` β
|
| 2271 |
+
# measured against a browser-read ground truth to the digit. Same column name, same type, same
|
| 2272 |
+
# rollup; the engine underneath is swappable and the preset database never moved. That is the
|
| 2273 |
+
# owner's schema ruling working exactly as intended.
|
| 2274 |
+
# β STILL BLANK, NEVER ZERO, and still only on the paid rung.
|
| 2275 |
+
field_def("views", "Views", "int"),
|
| 2276 |
field_def("plays", "Plays", "int"),
|
| 2277 |
field_def("likes", "Likes", "int"),
|
| 2278 |
field_def("comments", "Comments", "int"),
|
|
|
|
| 2340 |
field_def("comments", "Comments", "int"),
|
| 2341 |
# Plays move independently of likes on video, so it is its own series rather than a thing to
|
| 2342 |
# derive. Paid rung only; blank means not read.
|
| 2343 |
+
# The series regains Views alongside the latest-value projection on the Post row β one store
|
| 2344 |
+
# for one series (R3) is untouched; this IS that store.
|
| 2345 |
+
field_def("views", "Views", "int"),
|
| 2346 |
field_def("plays", "Plays", "int"),
|
| 2347 |
field_def("source_payload", "Source data", "json"),
|
| 2348 |
field_def("post_link", "Post", "link",
|
|
|
|
| 3149 |
f"or collect {sid} from the control panel")
|
| 3150 |
|
| 3151 |
|
| 3152 |
+
# =============================================================================================
|
| 3153 |
+
# APIFY β THE SECOND PROVIDER (owner ruling 2026-08-08). Registered in `providers.py`; this is
|
| 3154 |
+
# only the wire.
|
| 3155 |
+
#
|
| 3156 |
+
# β IT EXISTS FOR EXACTLY ONE CAPABILITY TODAY: `ig_post_views`. Bright Data answers profile,
|
| 3157 |
+
# likes and comments correctly and cannot answer a per-reel view count on ANY route; Apify's
|
| 3158 |
+
# `videoPlayCount` matched a browser-read ground truth to the digit (137,684 vs ~137K and 299,493
|
| 3159 |
+
# vs 299K). So this runs as a TOP-UP over Bright Data's rows, never as a replacement for them β
|
| 3160 |
+
# re-buying the profile and the likes from a second vendor is the expensive mistake the capability
|
| 3161 |
+
# split exists to prevent.
|
| 3162 |
+
# β `videoPlayCount`, NEVER `videoViewCount`. Apify ships BOTH, and the wrong one (10,678) agrees
|
| 3163 |
+
# with Bright Data's useless `views` (10,638) for the same reel whose true count is ~137K. Two
|
| 3164 |
+
# vendors' junk fields agreeing is what makes picking by name so dangerous here.
|
| 3165 |
+
# =============================================================================================
|
| 3166 |
+
|
| 3167 |
+
APIFY_BASE = "https://api.apify.com/v2"
|
| 3168 |
+
#: The general Instagram scraper. `directUrls` accepts /p/ and /reel/ permalinks alike.
|
| 3169 |
+
APIFY_ACTOR_POSTS = os.environ.get("AIOS_APIFY_ACTOR") or "apify~instagram-scraper"
|
| 3170 |
+
#: Seconds to let a synchronous actor run before giving up. Apify bills the run either way, so a
|
| 3171 |
+
#: timeout here is money already spent β generous on purpose.
|
| 3172 |
+
APIFY_WAIT = float(os.environ.get("AIOS_APIFY_WAIT") or 240)
|
| 3173 |
+
|
| 3174 |
+
|
| 3175 |
+
def apify_key():
|
| 3176 |
+
"""The key, or ''. Read fresh every call, exactly as `bd_key` is."""
|
| 3177 |
+
return (os.environ.get("AIOS_APIFY_KEY") or "").strip()
|
| 3178 |
+
|
| 3179 |
+
|
| 3180 |
+
def apify_ready():
|
| 3181 |
+
return bool(apify_key())
|
| 3182 |
+
|
| 3183 |
+
|
| 3184 |
+
def apify_posts(urls):
|
| 3185 |
+
"""Post/reel permalinks -> (canonical rows, note). A note means it did NOT answer.
|
| 3186 |
+
|
| 3187 |
+
Returns rows already normalised through `providers.normalize_post_apify`, so nothing
|
| 3188 |
+
downstream ever sees an Apify-shaped key. That is the owner's schema ruling enforced at the
|
| 3189 |
+
only place it can be: the boundary.
|
| 3190 |
+
"""
|
| 3191 |
+
want = [str(u) for u in (urls or []) if u]
|
| 3192 |
+
if not want:
|
| 3193 |
+
return [], ""
|
| 3194 |
+
key = apify_key()
|
| 3195 |
+
if not key:
|
| 3196 |
+
return [], "AIOS_APIFY_KEY is not configured β the Apify rung is closed"
|
| 3197 |
+
url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items"
|
| 3198 |
+
f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}")
|
| 3199 |
+
body = {"directUrls": want, "resultsType": "posts", "resultsLimit": max(1, len(want)),
|
| 3200 |
+
"addParentData": False}
|
| 3201 |
+
try:
|
| 3202 |
+
status, raw = fetch_json(url, body, timeout=APIFY_WAIT + 60, max_kb=BD_MAX_KB)
|
| 3203 |
+
except Refused as e:
|
| 3204 |
+
return [], f"Apify's address was refused by the URL rail: {e}"
|
| 3205 |
+
except Exception as e: # noqa: BLE001
|
| 3206 |
+
# β TYPE NAME ONLY β the token is in the query string of `url`, so a str(e) that echoed
|
| 3207 |
+
# the request would put a live credential in the run log.
|
| 3208 |
+
return [], f"Apify did not answer ({type(e).__name__})"
|
| 3209 |
+
if status == 402:
|
| 3210 |
+
return [], "the Apify account is out of credit"
|
| 3211 |
+
if status in (401, 403):
|
| 3212 |
+
return [], f"Apify refused our key ({status})"
|
| 3213 |
+
if status >= 400:
|
| 3214 |
+
return [], f"Apify answered {status}"
|
| 3215 |
+
try:
|
| 3216 |
+
items = json.loads(raw.decode("utf-8", "replace"))
|
| 3217 |
+
except Exception: # noqa: BLE001
|
| 3218 |
+
return [], "Apify's answer was not readable JSON"
|
| 3219 |
+
if not isinstance(items, list):
|
| 3220 |
+
return [], "Apify answered with something other than a list of items"
|
| 3221 |
+
rows = [providers.normalize_post_apify(x) for x in items]
|
| 3222 |
+
return [r for r in rows if r], ""
|
| 3223 |
+
|
| 3224 |
+
|
| 3225 |
#: ββ 2026-08-07 β HOW LONG THE CORPUS RUNG MAY WAIT. A corpus query is a scan over 620M rows
|
| 3226 |
#: and takes MINUTES, not seconds (Β§2d MEASURED ~4 min for a narrow one). It is a FALLBACK behind
|
| 3227 |
#: a scrape that already failed, so the budget is generous β but bounded, because an enrich step
|
|
|
|
| 3439 |
elif metric_notes:
|
| 3440 |
note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
|
| 3441 |
|
| 3442 |
+
# ββ THE VIEWS TOP-UP β capability routing, not provider failover (owner ruling
|
| 3443 |
+
# 2026-08-08, `providers.py`). Bright Data has already answered profile + likes + comments
|
| 3444 |
+
# above and is declared INCAPABLE of `ig_post_views`, so the chain for that ONE capability
|
| 3445 |
+
# resolves to Apify and only the video permalinks are re-bought. Re-running the whole
|
| 3446 |
+
# profile on the second vendor would pay twice for the 90% that already worked, which at
|
| 3447 |
+
# thousands of tenants x twelve posts is the entire bill.
|
| 3448 |
+
# β VIDEO ONLY. A carousel or image has no view count, so sending it would buy a record
|
| 3449 |
+
# that can only come back blank.
|
| 3450 |
+
video_urls = [p["url"] for p in posts
|
| 3451 |
+
if p.get("type") == "video" and p.get("url")]
|
| 3452 |
+
if video_urls:
|
| 3453 |
+
def _views_work(prov, _urls=tuple(video_urls)):
|
| 3454 |
+
if prov.key == "apify":
|
| 3455 |
+
return apify_posts(list(_urls))
|
| 3456 |
+
# A provider in the chain with no runner here is a configuration error, not a
|
| 3457 |
+
# vendor outage β say so rather than returning an empty list that reads as "the
|
| 3458 |
+
# vendor had nothing".
|
| 3459 |
+
return None, f"no {prov.key} runner is wired for ig_post_views"
|
| 3460 |
+
|
| 3461 |
+
got_views, attempts = providers.run(
|
| 3462 |
+
"ig_post_views", _views_work,
|
| 3463 |
+
# β THE SATISFIED PREDICATE IS THE FALLBACK TRIGGER. Rows that come back with
|
| 3464 |
+
# every `views` blank are a FAILURE for this capability even at HTTP 200 β which
|
| 3465 |
+
# is exactly how Bright Data behaves, and why a chain that only caught exceptions
|
| 3466 |
+
# would never have reached a second provider.
|
| 3467 |
+
satisfied=lambda rows: bool(rows) and any(r.get("views") for r in rows),
|
| 3468 |
+
log=log)
|
| 3469 |
+
for row in (got_views or []):
|
| 3470 |
+
target = next((p for p in posts
|
| 3471 |
+
if p.get("shortcode") == row.get("shortcode")), None)
|
| 3472 |
+
# β TAKE ONLY THE CAPABILITY THAT WAS ASKED FOR. This provider also returns
|
| 3473 |
+
# likes/comments/caption, and letting them land would silently switch the source
|
| 3474 |
+
# of columns Bright Data already answered β the schema would be stable but the
|
| 3475 |
+
# PROVENANCE would flip halfway through a row.
|
| 3476 |
+
if target and row.get("views") not in (None, ""):
|
| 3477 |
+
target["views"] = row["views"]
|
| 3478 |
+
used = next((a.provider for a in attempts if a.ok), "")
|
| 3479 |
+
if used:
|
| 3480 |
+
note = "; ".join(x for x in (note, f"view counts via {used}") if x)
|
| 3481 |
+
elif attempts:
|
| 3482 |
+
note = "; ".join(x for x in (
|
| 3483 |
+
note, "no provider could supply view counts: "
|
| 3484 |
+
+ "; ".join(f"{a.provider} {a.note}" for a in attempts)) if x)
|
| 3485 |
+
|
| 3486 |
comments = []
|
| 3487 |
if comment_metrics and posts:
|
| 3488 |
# This is the cost boundary: do not make the full comment request unless the stored
|
|
|
|
| 4002 |
# (`MAX_POSTS_PER_PULL`), so a window wider than that would silently average fewer posts than its own
|
| 4003 |
# label claims. `sortBy` is mandatory alongside a `limit`, and this is why β "the last 12"
|
| 4004 |
# has to name what makes one post later than another.
|
| 4005 |
+
# β UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours
|
| 4006 |
+
# because its input was an account-grain constant; with the input now a real per-reel
|
| 4007 |
+
# measurement the average means what its label says again.
|
| 4008 |
+
field_def("avg_views_12", "Avg views Β· last 12 posts", "rollup",
|
| 4009 |
+
rollup={"link": "posts_link", "field": "views", "fn": "average",
|
| 4010 |
+
"limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
|
| 4011 |
+
"distinctBy": "shortcode"}),
|
| 4012 |
field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup",
|
| 4013 |
rollup={"link": "posts_link", "field": "plays", "fn": "average",
|
| 4014 |
"limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
|
|
|
|
| 4151 |
else _ig_profile_tables(rt)))
|
| 4152 |
backlinks, wanted, drops = [], {}, {
|
| 4153 |
IG_SNAPSHOTS_TABLE: {"post_hashtags"},
|
| 4154 |
+
# β `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today
|
| 4155 |
+
# while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS
|
| 4156 |
+
# and fed by the `ig_post_views` capability. Leaving it here would have made the migration
|
| 4157 |
+
# delete, on every authenticated read, the column the same release just added β the
|
| 4158 |
+
# drop set and the field list are two halves of ONE contract and must move together.
|
|
|
|
|
|
|
|
|
|
| 4159 |
}
|
| 4160 |
for table_key in profile_keys:
|
| 4161 |
table = tables.get(table_key) or {}
|
|
|
|
| 4167 |
wanted[table_key] = _profile_schema_for(bound)
|
| 4168 |
# `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
|
| 4169 |
# to disagree, so it is retired by key just like the old nested Posts JSON.
|
| 4170 |
+
drops[table_key] = {"posts", "post_hashtags", "tracked"}
|
|
|
|
|
|
|
| 4171 |
wanted.update({
|
| 4172 |
IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
|
| 4173 |
IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
|
|
|
|
| 5536 |
ident[k] = _s(p.get(k), n)
|
| 5537 |
if p.get("source_payload"):
|
| 5538 |
ident["source_payload"] = str(p["source_payload"])
|
| 5539 |
+
metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
|
| 5540 |
# ββ 2026-08-07 β THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
|
| 5541 |
#
|
| 5542 |
# This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
|
|
|
|
| 5567 |
"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5568 |
"pulled_at": pulled,
|
| 5569 |
"likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
|
| 5570 |
+
"views": _s(metrics["views"]), "plays": _s(metrics["plays"]),
|
| 5571 |
"source_payload": str(p.get("source_payload") or "")})
|
| 5572 |
for comment in p.get("embedded_comments") or []:
|
| 5573 |
# The embedded shape often names only the commenter. Its parent Post is the
|
|
|
|
| 5622 |
PRESET_WRITTEN_ELSEWHERE = (
|
| 5623 |
"platform", "handle", "enriched_at",
|
| 5624 |
"posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link",
|
| 5625 |
+
"avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12",
|
| 5626 |
"posts_captured", "profile_reads", "post_measurements_captured", "comments_captured",
|
| 5627 |
)
|
| 5628 |
|
api/providers.py
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""THE PROVIDER LAYER β many vendors behind one capability, chosen per FIELD and per cost.
|
| 2 |
+
|
| 3 |
+
ββ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if
|
| 4 |
+
anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize
|
| 5 |
+
when we scale to thousands+ customer β¦ make sure our code is customizable and modular."*
|
| 6 |
+
|
| 7 |
+
β THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B".
|
| 8 |
+
A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious
|
| 9 |
+
shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments
|
| 10 |
+
correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A
|
| 11 |
+
provider-level fallback would notice the missing views and re-buy the profile, the likes and the
|
| 12 |
+
comments from Apify as well β **paying twice for the 90% that already worked**. At one profile that
|
| 13 |
+
is noise; at thousands of customers Γ twelve posts each it is the whole bill.
|
| 14 |
+
|
| 15 |
+
So routing is per CAPABILITY, and a capability is the smallest independently-billable thing:
|
| 16 |
+
|
| 17 |
+
ig_profile -> brightdata (36 fields, works)
|
| 18 |
+
ig_post_metrics -> brightdata (likes + comments, works)
|
| 19 |
+
ig_post_views -> apify (Bright Data is INCAPABLE β see `capable=False` below)
|
| 20 |
+
ig_comments -> brightdata
|
| 21 |
+
|
| 22 |
+
Each capability has an ORDERED list of providers. The first one that is configured AND capable AND
|
| 23 |
+
succeeds wins; the rest are never called, so the common path costs exactly one vendor record.
|
| 24 |
+
|
| 25 |
+
β "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in
|
| 26 |
+
the one column you asked for has failed for our purposes, and a chain that only catches exceptions
|
| 27 |
+
would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying
|
| 28 |
+
answer exactly as it would on a 500.
|
| 29 |
+
|
| 30 |
+
β ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time
|
| 31 |
+
(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what
|
| 32 |
+
lets a tenant be moved off a vendor without a release.
|
| 33 |
+
"""
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
+
import json
|
| 37 |
+
import os
|
| 38 |
+
import time
|
| 39 |
+
from dataclasses import dataclass, field
|
| 40 |
+
from typing import Callable
|
| 41 |
+
|
| 42 |
+
#: β APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK
|
| 43 |
+
#: providers and to estimate a run's spend for the operator. They are not billing truth β the
|
| 44 |
+
#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is
|
| 45 |
+
#: why they are named and dated rather than guessed silently.
|
| 46 |
+
#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper
|
| 47 |
+
#: ~ $0.0023/result (2026-08 list). Both re-checked when a provider is added.
|
| 48 |
+
_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015)
|
| 49 |
+
_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0023)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@dataclass(frozen=True)
|
| 53 |
+
class Capability:
|
| 54 |
+
"""What ONE provider can do for ONE capability, and what it costs to ask."""
|
| 55 |
+
#: β `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never
|
| 56 |
+
#: called for this capability at all β it cannot be reached by a fallback, cannot be put first
|
| 57 |
+
#: by an env override, and cannot silently start being billed because somebody reordered a
|
| 58 |
+
#: list. Bright Data's `ig_post_views` is False on the strength of Β§4e-Β§4h: every route
|
| 59 |
+
#: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four
|
| 60 |
+
#: accounts from 13K to 268M followers returned an account-grain constant or a null.
|
| 61 |
+
capable: bool = True
|
| 62 |
+
cost_per_record: float = 0.0
|
| 63 |
+
#: Free-text, shown to an operator deciding where their money went.
|
| 64 |
+
note: str = ""
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
@dataclass
|
| 68 |
+
class Provider:
|
| 69 |
+
key: str
|
| 70 |
+
label: str
|
| 71 |
+
#: The env var holding this provider's credential. NEVER the credential itself β this module
|
| 72 |
+
#: is imported by surfaces that serialise their config.
|
| 73 |
+
key_env: str = ""
|
| 74 |
+
caps: dict = field(default_factory=dict)
|
| 75 |
+
|
| 76 |
+
def configured(self) -> bool:
|
| 77 |
+
return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True
|
| 78 |
+
|
| 79 |
+
def cap(self, capability: str) -> Capability | None:
|
| 80 |
+
return self.caps.get(capability)
|
| 81 |
+
|
| 82 |
+
def can(self, capability: str) -> bool:
|
| 83 |
+
c = self.cap(capability)
|
| 84 |
+
return bool(c and c.capable and self.configured())
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
#: β ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER β no change to any caller. That is the
|
| 88 |
+
#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS.
|
| 89 |
+
PROVIDERS: dict[str, Provider] = {
|
| 90 |
+
"brightdata": Provider(
|
| 91 |
+
key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY",
|
| 92 |
+
caps={
|
| 93 |
+
"ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"),
|
| 94 |
+
"ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"),
|
| 95 |
+
"ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"),
|
| 96 |
+
# β THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG.
|
| 97 |
+
"ig_post_views": Capability(
|
| 98 |
+
False, _COST_BRIGHTDATA,
|
| 99 |
+
"MEASURED INCAPABLE: returns one account-grain number for every reel of a "
|
| 100 |
+
"creator (identical for two shortcodes in one call, ticking upward between "
|
| 101 |
+
"calls) and never populates video_play_count. Their own documented example "
|
| 102 |
+
"account no longer reproduces it."),
|
| 103 |
+
}),
|
| 104 |
+
"apify": Provider(
|
| 105 |
+
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
|
| 106 |
+
caps={
|
| 107 |
+
# β MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
|
| 108 |
+
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
|
| 109 |
+
"ig_post_views": Capability(True, _COST_APIFY,
|
| 110 |
+
"videoPlayCount β matches Instagram's displayed views"),
|
| 111 |
+
"ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
|
| 112 |
+
"ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
|
| 113 |
+
}),
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
#: The DEFAULT chain per capability, cheapest-capable-first. Overridable β see `chain()`.
|
| 117 |
+
DEFAULT_CHAINS: dict[str, tuple] = {
|
| 118 |
+
"ig_profile": ("brightdata", "apify"),
|
| 119 |
+
"ig_post_metrics": ("brightdata", "apify"),
|
| 120 |
+
# Only one entry, and that is the point: Bright Data is declared incapable, so listing it here
|
| 121 |
+
# would be a lie that costs a wasted call on every single post.
|
| 122 |
+
"ig_post_views": ("apify",),
|
| 123 |
+
"ig_comments": ("brightdata",),
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def chain(capability: str) -> list:
|
| 128 |
+
"""The provider order for `capability` β env override first, then the default.
|
| 129 |
+
|
| 130 |
+
`AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the
|
| 131 |
+
owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a
|
| 132 |
+
vendor that is having a bad day.
|
| 133 |
+
β An override may REORDER and may DROP, but it can never make an incapable provider capable β
|
| 134 |
+
`can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore
|
| 135 |
+
resolves to EMPTY rather than to a provider that would return a wrong number, because a
|
| 136 |
+
plausible wrong number is worse than an honest refusal.
|
| 137 |
+
"""
|
| 138 |
+
raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip()
|
| 139 |
+
names = None
|
| 140 |
+
for clause in raw.split(";"):
|
| 141 |
+
if "=" in clause:
|
| 142 |
+
cap_name, _, order = clause.partition("=")
|
| 143 |
+
if cap_name.strip() == capability:
|
| 144 |
+
names = [x.strip() for x in order.split(",") if x.strip()]
|
| 145 |
+
if names is None:
|
| 146 |
+
names = list(DEFAULT_CHAINS.get(capability) or ())
|
| 147 |
+
return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)]
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def estimate(capability: str, records: int) -> dict:
|
| 151 |
+
"""What the FIRST capable provider would cost for `records` β the number an operator plans on.
|
| 152 |
+
|
| 153 |
+
Reported per capability rather than per run because that is the unit that scales: at a thousand
|
| 154 |
+
tenants the question is never "what did this run cost", it is "what does adding view counts to
|
| 155 |
+
every post cost per month".
|
| 156 |
+
"""
|
| 157 |
+
ch = chain(capability)
|
| 158 |
+
if not ch:
|
| 159 |
+
return {"capability": capability, "records": records, "provider": None, "usd": 0.0,
|
| 160 |
+
"note": "no configured, capable provider"}
|
| 161 |
+
p = ch[0]
|
| 162 |
+
c = p.cap(capability)
|
| 163 |
+
return {"capability": capability, "records": records, "provider": p.key,
|
| 164 |
+
"usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4),
|
| 165 |
+
"note": c.note}
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
@dataclass
|
| 169 |
+
class Attempt:
|
| 170 |
+
provider: str
|
| 171 |
+
ok: bool
|
| 172 |
+
note: str = ""
|
| 173 |
+
records: int = 0
|
| 174 |
+
seconds: float = 0.0
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def run(capability, work, satisfied=None, log=None):
|
| 178 |
+
"""Walk the chain for `capability` until one provider gives a SATISFYING answer.
|
| 179 |
+
|
| 180 |
+
`work(provider) -> (result, note)`; a non-empty note means it did not answer.
|
| 181 |
+
`satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any
|
| 182 |
+
truthy result.
|
| 183 |
+
|
| 184 |
+
Returns `(result, attempts)`. `attempts` is the audit trail β every provider tried, whether it
|
| 185 |
+
satisfied, and how long it took β because "where did this number come from and what did it
|
| 186 |
+
cost" is a question somebody asks about a bill, not about a stack trace.
|
| 187 |
+
|
| 188 |
+
β THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the
|
| 189 |
+
failure it must catch is not an error: a vendor answering 200 with the one field you needed
|
| 190 |
+
left blank. That is exactly how Bright Data behaves on view counts, and a chain that only
|
| 191 |
+
caught exceptions would have stopped there forever and never reached Apify.
|
| 192 |
+
"""
|
| 193 |
+
log = log or (lambda *_a: None)
|
| 194 |
+
ok = satisfied or (lambda r: bool(r))
|
| 195 |
+
attempts, last = [], None
|
| 196 |
+
for provider in chain(capability):
|
| 197 |
+
started = time.time()
|
| 198 |
+
try:
|
| 199 |
+
result, note = work(provider)
|
| 200 |
+
except Exception as e: # noqa: BLE001
|
| 201 |
+
# β TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is
|
| 202 |
+
# one refactor away from being a query param; this line must not be what leaks it.
|
| 203 |
+
result, note = None, f"{type(e).__name__}"
|
| 204 |
+
secs = round(time.time() - started, 2)
|
| 205 |
+
if note:
|
| 206 |
+
attempts.append(Attempt(provider.key, False, note, 0, secs))
|
| 207 |
+
log(f" {provider.label}: {note} β falling through")
|
| 208 |
+
continue
|
| 209 |
+
if not ok(result):
|
| 210 |
+
attempts.append(Attempt(provider.key, False, "answered without the field asked for",
|
| 211 |
+
0, secs))
|
| 212 |
+
log(f" {provider.label}: answered, but not with what was asked for β falling through")
|
| 213 |
+
last = result if last is None else last
|
| 214 |
+
continue
|
| 215 |
+
attempts.append(Attempt(provider.key, True, "", _count(result), secs))
|
| 216 |
+
log(f" {provider.label}: ok ({_count(result)} records, {secs}s)")
|
| 217 |
+
return result, attempts
|
| 218 |
+
return last, attempts
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _count(result):
|
| 222 |
+
if isinstance(result, (list, tuple, set)):
|
| 223 |
+
return len(result)
|
| 224 |
+
if isinstance(result, dict):
|
| 225 |
+
return len(result)
|
| 226 |
+
return 1 if result else 0
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def wire():
|
| 230 |
+
"""What the Settings surface renders. Booleans and labels β NEVER a credential.
|
| 231 |
+
|
| 232 |
+
Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a
|
| 233 |
+
provider is usable, never what the key is.
|
| 234 |
+
"""
|
| 235 |
+
return {
|
| 236 |
+
"providers": [
|
| 237 |
+
{"key": p.key, "label": p.label, "configured": p.configured(),
|
| 238 |
+
"capabilities": sorted(k for k, c in p.caps.items() if c.capable)}
|
| 239 |
+
for p in PROVIDERS.values()
|
| 240 |
+
],
|
| 241 |
+
"chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS},
|
| 242 |
+
"incapable": {
|
| 243 |
+
f"{p.key}:{cap}": c.note
|
| 244 |
+
for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable
|
| 245 |
+
},
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
# =============================================================================================
|
| 250 |
+
# THE CANONICAL SCHEMA β owner ruling 2026-08-08:
|
| 251 |
+
# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
|
| 252 |
+
# database even if the underlying engine changes"*
|
| 253 |
+
#
|
| 254 |
+
# β THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and
|
| 255 |
+
# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered β a
|
| 256 |
+
# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves
|
| 257 |
+
# with the vendor turns a routing change into a data migration.
|
| 258 |
+
#
|
| 259 |
+
# So every provider normalises INTO the keys below and nothing reads a vendor row downstream.
|
| 260 |
+
# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so
|
| 261 |
+
# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column
|
| 262 |
+
# nobody declared.
|
| 263 |
+
#
|
| 264 |
+
# β ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded
|
| 265 |
+
# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a
|
| 266 |
+
# `views` and a `plays` column would be modelling a distinction the platform deleted β and it is
|
| 267 |
+
# exactly the distinction that let an account-grain number wear the "Views" label for a month.
|
| 268 |
+
# β THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH
|
| 269 |
+
# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count
|
| 270 |
+
# is ~137K β and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk
|
| 271 |
+
# fields AGREE with each other, which is precisely what makes picking by name so dangerous.
|
| 272 |
+
# canonical `views` <- apify `videoPlayCount` β
matches the grid
|
| 273 |
+
# canonical `views` <- apify `videoViewCount` β off by 13x
|
| 274 |
+
# canonical `views` <- brightdata `views` β off by 13x AND account-grain
|
| 275 |
+
# =============================================================================================
|
| 276 |
+
|
| 277 |
+
#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider
|
| 278 |
+
#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier
|
| 279 |
+
#: paid run learned.
|
| 280 |
+
CANONICAL_POST_KEYS = (
|
| 281 |
+
"shortcode", "url", "influencer_key", "posted_at", "type", "caption",
|
| 282 |
+
"likes", "comments", "views", "paid_partnership", "partner", "hashtags",
|
| 283 |
+
"alt_text", "tagged_location", "source_payload",
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _int_or_none(v):
|
| 288 |
+
"""`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like
|
| 289 |
+
count β a real state that is not a measurement. Writing -1 would render as a negative like
|
| 290 |
+
count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted."""
|
| 291 |
+
try:
|
| 292 |
+
n = int(v)
|
| 293 |
+
except (TypeError, ValueError):
|
| 294 |
+
return None
|
| 295 |
+
return None if n < 0 else n
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def normalize_post_apify(row):
|
| 299 |
+
"""One Apify `instagram-scraper` item -> the canonical post row."""
|
| 300 |
+
if not isinstance(row, dict):
|
| 301 |
+
return None
|
| 302 |
+
code = str(row.get("shortCode") or "").strip()
|
| 303 |
+
if not code:
|
| 304 |
+
return None
|
| 305 |
+
out = {
|
| 306 |
+
"shortcode": code,
|
| 307 |
+
"url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"),
|
| 308 |
+
"influencer_key": str(row.get("ownerUsername") or ""),
|
| 309 |
+
"posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16],
|
| 310 |
+
"type": "video" if str(row.get("type") or "").lower() == "video" else
|
| 311 |
+
("carousel" if row.get("childPosts") else "image"),
|
| 312 |
+
"caption": str(row.get("caption") or ""),
|
| 313 |
+
# β THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`.
|
| 314 |
+
"views": _int_or_none(row.get("videoPlayCount")),
|
| 315 |
+
"likes": _int_or_none(row.get("likesCount")),
|
| 316 |
+
"comments": _int_or_none(row.get("commentsCount")),
|
| 317 |
+
"hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h),
|
| 318 |
+
"alt_text": str(row.get("alt") or ""),
|
| 319 |
+
"paid_partnership": "1" if row.get("paidPartnership") else "",
|
| 320 |
+
"partner": ", ".join(str((s or {}).get("username") or "")
|
| 321 |
+
for s in (row.get("sponsors") or []) if isinstance(s, dict)),
|
| 322 |
+
"tagged_location": str((row.get("locationName") or "")),
|
| 323 |
+
"source_payload": json.dumps(row, default=str)[:32_000_000],
|
| 324 |
+
}
|
| 325 |
+
return {k: v for k, v in out.items() if v not in (None, "")}
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def canonical_gaps(row, want=("views",)):
|
| 329 |
+
"""Which requested canonical fields this row does NOT carry β the `satisfied` input.
|
| 330 |
+
|
| 331 |
+
Named rather than inlined because "did the vendor actually answer the question" is the whole
|
| 332 |
+
fallback trigger, and a chain that asks it differently in two places will drift.
|
| 333 |
+
"""
|
| 334 |
+
row = row if isinstance(row, dict) else {}
|
| 335 |
+
return [k for k in want if row.get(k) in (None, "")]
|