fsanyoto commited on
Commit
c25ccce
Β·
verified Β·
1 Parent(s): 1c766f4

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
Files changed (4) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/automation_engine.py +148 -23
  4. api/providers.py +335 -0
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "dce0fee",
3
  "releases": [
4
  {
5
  "version": "v14",
 
1
  {
2
+ "current": "8ca7046",
3
  "releases": [
4
  {
5
  "version": "v14",
VERSION CHANGED
@@ -1 +1 @@
1
- dce0fee
 
1
+ 8ca7046
api/automation_engine.py CHANGED
@@ -37,6 +37,8 @@ from urllib.parse import urljoin, urlparse
37
 
38
  import requests
39
 
 
 
40
  try: # the parser the extraction half needs
41
  from bs4 import BeautifulSoup
42
  except Exception: # pragma: no cover β€” deploy lag; see mailbox β‘‘
@@ -1042,6 +1044,8 @@ IG_FIELD_DESCRIPTIONS = {
1042
  "avg_engagement": "Average engagement rate reported for the profile.",
1043
  "avg_likes_12": "Average likes across the 12 most recent captured posts.",
1044
  "avg_plays_12": "Average plays across the 12 most recent captured posts.",
 
 
1045
  "bio": "Biography shown on the Instagram profile.",
1046
  "bio_hashtags": "Hashtags listed in the profile biography.",
1047
  "business_category": "Instagram's business category for the account.",
@@ -2261,6 +2265,14 @@ POST_FIELDS = [
2261
  # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named
2262
  # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10
2263
  # (folded into a single "Views"), not because we are reading the wrong key.
 
 
 
 
 
 
 
 
2264
  field_def("plays", "Plays", "int"),
2265
  field_def("likes", "Likes", "int"),
2266
  field_def("comments", "Comments", "int"),
@@ -2328,9 +2340,9 @@ POST_SNAPSHOT_FIELDS = [
2328
  field_def("comments", "Comments", "int"),
2329
  # Plays move independently of likes on video, so it is its own series rather than a thing to
2330
  # derive. Paid rung only; blank means not read.
2331
- # β›” No `views` column, for the reason POST_FIELDS gives at length: the vendor's `views` is
2332
- # account-grain and undefinable, so a per-post SERIES of it would be a time series of a
2333
- # number we cannot name. Β§4e/Β§4f.
2334
  field_def("plays", "Plays", "int"),
2335
  field_def("source_payload", "Source data", "json"),
2336
  field_def("post_link", "Post", "link",
@@ -3137,6 +3149,79 @@ def bd_scrape(dataset_id, urls, wait=None, deferred=None):
3137
  f"or collect {sid} from the control panel")
3138
 
3139
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3140
  #: ⭐⭐ 2026-08-07 β€” HOW LONG THE CORPUS RUNG MAY WAIT. A corpus query is a scan over 620M rows
3141
  #: and takes MINUTES, not seconds (Β§2d MEASURED ~4 min for a narrow one). It is a FALLBACK behind
3142
  #: a scrape that already failed, so the budget is generous β€” but bounded, because an enrich step
@@ -3354,6 +3439,50 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False,
3354
  elif metric_notes:
3355
  note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
3356
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3357
  comments = []
3358
  if comment_metrics and posts:
3359
  # This is the cost boundary: do not make the full comment request unless the stored
@@ -3873,12 +4002,13 @@ PRESET_PROFILE_FIELDS = [
3873
  # (`MAX_POSTS_PER_PULL`), so a window wider than that would silently average fewer posts than its own
3874
  # label claims. `sortBy` is mandatory alongside a `limit`, and this is why β€” "the last 12"
3875
  # has to name what makes one post later than another.
3876
- # β›” `avg_views_12` IS RETIRED (2026-08-08, Β§4e β€” owner-approved). It averaged `ut_ig_posts`'
3877
- # `views`, a column that no longer exists because the vendor delivers that number at ACCOUNT
3878
- # grain. An average over one repeated account number is not an average of anything, and while
3879
- # the column stood blank the rollup beside it read as "not read yet" and implied another paid
3880
- # run could fill it. It could not. Retired by KEY in `_ig_schema_contract`'s drop set so it
3881
- # leaves tables already in production, not just new ones.
 
3882
  field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup",
3883
  rollup={"link": "posts_link", "field": "plays", "fn": "average",
3884
  "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
@@ -4021,14 +4151,11 @@ def _ig_schema_contract(rt, profile_tables=None):
4021
  else _ig_profile_tables(rt)))
4022
  backlinks, wanted, drops = [], {}, {
4023
  IG_SNAPSHOTS_TABLE: {"post_hashtags"},
4024
- # β›” 2026-08-08 (owner-approved, Β§4e/Β§4f) β€” the account-grain `views` leaves the post
4025
- # tables. The CELLS go with the COLUMN (`_reconcile_ig_graph_fields` pops both), which is
4026
- # the point: a row dict keeping a `views` key whose field no longer exists is invisible
4027
- # everywhere except the next export, where it reappears as a column nobody declared.
4028
- # ⚠ Retiring by KEY is what reaches nurilab's 360 live post rows and 348 snapshot rows;
4029
- # deleting the `field_def` alone would only have changed tables created after today.
4030
- IG_POSTS_TABLE: {"views"},
4031
- IG_POST_SNAPSHOTS_TABLE: {"views"},
4032
  }
4033
  for table_key in profile_keys:
4034
  table = tables.get(table_key) or {}
@@ -4040,9 +4167,7 @@ def _ig_schema_contract(rt, profile_tables=None):
4040
  wanted[table_key] = _profile_schema_for(bound)
4041
  # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
4042
  # to disagree, so it is retired by key just like the old nested Posts JSON.
4043
- # `avg_views_12` joins them: a rollup over a column that no longer exists would otherwise
4044
- # survive on every profile table already in production and render as a permanent blank.
4045
- drops[table_key] = {"posts", "post_hashtags", "tracked", "avg_views_12"}
4046
  wanted.update({
4047
  IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
4048
  IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
@@ -5411,7 +5536,7 @@ def capture_rows(res, pulled):
5411
  ident[k] = _s(p.get(k), n)
5412
  if p.get("source_payload"):
5413
  ident["source_payload"] = str(p["source_payload"])
5414
- metrics = {k: p.get(k) for k in ("likes", "comments", "plays")}
5415
  # ⭐⭐ 2026-08-07 β€” THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
5416
  #
5417
  # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
@@ -5442,7 +5567,7 @@ def capture_rows(res, pulled):
5442
  "shortcode": p["shortcode"], "influencer_key": prof.get("username"),
5443
  "pulled_at": pulled,
5444
  "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
5445
- "plays": _s(metrics["plays"]),
5446
  "source_payload": str(p.get("source_payload") or "")})
5447
  for comment in p.get("embedded_comments") or []:
5448
  # The embedded shape often names only the commenter. Its parent Post is the
@@ -5497,7 +5622,7 @@ PRESET_FROM_PROFILE = {
5497
  PRESET_WRITTEN_ELSEWHERE = (
5498
  "platform", "handle", "enriched_at",
5499
  "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link",
5500
- "avg_plays_12", "avg_likes_12", "avg_comments_12",
5501
  "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured",
5502
  )
5503
 
 
37
 
38
  import requests
39
 
40
+ import providers
41
+
42
  try: # the parser the extraction half needs
43
  from bs4 import BeautifulSoup
44
  except Exception: # pragma: no cover β€” deploy lag; see mailbox β‘‘
 
1044
  "avg_engagement": "Average engagement rate reported for the profile.",
1045
  "avg_likes_12": "Average likes across the 12 most recent captured posts.",
1046
  "avg_plays_12": "Average plays across the 12 most recent captured posts.",
1047
+ "avg_views_12": "Average views across the 12 most recent captured posts.",
1048
+ "views": "View count for the post, as Instagram displays it.",
1049
  "bio": "Biography shown on the Instagram profile.",
1050
  "bio_hashtags": "Hashtags listed in the profile biography.",
1051
  "business_category": "Instagram's business category for the account.",
 
2265
  # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named
2266
  # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10
2267
  # (folded into a single "Views"), not because we are reading the wrong key.
2268
+ # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED.
2269
+ # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the
2270
+ # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` β€”
2271
+ # measured against a browser-read ground truth to the digit. Same column name, same type, same
2272
+ # rollup; the engine underneath is swappable and the preset database never moved. That is the
2273
+ # owner's schema ruling working exactly as intended.
2274
+ # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung.
2275
+ field_def("views", "Views", "int"),
2276
  field_def("plays", "Plays", "int"),
2277
  field_def("likes", "Likes", "int"),
2278
  field_def("comments", "Comments", "int"),
 
2340
  field_def("comments", "Comments", "int"),
2341
  # Plays move independently of likes on video, so it is its own series rather than a thing to
2342
  # derive. Paid rung only; blank means not read.
2343
+ # The series regains Views alongside the latest-value projection on the Post row β€” one store
2344
+ # for one series (R3) is untouched; this IS that store.
2345
+ field_def("views", "Views", "int"),
2346
  field_def("plays", "Plays", "int"),
2347
  field_def("source_payload", "Source data", "json"),
2348
  field_def("post_link", "Post", "link",
 
3149
  f"or collect {sid} from the control panel")
3150
 
3151
 
3152
+ # =============================================================================================
3153
+ # APIFY β€” THE SECOND PROVIDER (owner ruling 2026-08-08). Registered in `providers.py`; this is
3154
+ # only the wire.
3155
+ #
3156
+ # ⭐ IT EXISTS FOR EXACTLY ONE CAPABILITY TODAY: `ig_post_views`. Bright Data answers profile,
3157
+ # likes and comments correctly and cannot answer a per-reel view count on ANY route; Apify's
3158
+ # `videoPlayCount` matched a browser-read ground truth to the digit (137,684 vs ~137K and 299,493
3159
+ # vs 299K). So this runs as a TOP-UP over Bright Data's rows, never as a replacement for them β€”
3160
+ # re-buying the profile and the likes from a second vendor is the expensive mistake the capability
3161
+ # split exists to prevent.
3162
+ # β›” `videoPlayCount`, NEVER `videoViewCount`. Apify ships BOTH, and the wrong one (10,678) agrees
3163
+ # with Bright Data's useless `views` (10,638) for the same reel whose true count is ~137K. Two
3164
+ # vendors' junk fields agreeing is what makes picking by name so dangerous here.
3165
+ # =============================================================================================
3166
+
3167
+ APIFY_BASE = "https://api.apify.com/v2"
3168
+ #: The general Instagram scraper. `directUrls` accepts /p/ and /reel/ permalinks alike.
3169
+ APIFY_ACTOR_POSTS = os.environ.get("AIOS_APIFY_ACTOR") or "apify~instagram-scraper"
3170
+ #: Seconds to let a synchronous actor run before giving up. Apify bills the run either way, so a
3171
+ #: timeout here is money already spent β€” generous on purpose.
3172
+ APIFY_WAIT = float(os.environ.get("AIOS_APIFY_WAIT") or 240)
3173
+
3174
+
3175
+ def apify_key():
3176
+ """The key, or ''. Read fresh every call, exactly as `bd_key` is."""
3177
+ return (os.environ.get("AIOS_APIFY_KEY") or "").strip()
3178
+
3179
+
3180
+ def apify_ready():
3181
+ return bool(apify_key())
3182
+
3183
+
3184
+ def apify_posts(urls):
3185
+ """Post/reel permalinks -> (canonical rows, note). A note means it did NOT answer.
3186
+
3187
+ Returns rows already normalised through `providers.normalize_post_apify`, so nothing
3188
+ downstream ever sees an Apify-shaped key. That is the owner's schema ruling enforced at the
3189
+ only place it can be: the boundary.
3190
+ """
3191
+ want = [str(u) for u in (urls or []) if u]
3192
+ if not want:
3193
+ return [], ""
3194
+ key = apify_key()
3195
+ if not key:
3196
+ return [], "AIOS_APIFY_KEY is not configured β€” the Apify rung is closed"
3197
+ url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items"
3198
+ f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}")
3199
+ body = {"directUrls": want, "resultsType": "posts", "resultsLimit": max(1, len(want)),
3200
+ "addParentData": False}
3201
+ try:
3202
+ status, raw = fetch_json(url, body, timeout=APIFY_WAIT + 60, max_kb=BD_MAX_KB)
3203
+ except Refused as e:
3204
+ return [], f"Apify's address was refused by the URL rail: {e}"
3205
+ except Exception as e: # noqa: BLE001
3206
+ # ⚠ TYPE NAME ONLY β€” the token is in the query string of `url`, so a str(e) that echoed
3207
+ # the request would put a live credential in the run log.
3208
+ return [], f"Apify did not answer ({type(e).__name__})"
3209
+ if status == 402:
3210
+ return [], "the Apify account is out of credit"
3211
+ if status in (401, 403):
3212
+ return [], f"Apify refused our key ({status})"
3213
+ if status >= 400:
3214
+ return [], f"Apify answered {status}"
3215
+ try:
3216
+ items = json.loads(raw.decode("utf-8", "replace"))
3217
+ except Exception: # noqa: BLE001
3218
+ return [], "Apify's answer was not readable JSON"
3219
+ if not isinstance(items, list):
3220
+ return [], "Apify answered with something other than a list of items"
3221
+ rows = [providers.normalize_post_apify(x) for x in items]
3222
+ return [r for r in rows if r], ""
3223
+
3224
+
3225
  #: ⭐⭐ 2026-08-07 β€” HOW LONG THE CORPUS RUNG MAY WAIT. A corpus query is a scan over 620M rows
3226
  #: and takes MINUTES, not seconds (Β§2d MEASURED ~4 min for a narrow one). It is a FALLBACK behind
3227
  #: a scrape that already failed, so the budget is generous β€” but bounded, because an enrich step
 
3439
  elif metric_notes:
3440
  note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
3441
 
3442
+ # ⭐⭐ THE VIEWS TOP-UP β€” capability routing, not provider failover (owner ruling
3443
+ # 2026-08-08, `providers.py`). Bright Data has already answered profile + likes + comments
3444
+ # above and is declared INCAPABLE of `ig_post_views`, so the chain for that ONE capability
3445
+ # resolves to Apify and only the video permalinks are re-bought. Re-running the whole
3446
+ # profile on the second vendor would pay twice for the 90% that already worked, which at
3447
+ # thousands of tenants x twelve posts is the entire bill.
3448
+ # ⚠ VIDEO ONLY. A carousel or image has no view count, so sending it would buy a record
3449
+ # that can only come back blank.
3450
+ video_urls = [p["url"] for p in posts
3451
+ if p.get("type") == "video" and p.get("url")]
3452
+ if video_urls:
3453
+ def _views_work(prov, _urls=tuple(video_urls)):
3454
+ if prov.key == "apify":
3455
+ return apify_posts(list(_urls))
3456
+ # A provider in the chain with no runner here is a configuration error, not a
3457
+ # vendor outage β€” say so rather than returning an empty list that reads as "the
3458
+ # vendor had nothing".
3459
+ return None, f"no {prov.key} runner is wired for ig_post_views"
3460
+
3461
+ got_views, attempts = providers.run(
3462
+ "ig_post_views", _views_work,
3463
+ # β›” THE SATISFIED PREDICATE IS THE FALLBACK TRIGGER. Rows that come back with
3464
+ # every `views` blank are a FAILURE for this capability even at HTTP 200 β€” which
3465
+ # is exactly how Bright Data behaves, and why a chain that only caught exceptions
3466
+ # would never have reached a second provider.
3467
+ satisfied=lambda rows: bool(rows) and any(r.get("views") for r in rows),
3468
+ log=log)
3469
+ for row in (got_views or []):
3470
+ target = next((p for p in posts
3471
+ if p.get("shortcode") == row.get("shortcode")), None)
3472
+ # β›” TAKE ONLY THE CAPABILITY THAT WAS ASKED FOR. This provider also returns
3473
+ # likes/comments/caption, and letting them land would silently switch the source
3474
+ # of columns Bright Data already answered β€” the schema would be stable but the
3475
+ # PROVENANCE would flip halfway through a row.
3476
+ if target and row.get("views") not in (None, ""):
3477
+ target["views"] = row["views"]
3478
+ used = next((a.provider for a in attempts if a.ok), "")
3479
+ if used:
3480
+ note = "; ".join(x for x in (note, f"view counts via {used}") if x)
3481
+ elif attempts:
3482
+ note = "; ".join(x for x in (
3483
+ note, "no provider could supply view counts: "
3484
+ + "; ".join(f"{a.provider} {a.note}" for a in attempts)) if x)
3485
+
3486
  comments = []
3487
  if comment_metrics and posts:
3488
  # This is the cost boundary: do not make the full comment request unless the stored
 
4002
  # (`MAX_POSTS_PER_PULL`), so a window wider than that would silently average fewer posts than its own
4003
  # label claims. `sortBy` is mandatory alongside a `limit`, and this is why β€” "the last 12"
4004
  # has to name what makes one post later than another.
4005
+ # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours
4006
+ # because its input was an account-grain constant; with the input now a real per-reel
4007
+ # measurement the average means what its label says again.
4008
+ field_def("avg_views_12", "Avg views Β· last 12 posts", "rollup",
4009
+ rollup={"link": "posts_link", "field": "views", "fn": "average",
4010
+ "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
4011
+ "distinctBy": "shortcode"}),
4012
  field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup",
4013
  rollup={"link": "posts_link", "field": "plays", "fn": "average",
4014
  "limit": MAX_POSTS_PER_PULL, "sortBy": "posted_at", "sortDir": "desc",
 
4151
  else _ig_profile_tables(rt)))
4152
  backlinks, wanted, drops = [], {}, {
4153
  IG_SNAPSHOTS_TABLE: {"post_hashtags"},
4154
+ # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today
4155
+ # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS
4156
+ # and fed by the `ig_post_views` capability. Leaving it here would have made the migration
4157
+ # delete, on every authenticated read, the column the same release just added β€” the
4158
+ # drop set and the field list are two halves of ONE contract and must move together.
 
 
 
4159
  }
4160
  for table_key in profile_keys:
4161
  table = tables.get(table_key) or {}
 
4167
  wanted[table_key] = _profile_schema_for(bound)
4168
  # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
4169
  # to disagree, so it is retired by key just like the old nested Posts JSON.
4170
+ drops[table_key] = {"posts", "post_hashtags", "tracked"}
 
 
4171
  wanted.update({
4172
  IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
4173
  IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
 
5536
  ident[k] = _s(p.get(k), n)
5537
  if p.get("source_payload"):
5538
  ident["source_payload"] = str(p["source_payload"])
5539
+ metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
5540
  # ⭐⭐ 2026-08-07 β€” THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
5541
  #
5542
  # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
 
5567
  "shortcode": p["shortcode"], "influencer_key": prof.get("username"),
5568
  "pulled_at": pulled,
5569
  "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
5570
+ "views": _s(metrics["views"]), "plays": _s(metrics["plays"]),
5571
  "source_payload": str(p.get("source_payload") or "")})
5572
  for comment in p.get("embedded_comments") or []:
5573
  # The embedded shape often names only the commenter. Its parent Post is the
 
5622
  PRESET_WRITTEN_ELSEWHERE = (
5623
  "platform", "handle", "enriched_at",
5624
  "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link",
5625
+ "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12",
5626
  "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured",
5627
  )
5628
 
api/providers.py ADDED
@@ -0,0 +1,335 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """THE PROVIDER LAYER β€” many vendors behind one capability, chosen per FIELD and per cost.
2
+
3
+ ⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if
4
+ anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize
5
+ when we scale to thousands+ customer … make sure our code is customizable and modular."*
6
+
7
+ β›” THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B".
8
+ A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious
9
+ shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments
10
+ correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A
11
+ provider-level fallback would notice the missing views and re-buy the profile, the likes and the
12
+ comments from Apify as well β€” **paying twice for the 90% that already worked**. At one profile that
13
+ is noise; at thousands of customers Γ— twelve posts each it is the whole bill.
14
+
15
+ So routing is per CAPABILITY, and a capability is the smallest independently-billable thing:
16
+
17
+ ig_profile -> brightdata (36 fields, works)
18
+ ig_post_metrics -> brightdata (likes + comments, works)
19
+ ig_post_views -> apify (Bright Data is INCAPABLE β€” see `capable=False` below)
20
+ ig_comments -> brightdata
21
+
22
+ Each capability has an ORDERED list of providers. The first one that is configured AND capable AND
23
+ succeeds wins; the rest are never called, so the common path costs exactly one vendor record.
24
+
25
+ ⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in
26
+ the one column you asked for has failed for our purposes, and a chain that only catches exceptions
27
+ would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying
28
+ answer exactly as it would on a 500.
29
+
30
+ ⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time
31
+ (`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what
32
+ lets a tenant be moved off a vendor without a release.
33
+ """
34
+ from __future__ import annotations
35
+
36
+ import json
37
+ import os
38
+ import time
39
+ from dataclasses import dataclass, field
40
+ from typing import Callable
41
+
42
+ #: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK
43
+ #: providers and to estimate a run's spend for the operator. They are not billing truth β€” the
44
+ #: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is
45
+ #: why they are named and dated rather than guessed silently.
46
+ #: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper
47
+ #: ~ $0.0023/result (2026-08 list). Both re-checked when a provider is added.
48
+ _COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015)
49
+ _COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0023)
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class Capability:
54
+ """What ONE provider can do for ONE capability, and what it costs to ask."""
55
+ #: β›” `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never
56
+ #: called for this capability at all β€” it cannot be reached by a fallback, cannot be put first
57
+ #: by an env override, and cannot silently start being billed because somebody reordered a
58
+ #: list. Bright Data's `ig_post_views` is False on the strength of Β§4e-Β§4h: every route
59
+ #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four
60
+ #: accounts from 13K to 268M followers returned an account-grain constant or a null.
61
+ capable: bool = True
62
+ cost_per_record: float = 0.0
63
+ #: Free-text, shown to an operator deciding where their money went.
64
+ note: str = ""
65
+
66
+
67
+ @dataclass
68
+ class Provider:
69
+ key: str
70
+ label: str
71
+ #: The env var holding this provider's credential. NEVER the credential itself β€” this module
72
+ #: is imported by surfaces that serialise their config.
73
+ key_env: str = ""
74
+ caps: dict = field(default_factory=dict)
75
+
76
+ def configured(self) -> bool:
77
+ return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True
78
+
79
+ def cap(self, capability: str) -> Capability | None:
80
+ return self.caps.get(capability)
81
+
82
+ def can(self, capability: str) -> bool:
83
+ c = self.cap(capability)
84
+ return bool(c and c.capable and self.configured())
85
+
86
+
87
+ #: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER β€” no change to any caller. That is the
88
+ #: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS.
89
+ PROVIDERS: dict[str, Provider] = {
90
+ "brightdata": Provider(
91
+ key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY",
92
+ caps={
93
+ "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"),
94
+ "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"),
95
+ "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"),
96
+ # β›” THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG.
97
+ "ig_post_views": Capability(
98
+ False, _COST_BRIGHTDATA,
99
+ "MEASURED INCAPABLE: returns one account-grain number for every reel of a "
100
+ "creator (identical for two shortcodes in one call, ticking upward between "
101
+ "calls) and never populates video_play_count. Their own documented example "
102
+ "account no longer reproduces it."),
103
+ }),
104
+ "apify": Provider(
105
+ key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
106
+ caps={
107
+ # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
108
+ # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
109
+ "ig_post_views": Capability(True, _COST_APIFY,
110
+ "videoPlayCount β€” matches Instagram's displayed views"),
111
+ "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
112
+ "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
113
+ }),
114
+ }
115
+
116
+ #: The DEFAULT chain per capability, cheapest-capable-first. Overridable β€” see `chain()`.
117
+ DEFAULT_CHAINS: dict[str, tuple] = {
118
+ "ig_profile": ("brightdata", "apify"),
119
+ "ig_post_metrics": ("brightdata", "apify"),
120
+ # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here
121
+ # would be a lie that costs a wasted call on every single post.
122
+ "ig_post_views": ("apify",),
123
+ "ig_comments": ("brightdata",),
124
+ }
125
+
126
+
127
+ def chain(capability: str) -> list:
128
+ """The provider order for `capability` β€” env override first, then the default.
129
+
130
+ `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the
131
+ owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a
132
+ vendor that is having a bad day.
133
+ β›” An override may REORDER and may DROP, but it can never make an incapable provider capable β€”
134
+ `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore
135
+ resolves to EMPTY rather than to a provider that would return a wrong number, because a
136
+ plausible wrong number is worse than an honest refusal.
137
+ """
138
+ raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip()
139
+ names = None
140
+ for clause in raw.split(";"):
141
+ if "=" in clause:
142
+ cap_name, _, order = clause.partition("=")
143
+ if cap_name.strip() == capability:
144
+ names = [x.strip() for x in order.split(",") if x.strip()]
145
+ if names is None:
146
+ names = list(DEFAULT_CHAINS.get(capability) or ())
147
+ return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)]
148
+
149
+
150
+ def estimate(capability: str, records: int) -> dict:
151
+ """What the FIRST capable provider would cost for `records` β€” the number an operator plans on.
152
+
153
+ Reported per capability rather than per run because that is the unit that scales: at a thousand
154
+ tenants the question is never "what did this run cost", it is "what does adding view counts to
155
+ every post cost per month".
156
+ """
157
+ ch = chain(capability)
158
+ if not ch:
159
+ return {"capability": capability, "records": records, "provider": None, "usd": 0.0,
160
+ "note": "no configured, capable provider"}
161
+ p = ch[0]
162
+ c = p.cap(capability)
163
+ return {"capability": capability, "records": records, "provider": p.key,
164
+ "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4),
165
+ "note": c.note}
166
+
167
+
168
+ @dataclass
169
+ class Attempt:
170
+ provider: str
171
+ ok: bool
172
+ note: str = ""
173
+ records: int = 0
174
+ seconds: float = 0.0
175
+
176
+
177
+ def run(capability, work, satisfied=None, log=None):
178
+ """Walk the chain for `capability` until one provider gives a SATISFYING answer.
179
+
180
+ `work(provider) -> (result, note)`; a non-empty note means it did not answer.
181
+ `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any
182
+ truthy result.
183
+
184
+ Returns `(result, attempts)`. `attempts` is the audit trail β€” every provider tried, whether it
185
+ satisfied, and how long it took β€” because "where did this number come from and what did it
186
+ cost" is a question somebody asks about a bill, not about a stack trace.
187
+
188
+ β›” THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the
189
+ failure it must catch is not an error: a vendor answering 200 with the one field you needed
190
+ left blank. That is exactly how Bright Data behaves on view counts, and a chain that only
191
+ caught exceptions would have stopped there forever and never reached Apify.
192
+ """
193
+ log = log or (lambda *_a: None)
194
+ ok = satisfied or (lambda r: bool(r))
195
+ attempts, last = [], None
196
+ for provider in chain(capability):
197
+ started = time.time()
198
+ try:
199
+ result, note = work(provider)
200
+ except Exception as e: # noqa: BLE001
201
+ # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is
202
+ # one refactor away from being a query param; this line must not be what leaks it.
203
+ result, note = None, f"{type(e).__name__}"
204
+ secs = round(time.time() - started, 2)
205
+ if note:
206
+ attempts.append(Attempt(provider.key, False, note, 0, secs))
207
+ log(f" {provider.label}: {note} β€” falling through")
208
+ continue
209
+ if not ok(result):
210
+ attempts.append(Attempt(provider.key, False, "answered without the field asked for",
211
+ 0, secs))
212
+ log(f" {provider.label}: answered, but not with what was asked for β€” falling through")
213
+ last = result if last is None else last
214
+ continue
215
+ attempts.append(Attempt(provider.key, True, "", _count(result), secs))
216
+ log(f" {provider.label}: ok ({_count(result)} records, {secs}s)")
217
+ return result, attempts
218
+ return last, attempts
219
+
220
+
221
+ def _count(result):
222
+ if isinstance(result, (list, tuple, set)):
223
+ return len(result)
224
+ if isinstance(result, dict):
225
+ return len(result)
226
+ return 1 if result else 0
227
+
228
+
229
+ def wire():
230
+ """What the Settings surface renders. Booleans and labels β€” NEVER a credential.
231
+
232
+ Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a
233
+ provider is usable, never what the key is.
234
+ """
235
+ return {
236
+ "providers": [
237
+ {"key": p.key, "label": p.label, "configured": p.configured(),
238
+ "capabilities": sorted(k for k, c in p.caps.items() if c.capable)}
239
+ for p in PROVIDERS.values()
240
+ ],
241
+ "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS},
242
+ "incapable": {
243
+ f"{p.key}:{cap}": c.note
244
+ for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable
245
+ },
246
+ }
247
+
248
+
249
+ # =============================================================================================
250
+ # THE CANONICAL SCHEMA β€” owner ruling 2026-08-08:
251
+ # *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
252
+ # database even if the underlying engine changes"*
253
+ #
254
+ # β›” THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and
255
+ # `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered β€” a
256
+ # tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves
257
+ # with the vendor turns a routing change into a data migration.
258
+ #
259
+ # So every provider normalises INTO the keys below and nothing reads a vendor row downstream.
260
+ # `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so
261
+ # a third provider cannot ship with a near-miss key like `viewCount` and silently write a column
262
+ # nobody declared.
263
+ #
264
+ # ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded
265
+ # Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a
266
+ # `views` and a `plays` column would be modelling a distinction the platform deleted β€” and it is
267
+ # exactly the distinction that let an account-grain number wear the "Views" label for a month.
268
+ # β›” THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH
269
+ # `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count
270
+ # is ~137K β€” and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk
271
+ # fields AGREE with each other, which is precisely what makes picking by name so dangerous.
272
+ # canonical `views` <- apify `videoPlayCount` βœ… matches the grid
273
+ # canonical `views` <- apify `videoViewCount` β›” off by 13x
274
+ # canonical `views` <- brightdata `views` β›” off by 13x AND account-grain
275
+ # =============================================================================================
276
+
277
+ #: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider
278
+ #: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier
279
+ #: paid run learned.
280
+ CANONICAL_POST_KEYS = (
281
+ "shortcode", "url", "influencer_key", "posted_at", "type", "caption",
282
+ "likes", "comments", "views", "paid_partnership", "partner", "hashtags",
283
+ "alt_text", "tagged_location", "source_payload",
284
+ )
285
+
286
+
287
+ def _int_or_none(v):
288
+ """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like
289
+ count β€” a real state that is not a measurement. Writing -1 would render as a negative like
290
+ count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted."""
291
+ try:
292
+ n = int(v)
293
+ except (TypeError, ValueError):
294
+ return None
295
+ return None if n < 0 else n
296
+
297
+
298
+ def normalize_post_apify(row):
299
+ """One Apify `instagram-scraper` item -> the canonical post row."""
300
+ if not isinstance(row, dict):
301
+ return None
302
+ code = str(row.get("shortCode") or "").strip()
303
+ if not code:
304
+ return None
305
+ out = {
306
+ "shortcode": code,
307
+ "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"),
308
+ "influencer_key": str(row.get("ownerUsername") or ""),
309
+ "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16],
310
+ "type": "video" if str(row.get("type") or "").lower() == "video" else
311
+ ("carousel" if row.get("childPosts") else "image"),
312
+ "caption": str(row.get("caption") or ""),
313
+ # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`.
314
+ "views": _int_or_none(row.get("videoPlayCount")),
315
+ "likes": _int_or_none(row.get("likesCount")),
316
+ "comments": _int_or_none(row.get("commentsCount")),
317
+ "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h),
318
+ "alt_text": str(row.get("alt") or ""),
319
+ "paid_partnership": "1" if row.get("paidPartnership") else "",
320
+ "partner": ", ".join(str((s or {}).get("username") or "")
321
+ for s in (row.get("sponsors") or []) if isinstance(s, dict)),
322
+ "tagged_location": str((row.get("locationName") or "")),
323
+ "source_payload": json.dumps(row, default=str)[:32_000_000],
324
+ }
325
+ return {k: v for k, v in out.items() if v not in (None, "")}
326
+
327
+
328
+ def canonical_gaps(row, want=("views",)):
329
+ """Which requested canonical fields this row does NOT carry β€” the `satisfied` input.
330
+
331
+ Named rather than inlined because "did the vendor actually answer the question" is the whole
332
+ fallback trigger, and a chain that asks it differently in two places will drift.
333
+ """
334
+ row = row if isinstance(row, dict) else {}
335
+ return [k for k in want if row.get(k) in (None, "")]