fsanyoto commited on
Commit
e61c618
Β·
verified Β·
1 Parent(s): a0c3320

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
Files changed (4) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/automation_engine.py +141 -15
  4. api/connectors_ig.py +142 -5
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "a1ad7ff",
3
  "releases": [
4
  {
5
  "version": "v24",
 
1
  {
2
+ "current": "cf11549",
3
  "releases": [
4
  {
5
  "version": "v24",
VERSION CHANGED
@@ -1 +1 @@
1
- a1ad7ff
 
1
+ cf11549
api/automation_engine.py CHANGED
@@ -1133,7 +1133,7 @@ IG_FIELD_DESCRIPTIONS = {
1133
  "related_accounts": "Accounts Instagram suggests alongside this profile.",
1134
  "replies": "Reply count reported for the comment.",
1135
  "shortcode": "Instagram shortcode that uniquely identifies the post.",
1136
- "source_payload": "Full Bright Data source response for this record.",
1137
  "snapshot_key": "Unique ID for this profile snapshot.",
1138
  "source": "Method or provider used to read this profile.",
1139
  "tagged_location": "Location tagged on the post; it is not the creator's residence.",
@@ -1195,6 +1195,58 @@ MAX_POSTS_PER_PULL = 12
1195
  DEFAULT_POSTS_PER_PULL = 10
1196
 
1197
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1198
  def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True):
1199
  """`(maxPosts, error)` β€” refuses out-of-range rather than clamping into range.
1200
 
@@ -2593,9 +2645,10 @@ from connectors_ig import ( # noqa: E402
2593
  _bd_tagged_location,
2594
  _first, _ig_int,
2595
  DEFERRED_MARK,
 
2596
  bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready,
2597
  bd_snapshot_progress, depth_refusal,
2598
- ig_handle, pull_profile, top_up_views,
2599
  )
2600
 
2601
 
@@ -4416,7 +4469,9 @@ def capture_rows(res, pulled):
4416
  snap_row = {
4417
  "snapshot_key": f"{prof.get('username')}@{pulled}",
4418
  "influencer_key": prof.get("username"), "pulled_at": pulled,
4419
- "source": _s((res or {}).get("via")), "approx": _s(prof.get("approx")),
 
 
4420
  "bio": _s(prof.get("bio"), 500),
4421
  "external_url": _s(prof.get("external_url"), 300),
4422
  }
@@ -5298,7 +5353,10 @@ RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db,
5298
  # offers one anyway is offering a button that must refuse.
5299
  SOURCES = {
5300
  "instagram": {
5301
- "key": "instagram", "label": "Instagram", "vendor": "Bright Data",
 
 
 
5302
  "probe": bd_ready,
5303
  "capture": pull_profile,
5304
  "discover": bd_filter_start,
@@ -6119,6 +6177,9 @@ def _clean_action_config(kind, cfg, depth, seen, count):
6119
  e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
6120
  except (TypeError, ValueError):
6121
  return None, "the number of days has to be a number"
 
 
 
6122
  return {"profileField": _s(cfg.get("profileField"), 80).strip(),
6123
  "fromView": _s(cfg.get("fromView"), 80).strip(),
6124
  "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT,
@@ -6139,6 +6200,9 @@ def _clean_action_config(kind, cfg, depth, seen, count):
6139
  # until the user intentionally enables it.
6140
  "commentMetrics": bool(cfg.get("commentMetrics")),
6141
  "dryRun": bool(cfg.get("dryRun")),
 
 
 
6142
  "maxPosts": max_posts}, None
6143
  if kind == "find_records":
6144
  table = _s(cfg.get("table"), 60).strip()
@@ -6903,6 +6967,21 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6903
  pending_profile=(pend_prof
6904
  if not cfg.get("dryRun") else None),
6905
  prefetch=enrich_prefetch)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6906
  for _p in pend_prof:
6907
  _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso()
6908
  enrich["pendingProfiles"].extend(pend_prof)
@@ -7343,6 +7422,50 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
7343
  _progress_memo[sid] = bd_snapshot_progress(sid)
7344
  return _progress_memo[sid]
7345
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7346
  def _rows_of(sid):
7347
  if sid not in _rows_memo:
7348
  payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
@@ -7358,8 +7481,20 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
7358
  stale_h = _hours_since(task.get("requestedAt"))
7359
  state, records, empty_note = _progress_of(task["snapshotId"])
7360
  if state in ("done", "failed") and not records:
7361
- closed += 1
7362
  acc["profiles"] += 1
 
 
 
 
 
 
 
 
 
 
 
 
 
7363
  acc["blocked"] += 1
7364
  note = f"@{task['influencer']}: {_s(empty_note, 240)}"
7365
  notes.append(note)
@@ -7417,16 +7552,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
7417
  acc["notes"].append(note)
7418
  continue
7419
  acc["ok"] += 1
7420
- res = {"state": "ok", "profile": profile, "posts": [], "comments": [],
7421
- "via": "brightdata:deferred", "note": ""}
7422
- pulled = _iso()
7423
- snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
7424
- acc["snaps"].append(snap_row)
7425
- acc["posts"].extend(idents)
7426
- acc["psnaps"].extend(metric_rows)
7427
- acc["comments"].extend(comment_rows)
7428
- _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled))
7429
- affected.append(task["rowId"])
7430
  notes.append(f"@{task['influencer']}: collected the profile the source had already been "
7431
  f"paid for ({profile.get('followers')} followers)")
7432
 
 
1133
  "related_accounts": "Accounts Instagram suggests alongside this profile.",
1134
  "replies": "Reply count reported for the comment.",
1135
  "shortcode": "Instagram shortcode that uniquely identifies the post.",
1136
+ "source_payload": "Full source response for this record.",
1137
  "snapshot_key": "Unique ID for this profile snapshot.",
1138
  "source": "Method or provider used to read this profile.",
1139
  "tagged_location": "Location tagged on the post; it is not the creator's residence.",
 
1195
  DEFAULT_POSTS_PER_PULL = 10
1196
 
1197
 
1198
+ #: The post kinds a group filter may name β€” exactly what `_bd_type` maps a vendor row onto, so a
1199
+ #: filter cannot ask for a category that can never match a stored row.
1200
+ POST_TYPES = ("video", "image", "carousel")
1201
+
1202
+
1203
+ def clean_post_groups(raw, max_posts):
1204
+ """`(groups, error)` for `config.postGroups` β€” "last N reels, last M images".
1205
+
1206
+ ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape:
1207
+ `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`.
1208
+
1209
+ β›” ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today
1210
+ carries no such key, and inventing a filter for it would silently start DROPPING posts those
1211
+ automations have always captured. Empty list and missing are the same answer.
1212
+
1213
+ ⚠ A MALFORMED GROUP IS REFUSED, not dropped β€” the opposite of `tier`/`noFallback`, and the
1214
+ difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them
1215
+ would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken
1216
+ one, so the only way to see one is a person typing it, and a filter that silently ignores the
1217
+ type you asked for is how you end up paying for reels and storing carousels.
1218
+ """
1219
+ if raw in (None, "", [], {}):
1220
+ return [], None
1221
+ if not isinstance(raw, list):
1222
+ return None, "the post groups have to be a list of {type, limit} entries"
1223
+ out, seen = [], set()
1224
+ for item in raw:
1225
+ if not isinstance(item, dict):
1226
+ return None, "each post group has to be a {type, limit} entry"
1227
+ t = str(item.get("type") or "").strip().lower()
1228
+ if t not in POST_TYPES:
1229
+ return None, (f"'{t or 'blank'}' is not a post type β€” use one of "
1230
+ f"{', '.join(POST_TYPES)}")
1231
+ if t in seen:
1232
+ return None, f"the post groups name '{t}' twice β€” one limit per type"
1233
+ seen.add(t)
1234
+ try:
1235
+ lim = int(item.get("limit"))
1236
+ except (TypeError, ValueError):
1237
+ return None, f"how many {t} posts to keep has to be a whole number"
1238
+ if lim < 1:
1239
+ return None, f"a {t} group has to keep at least one post"
1240
+ # β›” BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures
1241
+ # 12 is not an error a person can act on β€” it is a promise the run cannot keep β€” so it is
1242
+ # refused HERE, where the number they typed is still on the screen in front of them.
1243
+ if lim > max_posts:
1244
+ return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} "
1245
+ f"group cannot keep {lim} β€” raise the post count first")
1246
+ out.append({"type": t, "limit": lim})
1247
+ return out, None
1248
+
1249
+
1250
  def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True):
1251
  """`(maxPosts, error)` β€” refuses out-of-range rather than clamping into range.
1252
 
 
2645
  _bd_tagged_location,
2646
  _first, _ig_int,
2647
  DEFERRED_MARK,
2648
+ apify_profile,
2649
  bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready,
2650
  bd_snapshot_progress, depth_refusal,
2651
+ ig_handle, public_source, pull_profile, select_post_groups, top_up_views,
2652
  )
2653
 
2654
 
 
4469
  snap_row = {
4470
  "snapshot_key": f"{prof.get('username')}@{pulled}",
4471
  "influencer_key": prof.get("username"), "pulled_at": pulled,
4472
+ # β›” THE PUBLIC WORD, NOT THE VENDOR KEY β€” this cell is rendered in a grid column called
4473
+ # "Read via". `res["via"]` keeps the real key for the call graph and the server log.
4474
+ "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")),
4475
  "bio": _s(prof.get("bio"), 500),
4476
  "external_url": _s(prof.get("external_url"), 300),
4477
  }
 
5353
  # offers one anyway is offering a button that must refuse.
5354
  SOURCES = {
5355
  "instagram": {
5356
+ # β›” `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing
5357
+ # copy, and which provider serves a capability is a routing decision the product owns β€”
5358
+ # `providers.py` still holds the real keys, chains and costs.
5359
+ "key": "instagram", "label": "Instagram", "vendor": "Scraper",
5360
  "probe": bd_ready,
5361
  "capture": pull_profile,
5362
  "discover": bd_filter_start,
 
6177
  e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
6178
  except (TypeError, ValueError):
6179
  return None, "the number of days has to be a number"
6180
+ groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts)
6181
+ if gerr:
6182
+ return None, gerr
6183
  return {"profileField": _s(cfg.get("profileField"), 80).strip(),
6184
  "fromView": _s(cfg.get("fromView"), 80).strip(),
6185
  "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT,
 
6200
  # until the user intentionally enables it.
6201
  "commentMetrics": bool(cfg.get("commentMetrics")),
6202
  "dryRun": bool(cfg.get("dryRun")),
6203
+ # ⭐ 2026-08-09 (owner) β€” "last 12 reels / 12 videos", per GROUP. Absent = OFF, so
6204
+ # every action stored before today keeps saving and keeps running unchanged.
6205
+ "postGroups": groups,
6206
  "maxPosts": max_posts}, None
6207
  if kind == "find_records":
6208
  table = _s(cfg.get("table"), 60).strip()
 
6967
  pending_profile=(pend_prof
6968
  if not cfg.get("dryRun") else None),
6969
  prefetch=enrich_prefetch)
6970
+ # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels /
6971
+ # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is
6972
+ # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped.
6973
+ # β›” DEFAULT OFF β€” with no `postGroups` the list is returned unchanged, so this
6974
+ # costs nothing and changes nothing until somebody configures it.
6975
+ # ⚠ IT FILTERS THE WINDOW WE ALREADY BOUGHT, it does not widen it. The profile
6976
+ # read is capped at 12 posts (MEASURED: asked 40, got 12), so "12 reels AND 12
6977
+ # images" needs `bd_profile_posts` β€” which is built and proven (30 rows in 90 s)
6978
+ # but deliberately NOT wired here: the views top-up runs INSIDE `pull_profile`
6979
+ # against the posts it captured, so swapping a wider list in afterwards would
6980
+ # hand back posts with no view counts. Widening belongs ahead of that step, in
6981
+ # the paid path, and that is its own change. See DEBT.
6982
+ if cfg.get("postGroups"):
6983
+ res = {**res, "posts": select_post_groups(res.get("posts") or [],
6984
+ cfg.get("postGroups"))}
6985
  for _p in pend_prof:
6986
  _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso()
6987
  enrich["pendingProfiles"].extend(pend_prof)
 
7422
  _progress_memo[sid] = bd_snapshot_progress(sid)
7423
  return _progress_memo[sid]
7424
 
7425
+ def _store_profile(task, profile, via):
7426
+ """Write ONE collected profile β€” snapshot row + preset cells on the record that asked.
7427
+
7428
+ ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the
7429
+ two paths cannot drift into writing different things β€” the same rule `top_up_views`
7430
+ follows for the inline/deferred split.
7431
+ """
7432
+ res = {"state": "ok", "profile": profile, "posts": [], "comments": [],
7433
+ "via": via, "note": ""}
7434
+ pulled = _iso()
7435
+ snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
7436
+ acc["snaps"].append(snap_row)
7437
+ acc["posts"].extend(idents)
7438
+ acc["psnaps"].extend(metric_rows)
7439
+ acc["comments"].extend(comment_rows)
7440
+ _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled))
7441
+ affected.append(task["rowId"])
7442
+
7443
+ def _backup_profile(handle):
7444
+ """The second rung, asked ONLY once the first has finished and delivered nothing.
7445
+
7446
+ ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS
7447
+ CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but
7448
+ a profile that DEFERRED never came back through `pull_profile` β€” it came back here, and
7449
+ this collector had no second rung at all. So the one path where the primary is most
7450
+ likely to have failed was the one path with no fallback.
7451
+
7452
+ β›” NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary
7453
+ answered is never re-bought from a second vendor β€” the mistake the capability split exists
7454
+ to prevent. A refusal returns None and the caller reports blocked exactly as before.
7455
+ """
7456
+ try:
7457
+ import providers as _p
7458
+ if not _p.PROVIDERS["apify"].can("ig_profile"):
7459
+ return None
7460
+ prof, note = apify_profile(str(handle))
7461
+ except Exception as exc: # noqa: BLE001 β€” a backup must not raise
7462
+ log(f"[aios-auto] backup profile rung failed for @{handle}: "
7463
+ f"{type(exc).__name__}: {exc}")
7464
+ return None
7465
+ if prof and prof.get("followers") is not None:
7466
+ return prof
7467
+ return None
7468
+
7469
  def _rows_of(sid):
7470
  if sid not in _rows_memo:
7471
  payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
 
7481
  stale_h = _hours_since(task.get("requestedAt"))
7482
  state, records, empty_note = _progress_of(task["snapshotId"])
7483
  if state in ("done", "failed") and not records:
 
7484
  acc["profiles"] += 1
7485
+ # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing,
7486
+ # so there is nothing left to wait for and no risk of buying the same record twice.
7487
+ backup = _backup_profile(task["influencer"])
7488
+ if backup is not None:
7489
+ ready += 1
7490
+ acc["ok"] += 1
7491
+ _store_profile(task, backup, "apify")
7492
+ note = (f"@{task['influencer']}: the primary source returned nothing, so a backup "
7493
+ f"source supplied the profile ({backup.get('followers')} followers)")
7494
+ notes.append(note)
7495
+ acc["notes"].append(note)
7496
+ continue
7497
+ closed += 1
7498
  acc["blocked"] += 1
7499
  note = f"@{task['influencer']}: {_s(empty_note, 240)}"
7500
  notes.append(note)
 
7552
  acc["notes"].append(note)
7553
  continue
7554
  acc["ok"] += 1
7555
+ _store_profile(task, profile, "brightdata:deferred")
 
 
 
 
 
 
 
 
 
7556
  notes.append(f"@{task['influencer']}: collected the profile the source had already been "
7557
  f"paid for ({profile.get('followers')} followers)")
7558
 
api/connectors_ig.py CHANGED
@@ -883,6 +883,105 @@ def bd_scrape(dataset_id, urls, wait=None, deferred=None):
883
  PROFILE_BATCH_URLS = max(1, int(os.environ.get("AIOS_BD_PROFILE_BATCH") or 5))
884
 
885
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
886
  def bd_profiles_batch(handles, deferred=None, wait=None):
887
  """Read MANY profiles per vendor call. Returns `({handle: node}, note)`.
888
 
@@ -1236,13 +1335,51 @@ def top_up_views(posts, log=print):
1236
  target["tagged_location"] = row["tagged_location"]
1237
  used = next((a.provider for a in attempts if a.ok), "")
1238
  if used:
1239
- return f"view counts via {used}"
 
 
 
 
1240
  if attempts:
1241
- return ("no provider could supply view counts: "
1242
- + "; ".join(f"{a.provider} {a.note}" for a in attempts))
 
 
1243
  return ""
1244
 
1245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1246
  #: Marker a caller puts in the prefetch map for a handle whose BATCH chunk was deferred.
1247
  #: β›” ITS JOB IS TO STOP A SECOND PURCHASE. The snapshot is paid for and already queued for
1248
  #: collection; a rung that saw "not in the prefetch" and scraped again would bill the same handle
@@ -1813,8 +1950,8 @@ def pull_profile(url, max_posts=None, log=print,
1813
  log(f" apify: profile ok ({a_prof.get('followers')} followers)")
1814
  return {"state": "ok", "profile": a_prof, "posts": [], "comments": [],
1815
  "via": "apify",
1816
- "note": ("Bright Data could not read this profile "
1817
- f"({_s(paid.get('note'), 90)}), so it came from Apify")}
1818
  # ⭐ A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here and
1819
  # carried to the return below, so the run can say the one thing that ends the loop
1820
  # instead of the fourth variation on "we could not read it".
 
883
  PROFILE_BATCH_URLS = max(1, int(os.environ.get("AIOS_BD_PROFILE_BATCH") or 5))
884
 
885
 
886
+ #: The widest post window a single enrichment may buy. A ceiling on SPEND, not a vendor limit.
887
+ #: ⭐ MEASURED 2026-08-09 (@theresalearns): `num_of_posts: 30` on the documented discover-by-url
888
+ #: route delivered exactly 30 rows in 90 s β€” 23 Reels + 7 Carousels. The long-standing "12" is the
889
+ #: PROFILE route's cap (asked for 40 there, got 12), never the vendor's.
890
+ MAX_POST_WINDOW = int(os.environ.get("AIOS_IG_MAX_POST_WINDOW") or 60)
891
+
892
+
893
+ def bd_profile_posts(handle, count, deferred=None, wait=None):
894
+ """A profile's LAST `count` posts via discover-by-url. `(rows, note)`.
895
+
896
+ β›” A DIFFERENT ROUTE FROM `pull_profile`, AND THAT IS THE WHOLE POINT. The profile record
897
+ embeds at most 12 posts however many you ask for β€” measured, not assumed. This is the
898
+ documented discovery contract (`type=discover_new&discover_by=url`, `num_of_posts: N`), the
899
+ only route that can answer "the last 30 posts", which is what makes a per-group filter
900
+ ("last 12 reels AND last 12 images") expressible at all.
901
+
902
+ ⚠ IT COSTS PER RECORD. 30 posts is ~2.5x the spend of 12, so no caller should reach for this
903
+ because wider is nicer β€” `MAX_POST_WINDOW` bounds it and the automation's own config decides.
904
+ """
905
+ try:
906
+ n = max(1, min(int(count), MAX_POST_WINDOW))
907
+ except (TypeError, ValueError):
908
+ n = 12
909
+ payload, note = bd_call(BD_PATH_TRIGGER,
910
+ {"dataset_id": BD_DS_POSTS, "type": "discover_new",
911
+ "discover_by": "url"},
912
+ body=[{"url": f"https://www.instagram.com/{ig_handle(handle) or handle}/",
913
+ "num_of_posts": n}])
914
+ if note:
915
+ return [], note
916
+ sid = _bd_deferral(payload) or str((payload or {}).get("snapshot_id") or "")
917
+ if not sid:
918
+ return _bd_rows(payload), ""
919
+ budget = BD_SCRAPE_WAIT if wait is None else float(wait)
920
+ waited = 0.0
921
+ while True:
922
+ state, records, empty_note = bd_snapshot_progress(sid)
923
+ if state in ("done", "failed") and not records:
924
+ return [], empty_note
925
+ if state != "running":
926
+ got, gnote = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
927
+ if not gnote:
928
+ rows = _bd_rows(got)
929
+ if rows and not _bd_deferral(got) and not (
930
+ len(rows) == 1 and str(rows[0].get("status") or "") in
931
+ ("running", "building", "collecting")):
932
+ return rows, ""
933
+ if waited >= budget:
934
+ break
935
+ time.sleep(BD_SCRAPE_POLL)
936
+ waited += BD_SCRAPE_POLL
937
+ if isinstance(deferred, list):
938
+ deferred.append({"snapshotId": sid, "datasetId": str(BD_DS_POSTS),
939
+ "urls": [f"https://www.instagram.com/{handle}/"], "kind": "posts"})
940
+ return [], (f"the post source deferred this request to {sid} and it was not ready within "
941
+ f"{int(budget)}s β€” the records are collected, not lost")
942
+
943
+
944
+ def select_post_groups(posts, groups):
945
+ """Keep the newest `limit` posts of each named TYPE. Returns the kept posts, newest first.
946
+
947
+ ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). `groups` is
948
+ `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. A type nobody named is
949
+ DROPPED, which is the point of a filter β€” asking for reels and getting carousels back is the
950
+ behaviour this replaces.
951
+
952
+ ⚠ NEWEST-FIRST BY `posted_at`, and a post with no date sorts LAST rather than first. A blank
953
+ date is not "just now"; ranking it as though it were would silently prefer exactly the rows we
954
+ know least about ([[sentinel-in-a-sort-key]] β€” partition, never rank a sentinel).
955
+ """
956
+ wanted = {}
957
+ for g in (groups or []):
958
+ if not isinstance(g, dict):
959
+ continue
960
+ t = str(g.get("type") or "").strip().lower()
961
+ try:
962
+ lim = int(g.get("limit"))
963
+ except (TypeError, ValueError):
964
+ continue
965
+ if t and lim > 0:
966
+ wanted[t] = max(wanted.get(t, 0), lim)
967
+ if not wanted:
968
+ return list(posts or [])
969
+ dated, undated = [], []
970
+ for p in (posts or []):
971
+ (dated if str((p or {}).get("posted_at") or "").strip() else undated).append(p)
972
+ dated.sort(key=lambda p: str(p.get("posted_at") or ""), reverse=True)
973
+ kept, seen = [], {}
974
+ for p in dated + undated:
975
+ t = str((p or {}).get("type") or "").strip().lower()
976
+ if t not in wanted:
977
+ continue
978
+ if seen.get(t, 0) >= wanted[t]:
979
+ continue
980
+ seen[t] = seen.get(t, 0) + 1
981
+ kept.append(p)
982
+ return kept
983
+
984
+
985
  def bd_profiles_batch(handles, deferred=None, wait=None):
986
  """Read MANY profiles per vendor call. Returns `({handle: node}, note)`.
987
 
 
1335
  target["tagged_location"] = row["tagged_location"]
1336
  used = next((a.provider for a in attempts if a.ok), "")
1337
  if used:
1338
+ # β›” NO VENDOR NAME IN AN OPERATOR-FACING SENTENCE (owner instruction 2026-08-09). Which
1339
+ # company answered is a routing detail the product owns; the person reading a run log
1340
+ # wants to know the view counts arrived. The provider key is still on `attempts` and in
1341
+ # the server log for debugging β€” it just stops being product copy. See `public_source`.
1342
+ return "view counts collected"
1343
  if attempts:
1344
+ # The REASONS still ride, because a person needs to know why a column is blank β€” only the
1345
+ # vendor's NAME is dropped, never its explanation.
1346
+ reasons = "; ".join(a.note for a in attempts if a.note)
1347
+ return f"view counts could not be collected{': ' + reasons if reasons else ''}"
1348
  return ""
1349
 
1350
 
1351
+ #: β›” THE ONE PLACE A VENDOR KEY BECOMES OPERATOR-FACING TEXT (owner instruction 2026-08-09:
1352
+ #: *"Remove any mention of APIfy or Bright Data from the user interface, places like Logs etc."*).
1353
+ #:
1354
+ #: The keys stay REAL everywhere they do work β€” `providers.py` chains, `AIOS_PROVIDER_ORDER`, the
1355
+ #: env var names, `res["via"]` in the call graph, and the server log. What changes is only what a
1356
+ #: person reads: which company we bought a record from is a routing decision the product owns, and
1357
+ #: naming it in a run log invites "why did this one use the other one" about a choice nobody made.
1358
+ #: ⚠ ROUTING, NOT QUALITY. "Primary"/"Backup" says which rung answered β€” the thing the column
1359
+ #: labelled *Read via* exists to record β€” without the brand. An unknown key falls through to a
1360
+ #: generic word rather than leaking itself.
1361
+ PUBLIC_SOURCE = {
1362
+ "brightdata": "Primary",
1363
+ "brightdata:deferred": "Primary (deferred)",
1364
+ "apify": "Backup",
1365
+ }
1366
+
1367
+
1368
+ def public_source(via):
1369
+ """A vendor key -> the word an operator reads. Never returns the key itself."""
1370
+ key = str(via or "").strip().lower()
1371
+ if not key:
1372
+ return ""
1373
+ if key in PUBLIC_SOURCE:
1374
+ return PUBLIC_SOURCE[key]
1375
+ # A compound we have not mapped ("x:deferred") still must not leak the vendor half.
1376
+ base = key.split(":", 1)[0]
1377
+ if base in PUBLIC_SOURCE:
1378
+ suffix = key.split(":", 1)[1] if ":" in key else ""
1379
+ return f"{PUBLIC_SOURCE[base]} ({suffix})" if suffix else PUBLIC_SOURCE[base]
1380
+ return "Automated"
1381
+
1382
+
1383
  #: Marker a caller puts in the prefetch map for a handle whose BATCH chunk was deferred.
1384
  #: β›” ITS JOB IS TO STOP A SECOND PURCHASE. The snapshot is paid for and already queued for
1385
  #: collection; a rung that saw "not in the prefetch" and scraped again would bill the same handle
 
1950
  log(f" apify: profile ok ({a_prof.get('followers')} followers)")
1951
  return {"state": "ok", "profile": a_prof, "posts": [], "comments": [],
1952
  "via": "apify",
1953
+ "note": ("the primary source could not read this profile "
1954
+ f"({_s(paid.get('note'), 90)}), so a backup source was used")}
1955
  # ⭐ A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here and
1956
  # carried to the return below, so the run can say the one thing that ends the loop
1957
  # instead of the fourth variation on "we could not read it".