Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +141 -15
- api/connectors_ig.py +142 -5
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "cf11549",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
cf11549
|
api/automation_engine.py
CHANGED
|
@@ -1133,7 +1133,7 @@ IG_FIELD_DESCRIPTIONS = {
|
|
| 1133 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1134 |
"replies": "Reply count reported for the comment.",
|
| 1135 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
| 1136 |
-
"source_payload": "Full
|
| 1137 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1138 |
"source": "Method or provider used to read this profile.",
|
| 1139 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
|
@@ -1195,6 +1195,58 @@ MAX_POSTS_PER_PULL = 12
|
|
| 1195 |
DEFAULT_POSTS_PER_PULL = 10
|
| 1196 |
|
| 1197 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1198 |
def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True):
|
| 1199 |
"""`(maxPosts, error)` β refuses out-of-range rather than clamping into range.
|
| 1200 |
|
|
@@ -2593,9 +2645,10 @@ from connectors_ig import ( # noqa: E402
|
|
| 2593 |
_bd_tagged_location,
|
| 2594 |
_first, _ig_int,
|
| 2595 |
DEFERRED_MARK,
|
|
|
|
| 2596 |
bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready,
|
| 2597 |
bd_snapshot_progress, depth_refusal,
|
| 2598 |
-
ig_handle, pull_profile, top_up_views,
|
| 2599 |
)
|
| 2600 |
|
| 2601 |
|
|
@@ -4416,7 +4469,9 @@ def capture_rows(res, pulled):
|
|
| 4416 |
snap_row = {
|
| 4417 |
"snapshot_key": f"{prof.get('username')}@{pulled}",
|
| 4418 |
"influencer_key": prof.get("username"), "pulled_at": pulled,
|
| 4419 |
-
|
|
|
|
|
|
|
| 4420 |
"bio": _s(prof.get("bio"), 500),
|
| 4421 |
"external_url": _s(prof.get("external_url"), 300),
|
| 4422 |
}
|
|
@@ -5298,7 +5353,10 @@ RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db,
|
|
| 5298 |
# offers one anyway is offering a button that must refuse.
|
| 5299 |
SOURCES = {
|
| 5300 |
"instagram": {
|
| 5301 |
-
|
|
|
|
|
|
|
|
|
|
| 5302 |
"probe": bd_ready,
|
| 5303 |
"capture": pull_profile,
|
| 5304 |
"discover": bd_filter_start,
|
|
@@ -6119,6 +6177,9 @@ def _clean_action_config(kind, cfg, depth, seen, count):
|
|
| 6119 |
e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
|
| 6120 |
except (TypeError, ValueError):
|
| 6121 |
return None, "the number of days has to be a number"
|
|
|
|
|
|
|
|
|
|
| 6122 |
return {"profileField": _s(cfg.get("profileField"), 80).strip(),
|
| 6123 |
"fromView": _s(cfg.get("fromView"), 80).strip(),
|
| 6124 |
"sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT,
|
|
@@ -6139,6 +6200,9 @@ def _clean_action_config(kind, cfg, depth, seen, count):
|
|
| 6139 |
# until the user intentionally enables it.
|
| 6140 |
"commentMetrics": bool(cfg.get("commentMetrics")),
|
| 6141 |
"dryRun": bool(cfg.get("dryRun")),
|
|
|
|
|
|
|
|
|
|
| 6142 |
"maxPosts": max_posts}, None
|
| 6143 |
if kind == "find_records":
|
| 6144 |
table = _s(cfg.get("table"), 60).strip()
|
|
@@ -6903,6 +6967,21 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 6903 |
pending_profile=(pend_prof
|
| 6904 |
if not cfg.get("dryRun") else None),
|
| 6905 |
prefetch=enrich_prefetch)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6906 |
for _p in pend_prof:
|
| 6907 |
_p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso()
|
| 6908 |
enrich["pendingProfiles"].extend(pend_prof)
|
|
@@ -7343,6 +7422,50 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
|
|
| 7343 |
_progress_memo[sid] = bd_snapshot_progress(sid)
|
| 7344 |
return _progress_memo[sid]
|
| 7345 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7346 |
def _rows_of(sid):
|
| 7347 |
if sid not in _rows_memo:
|
| 7348 |
payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
|
|
@@ -7358,8 +7481,20 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
|
|
| 7358 |
stale_h = _hours_since(task.get("requestedAt"))
|
| 7359 |
state, records, empty_note = _progress_of(task["snapshotId"])
|
| 7360 |
if state in ("done", "failed") and not records:
|
| 7361 |
-
closed += 1
|
| 7362 |
acc["profiles"] += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7363 |
acc["blocked"] += 1
|
| 7364 |
note = f"@{task['influencer']}: {_s(empty_note, 240)}"
|
| 7365 |
notes.append(note)
|
|
@@ -7417,16 +7552,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print
|
|
| 7417 |
acc["notes"].append(note)
|
| 7418 |
continue
|
| 7419 |
acc["ok"] += 1
|
| 7420 |
-
|
| 7421 |
-
"via": "brightdata:deferred", "note": ""}
|
| 7422 |
-
pulled = _iso()
|
| 7423 |
-
snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
|
| 7424 |
-
acc["snaps"].append(snap_row)
|
| 7425 |
-
acc["posts"].extend(idents)
|
| 7426 |
-
acc["psnaps"].extend(metric_rows)
|
| 7427 |
-
acc["comments"].extend(comment_rows)
|
| 7428 |
-
_act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled))
|
| 7429 |
-
affected.append(task["rowId"])
|
| 7430 |
notes.append(f"@{task['influencer']}: collected the profile the source had already been "
|
| 7431 |
f"paid for ({profile.get('followers')} followers)")
|
| 7432 |
|
|
|
|
| 1133 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1134 |
"replies": "Reply count reported for the comment.",
|
| 1135 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
| 1136 |
+
"source_payload": "Full source response for this record.",
|
| 1137 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1138 |
"source": "Method or provider used to read this profile.",
|
| 1139 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
|
|
|
| 1195 |
DEFAULT_POSTS_PER_PULL = 10
|
| 1196 |
|
| 1197 |
|
| 1198 |
+
#: The post kinds a group filter may name β exactly what `_bd_type` maps a vendor row onto, so a
|
| 1199 |
+
#: filter cannot ask for a category that can never match a stored row.
|
| 1200 |
+
POST_TYPES = ("video", "image", "carousel")
|
| 1201 |
+
|
| 1202 |
+
|
| 1203 |
+
def clean_post_groups(raw, max_posts):
|
| 1204 |
+
"""`(groups, error)` for `config.postGroups` β "last N reels, last M images".
|
| 1205 |
+
|
| 1206 |
+
β 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape:
|
| 1207 |
+
`[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`.
|
| 1208 |
+
|
| 1209 |
+
β ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today
|
| 1210 |
+
carries no such key, and inventing a filter for it would silently start DROPPING posts those
|
| 1211 |
+
automations have always captured. Empty list and missing are the same answer.
|
| 1212 |
+
|
| 1213 |
+
β A MALFORMED GROUP IS REFUSED, not dropped β the opposite of `tier`/`noFallback`, and the
|
| 1214 |
+
difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them
|
| 1215 |
+
would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken
|
| 1216 |
+
one, so the only way to see one is a person typing it, and a filter that silently ignores the
|
| 1217 |
+
type you asked for is how you end up paying for reels and storing carousels.
|
| 1218 |
+
"""
|
| 1219 |
+
if raw in (None, "", [], {}):
|
| 1220 |
+
return [], None
|
| 1221 |
+
if not isinstance(raw, list):
|
| 1222 |
+
return None, "the post groups have to be a list of {type, limit} entries"
|
| 1223 |
+
out, seen = [], set()
|
| 1224 |
+
for item in raw:
|
| 1225 |
+
if not isinstance(item, dict):
|
| 1226 |
+
return None, "each post group has to be a {type, limit} entry"
|
| 1227 |
+
t = str(item.get("type") or "").strip().lower()
|
| 1228 |
+
if t not in POST_TYPES:
|
| 1229 |
+
return None, (f"'{t or 'blank'}' is not a post type β use one of "
|
| 1230 |
+
f"{', '.join(POST_TYPES)}")
|
| 1231 |
+
if t in seen:
|
| 1232 |
+
return None, f"the post groups name '{t}' twice β one limit per type"
|
| 1233 |
+
seen.add(t)
|
| 1234 |
+
try:
|
| 1235 |
+
lim = int(item.get("limit"))
|
| 1236 |
+
except (TypeError, ValueError):
|
| 1237 |
+
return None, f"how many {t} posts to keep has to be a whole number"
|
| 1238 |
+
if lim < 1:
|
| 1239 |
+
return None, f"a {t} group has to keep at least one post"
|
| 1240 |
+
# β BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures
|
| 1241 |
+
# 12 is not an error a person can act on β it is a promise the run cannot keep β so it is
|
| 1242 |
+
# refused HERE, where the number they typed is still on the screen in front of them.
|
| 1243 |
+
if lim > max_posts:
|
| 1244 |
+
return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} "
|
| 1245 |
+
f"group cannot keep {lim} β raise the post count first")
|
| 1246 |
+
out.append({"type": t, "limit": lim})
|
| 1247 |
+
return out, None
|
| 1248 |
+
|
| 1249 |
+
|
| 1250 |
def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True):
|
| 1251 |
"""`(maxPosts, error)` β refuses out-of-range rather than clamping into range.
|
| 1252 |
|
|
|
|
| 2645 |
_bd_tagged_location,
|
| 2646 |
_first, _ig_int,
|
| 2647 |
DEFERRED_MARK,
|
| 2648 |
+
apify_profile,
|
| 2649 |
bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready,
|
| 2650 |
bd_snapshot_progress, depth_refusal,
|
| 2651 |
+
ig_handle, public_source, pull_profile, select_post_groups, top_up_views,
|
| 2652 |
)
|
| 2653 |
|
| 2654 |
|
|
|
|
| 4469 |
snap_row = {
|
| 4470 |
"snapshot_key": f"{prof.get('username')}@{pulled}",
|
| 4471 |
"influencer_key": prof.get("username"), "pulled_at": pulled,
|
| 4472 |
+
# β THE PUBLIC WORD, NOT THE VENDOR KEY β this cell is rendered in a grid column called
|
| 4473 |
+
# "Read via". `res["via"]` keeps the real key for the call graph and the server log.
|
| 4474 |
+
"source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")),
|
| 4475 |
"bio": _s(prof.get("bio"), 500),
|
| 4476 |
"external_url": _s(prof.get("external_url"), 300),
|
| 4477 |
}
|
|
|
|
| 5353 |
# offers one anyway is offering a button that must refuse.
|
| 5354 |
SOURCES = {
|
| 5355 |
"instagram": {
|
| 5356 |
+
# β `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing
|
| 5357 |
+
# copy, and which provider serves a capability is a routing decision the product owns β
|
| 5358 |
+
# `providers.py` still holds the real keys, chains and costs.
|
| 5359 |
+
"key": "instagram", "label": "Instagram", "vendor": "Scraper",
|
| 5360 |
"probe": bd_ready,
|
| 5361 |
"capture": pull_profile,
|
| 5362 |
"discover": bd_filter_start,
|
|
|
|
| 6177 |
e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
|
| 6178 |
except (TypeError, ValueError):
|
| 6179 |
return None, "the number of days has to be a number"
|
| 6180 |
+
groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts)
|
| 6181 |
+
if gerr:
|
| 6182 |
+
return None, gerr
|
| 6183 |
return {"profileField": _s(cfg.get("profileField"), 80).strip(),
|
| 6184 |
"fromView": _s(cfg.get("fromView"), 80).strip(),
|
| 6185 |
"sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT,
|
|
|
|
| 6200 |
# until the user intentionally enables it.
|
| 6201 |
"commentMetrics": bool(cfg.get("commentMetrics")),
|
| 6202 |
"dryRun": bool(cfg.get("dryRun")),
|
| 6203 |
+
# β 2026-08-09 (owner) β "last 12 reels / 12 videos", per GROUP. Absent = OFF, so
|
| 6204 |
+
# every action stored before today keeps saving and keeps running unchanged.
|
| 6205 |
+
"postGroups": groups,
|
| 6206 |
"maxPosts": max_posts}, None
|
| 6207 |
if kind == "find_records":
|
| 6208 |
table = _s(cfg.get("table"), 60).strip()
|
|
|
|
| 6967 |
pending_profile=(pend_prof
|
| 6968 |
if not cfg.get("dryRun") else None),
|
| 6969 |
prefetch=enrich_prefetch)
|
| 6970 |
+
# β 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels /
|
| 6971 |
+
# 12 videos etc β¦ not just last 12 but by group also"*). `config.postGroups` is
|
| 6972 |
+
# `[{"type": "video", "limit": 12}, β¦]`; a type nobody names is dropped.
|
| 6973 |
+
# β DEFAULT OFF β with no `postGroups` the list is returned unchanged, so this
|
| 6974 |
+
# costs nothing and changes nothing until somebody configures it.
|
| 6975 |
+
# β IT FILTERS THE WINDOW WE ALREADY BOUGHT, it does not widen it. The profile
|
| 6976 |
+
# read is capped at 12 posts (MEASURED: asked 40, got 12), so "12 reels AND 12
|
| 6977 |
+
# images" needs `bd_profile_posts` β which is built and proven (30 rows in 90 s)
|
| 6978 |
+
# but deliberately NOT wired here: the views top-up runs INSIDE `pull_profile`
|
| 6979 |
+
# against the posts it captured, so swapping a wider list in afterwards would
|
| 6980 |
+
# hand back posts with no view counts. Widening belongs ahead of that step, in
|
| 6981 |
+
# the paid path, and that is its own change. See DEBT.
|
| 6982 |
+
if cfg.get("postGroups"):
|
| 6983 |
+
res = {**res, "posts": select_post_groups(res.get("posts") or [],
|
| 6984 |
+
cfg.get("postGroups"))}
|
| 6985 |
for _p in pend_prof:
|
| 6986 |
_p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso()
|
| 6987 |
enrich["pendingProfiles"].extend(pend_prof)
|
|
|
|
| 7422 |
_progress_memo[sid] = bd_snapshot_progress(sid)
|
| 7423 |
return _progress_memo[sid]
|
| 7424 |
|
| 7425 |
+
def _store_profile(task, profile, via):
|
| 7426 |
+
"""Write ONE collected profile β snapshot row + preset cells on the record that asked.
|
| 7427 |
+
|
| 7428 |
+
ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the
|
| 7429 |
+
two paths cannot drift into writing different things β the same rule `top_up_views`
|
| 7430 |
+
follows for the inline/deferred split.
|
| 7431 |
+
"""
|
| 7432 |
+
res = {"state": "ok", "profile": profile, "posts": [], "comments": [],
|
| 7433 |
+
"via": via, "note": ""}
|
| 7434 |
+
pulled = _iso()
|
| 7435 |
+
snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
|
| 7436 |
+
acc["snaps"].append(snap_row)
|
| 7437 |
+
acc["posts"].extend(idents)
|
| 7438 |
+
acc["psnaps"].extend(metric_rows)
|
| 7439 |
+
acc["comments"].extend(comment_rows)
|
| 7440 |
+
_act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled))
|
| 7441 |
+
affected.append(task["rowId"])
|
| 7442 |
+
|
| 7443 |
+
def _backup_profile(handle):
|
| 7444 |
+
"""The second rung, asked ONLY once the first has finished and delivered nothing.
|
| 7445 |
+
|
| 7446 |
+
β 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS
|
| 7447 |
+
CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but
|
| 7448 |
+
a profile that DEFERRED never came back through `pull_profile` β it came back here, and
|
| 7449 |
+
this collector had no second rung at all. So the one path where the primary is most
|
| 7450 |
+
likely to have failed was the one path with no fallback.
|
| 7451 |
+
|
| 7452 |
+
β NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary
|
| 7453 |
+
answered is never re-bought from a second vendor β the mistake the capability split exists
|
| 7454 |
+
to prevent. A refusal returns None and the caller reports blocked exactly as before.
|
| 7455 |
+
"""
|
| 7456 |
+
try:
|
| 7457 |
+
import providers as _p
|
| 7458 |
+
if not _p.PROVIDERS["apify"].can("ig_profile"):
|
| 7459 |
+
return None
|
| 7460 |
+
prof, note = apify_profile(str(handle))
|
| 7461 |
+
except Exception as exc: # noqa: BLE001 β a backup must not raise
|
| 7462 |
+
log(f"[aios-auto] backup profile rung failed for @{handle}: "
|
| 7463 |
+
f"{type(exc).__name__}: {exc}")
|
| 7464 |
+
return None
|
| 7465 |
+
if prof and prof.get("followers") is not None:
|
| 7466 |
+
return prof
|
| 7467 |
+
return None
|
| 7468 |
+
|
| 7469 |
def _rows_of(sid):
|
| 7470 |
if sid not in _rows_memo:
|
| 7471 |
payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
|
|
|
|
| 7481 |
stale_h = _hours_since(task.get("requestedAt"))
|
| 7482 |
state, records, empty_note = _progress_of(task["snapshotId"])
|
| 7483 |
if state in ("done", "failed") and not records:
|
|
|
|
| 7484 |
acc["profiles"] += 1
|
| 7485 |
+
# β ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing,
|
| 7486 |
+
# so there is nothing left to wait for and no risk of buying the same record twice.
|
| 7487 |
+
backup = _backup_profile(task["influencer"])
|
| 7488 |
+
if backup is not None:
|
| 7489 |
+
ready += 1
|
| 7490 |
+
acc["ok"] += 1
|
| 7491 |
+
_store_profile(task, backup, "apify")
|
| 7492 |
+
note = (f"@{task['influencer']}: the primary source returned nothing, so a backup "
|
| 7493 |
+
f"source supplied the profile ({backup.get('followers')} followers)")
|
| 7494 |
+
notes.append(note)
|
| 7495 |
+
acc["notes"].append(note)
|
| 7496 |
+
continue
|
| 7497 |
+
closed += 1
|
| 7498 |
acc["blocked"] += 1
|
| 7499 |
note = f"@{task['influencer']}: {_s(empty_note, 240)}"
|
| 7500 |
notes.append(note)
|
|
|
|
| 7552 |
acc["notes"].append(note)
|
| 7553 |
continue
|
| 7554 |
acc["ok"] += 1
|
| 7555 |
+
_store_profile(task, profile, "brightdata:deferred")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7556 |
notes.append(f"@{task['influencer']}: collected the profile the source had already been "
|
| 7557 |
f"paid for ({profile.get('followers')} followers)")
|
| 7558 |
|
api/connectors_ig.py
CHANGED
|
@@ -883,6 +883,105 @@ def bd_scrape(dataset_id, urls, wait=None, deferred=None):
|
|
| 883 |
PROFILE_BATCH_URLS = max(1, int(os.environ.get("AIOS_BD_PROFILE_BATCH") or 5))
|
| 884 |
|
| 885 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 886 |
def bd_profiles_batch(handles, deferred=None, wait=None):
|
| 887 |
"""Read MANY profiles per vendor call. Returns `({handle: node}, note)`.
|
| 888 |
|
|
@@ -1236,13 +1335,51 @@ def top_up_views(posts, log=print):
|
|
| 1236 |
target["tagged_location"] = row["tagged_location"]
|
| 1237 |
used = next((a.provider for a in attempts if a.ok), "")
|
| 1238 |
if used:
|
| 1239 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1240 |
if attempts:
|
| 1241 |
-
|
| 1242 |
-
|
|
|
|
|
|
|
| 1243 |
return ""
|
| 1244 |
|
| 1245 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1246 |
#: Marker a caller puts in the prefetch map for a handle whose BATCH chunk was deferred.
|
| 1247 |
#: β ITS JOB IS TO STOP A SECOND PURCHASE. The snapshot is paid for and already queued for
|
| 1248 |
#: collection; a rung that saw "not in the prefetch" and scraped again would bill the same handle
|
|
@@ -1813,8 +1950,8 @@ def pull_profile(url, max_posts=None, log=print,
|
|
| 1813 |
log(f" apify: profile ok ({a_prof.get('followers')} followers)")
|
| 1814 |
return {"state": "ok", "profile": a_prof, "posts": [], "comments": [],
|
| 1815 |
"via": "apify",
|
| 1816 |
-
"note": ("
|
| 1817 |
-
f"({_s(paid.get('note'), 90)}), so
|
| 1818 |
# β A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here and
|
| 1819 |
# carried to the return below, so the run can say the one thing that ends the loop
|
| 1820 |
# instead of the fourth variation on "we could not read it".
|
|
|
|
| 883 |
PROFILE_BATCH_URLS = max(1, int(os.environ.get("AIOS_BD_PROFILE_BATCH") or 5))
|
| 884 |
|
| 885 |
|
| 886 |
+
#: The widest post window a single enrichment may buy. A ceiling on SPEND, not a vendor limit.
|
| 887 |
+
#: β MEASURED 2026-08-09 (@theresalearns): `num_of_posts: 30` on the documented discover-by-url
|
| 888 |
+
#: route delivered exactly 30 rows in 90 s β 23 Reels + 7 Carousels. The long-standing "12" is the
|
| 889 |
+
#: PROFILE route's cap (asked for 40 there, got 12), never the vendor's.
|
| 890 |
+
MAX_POST_WINDOW = int(os.environ.get("AIOS_IG_MAX_POST_WINDOW") or 60)
|
| 891 |
+
|
| 892 |
+
|
| 893 |
+
def bd_profile_posts(handle, count, deferred=None, wait=None):
|
| 894 |
+
"""A profile's LAST `count` posts via discover-by-url. `(rows, note)`.
|
| 895 |
+
|
| 896 |
+
β A DIFFERENT ROUTE FROM `pull_profile`, AND THAT IS THE WHOLE POINT. The profile record
|
| 897 |
+
embeds at most 12 posts however many you ask for β measured, not assumed. This is the
|
| 898 |
+
documented discovery contract (`type=discover_new&discover_by=url`, `num_of_posts: N`), the
|
| 899 |
+
only route that can answer "the last 30 posts", which is what makes a per-group filter
|
| 900 |
+
("last 12 reels AND last 12 images") expressible at all.
|
| 901 |
+
|
| 902 |
+
β IT COSTS PER RECORD. 30 posts is ~2.5x the spend of 12, so no caller should reach for this
|
| 903 |
+
because wider is nicer β `MAX_POST_WINDOW` bounds it and the automation's own config decides.
|
| 904 |
+
"""
|
| 905 |
+
try:
|
| 906 |
+
n = max(1, min(int(count), MAX_POST_WINDOW))
|
| 907 |
+
except (TypeError, ValueError):
|
| 908 |
+
n = 12
|
| 909 |
+
payload, note = bd_call(BD_PATH_TRIGGER,
|
| 910 |
+
{"dataset_id": BD_DS_POSTS, "type": "discover_new",
|
| 911 |
+
"discover_by": "url"},
|
| 912 |
+
body=[{"url": f"https://www.instagram.com/{ig_handle(handle) or handle}/",
|
| 913 |
+
"num_of_posts": n}])
|
| 914 |
+
if note:
|
| 915 |
+
return [], note
|
| 916 |
+
sid = _bd_deferral(payload) or str((payload or {}).get("snapshot_id") or "")
|
| 917 |
+
if not sid:
|
| 918 |
+
return _bd_rows(payload), ""
|
| 919 |
+
budget = BD_SCRAPE_WAIT if wait is None else float(wait)
|
| 920 |
+
waited = 0.0
|
| 921 |
+
while True:
|
| 922 |
+
state, records, empty_note = bd_snapshot_progress(sid)
|
| 923 |
+
if state in ("done", "failed") and not records:
|
| 924 |
+
return [], empty_note
|
| 925 |
+
if state != "running":
|
| 926 |
+
got, gnote = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
|
| 927 |
+
if not gnote:
|
| 928 |
+
rows = _bd_rows(got)
|
| 929 |
+
if rows and not _bd_deferral(got) and not (
|
| 930 |
+
len(rows) == 1 and str(rows[0].get("status") or "") in
|
| 931 |
+
("running", "building", "collecting")):
|
| 932 |
+
return rows, ""
|
| 933 |
+
if waited >= budget:
|
| 934 |
+
break
|
| 935 |
+
time.sleep(BD_SCRAPE_POLL)
|
| 936 |
+
waited += BD_SCRAPE_POLL
|
| 937 |
+
if isinstance(deferred, list):
|
| 938 |
+
deferred.append({"snapshotId": sid, "datasetId": str(BD_DS_POSTS),
|
| 939 |
+
"urls": [f"https://www.instagram.com/{handle}/"], "kind": "posts"})
|
| 940 |
+
return [], (f"the post source deferred this request to {sid} and it was not ready within "
|
| 941 |
+
f"{int(budget)}s β the records are collected, not lost")
|
| 942 |
+
|
| 943 |
+
|
| 944 |
+
def select_post_groups(posts, groups):
|
| 945 |
+
"""Keep the newest `limit` posts of each named TYPE. Returns the kept posts, newest first.
|
| 946 |
+
|
| 947 |
+
β 2026-08-09 (owner: *"not just last 12 but by group also"*). `groups` is
|
| 948 |
+
`[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. A type nobody named is
|
| 949 |
+
DROPPED, which is the point of a filter β asking for reels and getting carousels back is the
|
| 950 |
+
behaviour this replaces.
|
| 951 |
+
|
| 952 |
+
β NEWEST-FIRST BY `posted_at`, and a post with no date sorts LAST rather than first. A blank
|
| 953 |
+
date is not "just now"; ranking it as though it were would silently prefer exactly the rows we
|
| 954 |
+
know least about ([[sentinel-in-a-sort-key]] β partition, never rank a sentinel).
|
| 955 |
+
"""
|
| 956 |
+
wanted = {}
|
| 957 |
+
for g in (groups or []):
|
| 958 |
+
if not isinstance(g, dict):
|
| 959 |
+
continue
|
| 960 |
+
t = str(g.get("type") or "").strip().lower()
|
| 961 |
+
try:
|
| 962 |
+
lim = int(g.get("limit"))
|
| 963 |
+
except (TypeError, ValueError):
|
| 964 |
+
continue
|
| 965 |
+
if t and lim > 0:
|
| 966 |
+
wanted[t] = max(wanted.get(t, 0), lim)
|
| 967 |
+
if not wanted:
|
| 968 |
+
return list(posts or [])
|
| 969 |
+
dated, undated = [], []
|
| 970 |
+
for p in (posts or []):
|
| 971 |
+
(dated if str((p or {}).get("posted_at") or "").strip() else undated).append(p)
|
| 972 |
+
dated.sort(key=lambda p: str(p.get("posted_at") or ""), reverse=True)
|
| 973 |
+
kept, seen = [], {}
|
| 974 |
+
for p in dated + undated:
|
| 975 |
+
t = str((p or {}).get("type") or "").strip().lower()
|
| 976 |
+
if t not in wanted:
|
| 977 |
+
continue
|
| 978 |
+
if seen.get(t, 0) >= wanted[t]:
|
| 979 |
+
continue
|
| 980 |
+
seen[t] = seen.get(t, 0) + 1
|
| 981 |
+
kept.append(p)
|
| 982 |
+
return kept
|
| 983 |
+
|
| 984 |
+
|
| 985 |
def bd_profiles_batch(handles, deferred=None, wait=None):
|
| 986 |
"""Read MANY profiles per vendor call. Returns `({handle: node}, note)`.
|
| 987 |
|
|
|
|
| 1335 |
target["tagged_location"] = row["tagged_location"]
|
| 1336 |
used = next((a.provider for a in attempts if a.ok), "")
|
| 1337 |
if used:
|
| 1338 |
+
# β NO VENDOR NAME IN AN OPERATOR-FACING SENTENCE (owner instruction 2026-08-09). Which
|
| 1339 |
+
# company answered is a routing detail the product owns; the person reading a run log
|
| 1340 |
+
# wants to know the view counts arrived. The provider key is still on `attempts` and in
|
| 1341 |
+
# the server log for debugging β it just stops being product copy. See `public_source`.
|
| 1342 |
+
return "view counts collected"
|
| 1343 |
if attempts:
|
| 1344 |
+
# The REASONS still ride, because a person needs to know why a column is blank β only the
|
| 1345 |
+
# vendor's NAME is dropped, never its explanation.
|
| 1346 |
+
reasons = "; ".join(a.note for a in attempts if a.note)
|
| 1347 |
+
return f"view counts could not be collected{': ' + reasons if reasons else ''}"
|
| 1348 |
return ""
|
| 1349 |
|
| 1350 |
|
| 1351 |
+
#: β THE ONE PLACE A VENDOR KEY BECOMES OPERATOR-FACING TEXT (owner instruction 2026-08-09:
|
| 1352 |
+
#: *"Remove any mention of APIfy or Bright Data from the user interface, places like Logs etc."*).
|
| 1353 |
+
#:
|
| 1354 |
+
#: The keys stay REAL everywhere they do work β `providers.py` chains, `AIOS_PROVIDER_ORDER`, the
|
| 1355 |
+
#: env var names, `res["via"]` in the call graph, and the server log. What changes is only what a
|
| 1356 |
+
#: person reads: which company we bought a record from is a routing decision the product owns, and
|
| 1357 |
+
#: naming it in a run log invites "why did this one use the other one" about a choice nobody made.
|
| 1358 |
+
#: β ROUTING, NOT QUALITY. "Primary"/"Backup" says which rung answered β the thing the column
|
| 1359 |
+
#: labelled *Read via* exists to record β without the brand. An unknown key falls through to a
|
| 1360 |
+
#: generic word rather than leaking itself.
|
| 1361 |
+
PUBLIC_SOURCE = {
|
| 1362 |
+
"brightdata": "Primary",
|
| 1363 |
+
"brightdata:deferred": "Primary (deferred)",
|
| 1364 |
+
"apify": "Backup",
|
| 1365 |
+
}
|
| 1366 |
+
|
| 1367 |
+
|
| 1368 |
+
def public_source(via):
|
| 1369 |
+
"""A vendor key -> the word an operator reads. Never returns the key itself."""
|
| 1370 |
+
key = str(via or "").strip().lower()
|
| 1371 |
+
if not key:
|
| 1372 |
+
return ""
|
| 1373 |
+
if key in PUBLIC_SOURCE:
|
| 1374 |
+
return PUBLIC_SOURCE[key]
|
| 1375 |
+
# A compound we have not mapped ("x:deferred") still must not leak the vendor half.
|
| 1376 |
+
base = key.split(":", 1)[0]
|
| 1377 |
+
if base in PUBLIC_SOURCE:
|
| 1378 |
+
suffix = key.split(":", 1)[1] if ":" in key else ""
|
| 1379 |
+
return f"{PUBLIC_SOURCE[base]} ({suffix})" if suffix else PUBLIC_SOURCE[base]
|
| 1380 |
+
return "Automated"
|
| 1381 |
+
|
| 1382 |
+
|
| 1383 |
#: Marker a caller puts in the prefetch map for a handle whose BATCH chunk was deferred.
|
| 1384 |
#: β ITS JOB IS TO STOP A SECOND PURCHASE. The snapshot is paid for and already queued for
|
| 1385 |
#: collection; a rung that saw "not in the prefetch" and scraped again would bill the same handle
|
|
|
|
| 1950 |
log(f" apify: profile ok ({a_prof.get('followers')} followers)")
|
| 1951 |
return {"state": "ok", "profile": a_prof, "posts": [], "comments": [],
|
| 1952 |
"via": "apify",
|
| 1953 |
+
"note": ("the primary source could not read this profile "
|
| 1954 |
+
f"({_s(paid.get('note'), 90)}), so a backup source was used")}
|
| 1955 |
# β A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here and
|
| 1956 |
# carried to the return below, so the run can say the one thing that ends the loop
|
| 1957 |
# instead of the fourth variation on "we could not read it".
|