Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +278 -181
- platform/core/user_tables.py +5 -4
- web/src/automation/AutomationBuilder.tsx +17 -1
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "f4a0171",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
f4a0171
|
api/automation_engine.py
CHANGED
|
@@ -1038,7 +1038,7 @@ IG_FIELD_DESCRIPTIONS = {
|
|
| 1038 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1039 |
"replies": "Reply count reported for the comment.",
|
| 1040 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
| 1041 |
-
"source_payload": "Full Bright Data source response
|
| 1042 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1043 |
"source": "Method or provider used to read this profile.",
|
| 1044 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
|
@@ -1231,9 +1231,12 @@ def clean_config(kind, raw, previous=None):
|
|
| 1231 |
"noFallback": flag("noFallback"),
|
| 1232 |
# β THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
|
| 1233 |
# for: the Profiles row carries post IDENTITY for free but NO engagement
|
| 1234 |
-
# (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
|
| 1235 |
-
"postMetrics": flag("postMetrics"),
|
| 1236 |
-
|
|
|
|
|
|
|
|
|
|
| 1237 |
"lanes": lanes,
|
| 1238 |
"maxPosts": max_posts}, None
|
| 1239 |
if kind == "discover_instagram":
|
|
@@ -1470,7 +1473,8 @@ def _ensure_enrich_step(flow_raw):
|
|
| 1470 |
# with only step 1 lands this at the end, which IS step 2.
|
| 1471 |
seeded.insert(1, {
|
| 1472 |
"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 1473 |
-
"config": {"tier": "anonymous", "noFallback": False, "postMetrics": False,
|
|
|
|
| 1474 |
"dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
|
| 1475 |
"fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
|
| 1476 |
"limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
|
|
@@ -2081,8 +2085,8 @@ IG_MAX_RETRIES = 2
|
|
| 2081 |
#: has exactly the same cost as delaying capture β for that column β and is justified by exactly
|
| 2082 |
#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to
|
| 2083 |
#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices.
|
| 2084 |
-
#:
|
| 2085 |
-
#:
|
| 2086 |
#: β A blank in any of these means NOT READ β never "they have none". The anonymous rungs expose
|
| 2087 |
#: almost none of them, which is what the `source` column is for.
|
| 2088 |
SNAPSHOT_FIELDS = [
|
|
@@ -2096,7 +2100,7 @@ SNAPSHOT_FIELDS = [
|
|
| 2096 |
# The link in bio β the paid rung's field, and commercially the most useful single string an
|
| 2097 |
# influencer row can carry (it is the shop/affiliate destination).
|
| 2098 |
field_def("external_url", "Link in bio", "url"),
|
| 2099 |
-
# --- the rest of the vendor's Profiles schema
|
| 2100 |
field_def("ig_id", "Instagram id"),
|
| 2101 |
field_def("profile_url", "Profile", "url"),
|
| 2102 |
field_def("avg_engagement", "Avg engagement", "pct"),
|
|
@@ -2131,7 +2135,7 @@ POST_FIELDS = [
|
|
| 2131 |
# presented as the creator's location: a creator can tag a holiday, venue or brand location.
|
| 2132 |
field_def("tagged_location", "Tagged location"),
|
| 2133 |
# The per-field map below gives operational fields first; this locked JSON document retains
|
| 2134 |
-
# every other
|
| 2135 |
# instead of being silently discarded while the canonical field model catches up.
|
| 2136 |
field_def("source_payload", "Source data", "json"),
|
| 2137 |
# Sponsored-post detection β Β§2c called it one of the fields worth having that the anonymous
|
|
@@ -2203,9 +2207,9 @@ POST_FIELDS = [
|
|
| 2203 |
#: β WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as
|
| 2204 |
#: `config.postMetrics` and the Bright Data money switch W26/R15 kept β the relation costs
|
| 2205 |
#: nothing; the spend and the third-party ingest are a click somebody makes knowingly.
|
| 2206 |
-
#:
|
| 2207 |
-
#:
|
| 2208 |
-
#:
|
| 2209 |
COMMENT_FIELDS = [
|
| 2210 |
field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
|
| 2211 |
field_def("influencer_key", "Influencer"),
|
|
@@ -2217,11 +2221,10 @@ COMMENT_FIELDS = [
|
|
| 2217 |
description="Post record linked to this comment.",
|
| 2218 |
link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
|
| 2219 |
"single": True}),
|
| 2220 |
-
#
|
| 2221 |
-
#
|
| 2222 |
-
#
|
| 2223 |
-
|
| 2224 |
-
]
|
| 2225 |
POST_SNAPSHOT_FIELDS = [
|
| 2226 |
field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
|
| 2227 |
field_def("influencer_key", "Influencer"),
|
|
@@ -2231,6 +2234,7 @@ POST_SNAPSHOT_FIELDS = [
|
|
| 2231 |
# thing to derive. Paid rung only; blank means not read.
|
| 2232 |
field_def("views", "Views", "int"),
|
| 2233 |
field_def("plays", "Plays", "int"),
|
|
|
|
| 2234 |
field_def("post_link", "Post", "link",
|
| 2235 |
description="Post record linked to this measurement.",
|
| 2236 |
link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
|
|
@@ -2382,6 +2386,7 @@ BD_BASE_DEFAULT = "https://api.brightdata.com"
|
|
| 2382 |
BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram β Profiles. 36 fields, 620M records
|
| 2383 |
BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram β Posts. 43 fields
|
| 2384 |
BD_DS_REELS = "gd_lyclm20il4r5helnj" # Instagram Reels: views and play counts
|
|
|
|
| 2385 |
BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
|
| 2386 |
BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_β¦"}
|
| 2387 |
BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
|
|
@@ -2557,30 +2562,21 @@ def _first(node, *names, default=None):
|
|
| 2557 |
return default
|
| 2558 |
|
| 2559 |
|
| 2560 |
-
#:
|
| 2561 |
-
#:
|
| 2562 |
-
#:
|
| 2563 |
-
#:
|
| 2564 |
-
#: materially. `business_email`'s own description says it "no longer exists without login!".
|
| 2565 |
-
#: R11 (owner ruling, wave 20): NO Instagram comment-text capture. β This is not merely "do not
|
| 2566 |
-
#: buy the Comments dataset" β MEASURED 2026-08-05, a POSTS row arrives with `latest_comments`
|
| 2567 |
-
#: as an array of 15 comment texts WITH their authors, whether we asked or not. The ruling is
|
| 2568 |
-
#: enforced HERE, at the map, because that is the only place it can be.
|
| 2569 |
-
BD_NEVER_MAP = frozenset({
|
| 2570 |
-
"email_address", "phone_number", "business_email", "profile_image_link",
|
| 2571 |
-
"profile_pic_url", "latest_comments", "top_comments", "comment_user",
|
| 2572 |
-
"comment_user_url", "post_user", "comment", "user_commenting", "avatar",
|
| 2573 |
-
})
|
| 2574 |
|
| 2575 |
-
# `
|
| 2576 |
-
#
|
| 2577 |
-
#
|
| 2578 |
-
|
| 2579 |
|
| 2580 |
#: The vendor's content vocabulary β ours. MEASURED values: profiles' `posts[].content_type` is
|
| 2581 |
#: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`.
|
| 2582 |
-
_BD_TYPES = {"image": "image", "
|
| 2583 |
-
"
|
|
|
|
| 2584 |
"carousel_container": "carousel", "sidecar": "carousel", "posts": "image"}
|
| 2585 |
|
| 2586 |
|
|
@@ -2686,67 +2682,89 @@ def _bd_tagged_location(node):
|
|
| 2686 |
return ", ".join(parts)[:400] or None
|
| 2687 |
|
| 2688 |
|
| 2689 |
-
def
|
| 2690 |
-
"""
|
| 2691 |
|
| 2692 |
-
|
| 2693 |
-
|
| 2694 |
-
|
| 2695 |
-
|
|
|
|
| 2696 |
"""
|
| 2697 |
if not isinstance(row, dict):
|
| 2698 |
return None
|
| 2699 |
-
public = {str(k): v for k, v in row.items() if str(k) not in BD_NEVER_MAP}
|
| 2700 |
try:
|
| 2701 |
-
packed = json.dumps(
|
|
|
|
| 2702 |
except (TypeError, ValueError):
|
| 2703 |
return None
|
| 2704 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2705 |
|
| 2706 |
|
| 2707 |
-
def
|
| 2708 |
-
"""
|
| 2709 |
|
| 2710 |
-
|
| 2711 |
-
|
| 2712 |
-
|
| 2713 |
-
|
| 2714 |
-
if not isinstance(node, dict):
|
| 2715 |
-
return None
|
| 2716 |
-
public = {str(k): v for k, v in node.items()
|
| 2717 |
-
if str(k) not in BD_NEVER_MAP and str(k) != "posts"}
|
| 2718 |
-
try:
|
| 2719 |
-
packed = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
| 2720 |
-
except (TypeError, ValueError):
|
| 2721 |
-
return None
|
| 2722 |
-
return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
|
| 2723 |
-
|
| 2724 |
-
|
| 2725 |
-
def _bd_comment(row, influencer_key="", shortcode=""):
|
| 2726 |
-
"""One Bright Data Comments response -> the complete permitted Comment record.
|
| 2727 |
-
|
| 2728 |
-
The actual Comments endpoint remains an explicit paid opt-in. This pure mapper means that
|
| 2729 |
-
when it is enabled, every allowed response field is preserved in `source_payload` rather than
|
| 2730 |
-
reintroducing a partial child-table schema. Third-party author identity and comment text stay
|
| 2731 |
-
outside both the first-class fields and the source document.
|
| 2732 |
"""
|
| 2733 |
if not isinstance(row, dict):
|
| 2734 |
return None
|
| 2735 |
url = str(_first(row, "url", "post_url", default="") or "")
|
| 2736 |
code = str(shortcode or "") or ig_shortcode(url)
|
| 2737 |
-
comment_id = str(_first(row, "comment_id", "id", "pk", default="") or "")
|
| 2738 |
if not code or not comment_id:
|
| 2739 |
return None
|
| 2740 |
-
posted = str(_first(row, "comment_date", "date_of_comment",
|
|
|
|
| 2741 |
return {
|
| 2742 |
"comment_key": comment_id,
|
| 2743 |
"shortcode": code,
|
| 2744 |
"influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
|
| 2745 |
"commented_at": posted.replace("T", " ")[:16],
|
| 2746 |
-
"likes": _ig_int(_first(row, "likes_number", "likes", "like_count")),
|
| 2747 |
-
"replies": _ig_int(_first(row, "replies_number", "num_replies", "replies"
|
| 2748 |
-
|
|
|
|
| 2749 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2750 |
|
| 2751 |
|
| 2752 |
def _bd_profile(node, handle):
|
|
@@ -2792,11 +2810,8 @@ def _bd_profile(node, handle):
|
|
| 2792 |
# `wave20-split.md`), and the first four were explicitly logged as *"populated on scrape
|
| 2793 |
# and NOT in any previous list"* β so these are not speculative names, they are fields the
|
| 2794 |
# vendor was already sending and this map was already throwing away.
|
| 2795 |
-
#
|
| 2796 |
-
#
|
| 2797 |
-
# `latest_comments` (R11) on the way out. "Every field" means every field two owner
|
| 2798 |
-
# rulings have not removed β and the filter below is what makes that true rather than
|
| 2799 |
-
# promised.
|
| 2800 |
"profile_name": str(_first(node, "profile_name", default="") or ""),
|
| 2801 |
"is_joined_recently": _bd_flag(node, "is_joined_recently"),
|
| 2802 |
"has_channel": _bd_flag(node, "has_channel"),
|
|
@@ -2816,12 +2831,9 @@ def _bd_profile(node, handle):
|
|
| 2816 |
# was `None` on all five corpus rows. It is a row field regardless, and reading it costs
|
| 2817 |
# nothing; a blank here means not read, like every other blank in this map.
|
| 2818 |
"country_code": str(_first(node, "country_code", default="") or ""),
|
| 2819 |
-
"source_payload":
|
| 2820 |
-
}
|
| 2821 |
-
|
| 2822 |
-
# field, and this asserts it stays that way after the next edit β a map is exactly the kind
|
| 2823 |
-
# of code somebody extends in a hurry.
|
| 2824 |
-
return {k: v for k, v in out.items() if k not in BD_NEVER_MAP}
|
| 2825 |
|
| 2826 |
|
| 2827 |
def _bd_post_identity(p):
|
|
@@ -2850,9 +2862,13 @@ def _bd_post_identity(p):
|
|
| 2850 |
caption = _first(p, "caption", "description", default="")
|
| 2851 |
if isinstance(caption, dict):
|
| 2852 |
caption = _first(caption, "text", default="")
|
| 2853 |
-
if caption:
|
| 2854 |
-
out["caption"] = str(caption)
|
| 2855 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2856 |
|
| 2857 |
|
| 2858 |
def _bd_post_metrics(row):
|
|
@@ -2867,7 +2883,8 @@ def _bd_post_metrics(row):
|
|
| 2867 |
is declared `text`, the others `number`). A candidate list may span types only when the
|
| 2868 |
MEANING is identical β never the way `like_and_view_counts_disabled`, a boolean, would have
|
| 2869 |
turned "likes are hidden" into "1 like".
|
| 2870 |
-
|
|
|
|
| 2871 |
"""
|
| 2872 |
if not isinstance(row, dict):
|
| 2873 |
return None
|
|
@@ -2899,12 +2916,12 @@ def _bd_post_metrics(row):
|
|
| 2899 |
"hashtags": _bd_list(row, "hashtags"),
|
| 2900 |
"alt_text": str(_first(row, "alt_text", default="") or ""),
|
| 2901 |
"tagged_location": _bd_tagged_location(row),
|
| 2902 |
-
"source_payload":
|
| 2903 |
-
}
|
| 2904 |
-
|
| 2905 |
-
|
| 2906 |
-
|
| 2907 |
-
return
|
| 2908 |
|
| 2909 |
|
| 2910 |
def _bd_rows(payload):
|
|
@@ -3057,21 +3074,23 @@ def bd_corpus_profile(handle, log=print):
|
|
| 3057 |
f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building β it is not lost)")
|
| 3058 |
|
| 3059 |
|
| 3060 |
-
def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False,
|
|
|
|
| 3061 |
"""The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides.
|
| 3062 |
|
| 3063 |
Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner
|
| 3064 |
treats every rung identically. `blocked` here means the VENDOR rung did not answer β the
|
| 3065 |
caller decides whether to drop to the anonymous ladder.
|
| 3066 |
|
| 3067 |
-
`post_metrics=True` adds the
|
| 3068 |
-
the
|
| 3069 |
-
|
|
|
|
| 3070 |
"""
|
| 3071 |
handle = ig_handle(url)
|
| 3072 |
if not handle:
|
| 3073 |
-
return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
|
| 3074 |
-
"profile": {}, "posts": [], "via": ""}
|
| 3075 |
rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"])
|
| 3076 |
node = rows[0] if rows else {}
|
| 3077 |
profile = _bd_profile(node, handle) if node else {}
|
|
@@ -3106,13 +3125,13 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
|
|
| 3106 |
f"the vendor's pre-collected profile corpus β the values are its last "
|
| 3107 |
f"capture, not a read taken just now")
|
| 3108 |
else:
|
| 3109 |
-
return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
|
| 3110 |
"note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"}
|
| 3111 |
if profile.get("followers") is None and profile.get("following") is None:
|
| 3112 |
# It answered 200 with something we could not read. Say THAT β not "0 followers".
|
| 3113 |
# β `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and
|
| 3114 |
# is discarded, so a row where it is the only "readable" field is a row we cannot read.
|
| 3115 |
-
return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
|
| 3116 |
"note": "the profile source answered, but no follower/following counts were readable in "
|
| 3117 |
"it (the field names may have moved β see instagram-capture.md)"}
|
| 3118 |
posts = []
|
|
@@ -3129,10 +3148,11 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
|
|
| 3129 |
# without duplicating any record, so enabling post metrics still buys one row per post.
|
| 3130 |
ordinary = [p for p in posts if p.get("type") != "video"]
|
| 3131 |
videos = [p for p in posts if p.get("type") == "video"]
|
| 3132 |
-
rows2, metric_notes = [], []
|
| 3133 |
if ordinary:
|
| 3134 |
got_rows, got_note = bd_scrape(BD_DS_POSTS, [p["url"] for p in ordinary])
|
| 3135 |
rows2.extend(got_rows)
|
|
|
|
| 3136 |
if got_note:
|
| 3137 |
metric_notes.append(f"posts: {got_note}")
|
| 3138 |
if videos:
|
|
@@ -3140,6 +3160,21 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
|
|
| 3140 |
rows2.extend(got_rows)
|
| 3141 |
if got_note:
|
| 3142 |
metric_notes.append(f"reels: {got_note}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3143 |
if metric_notes and not rows2:
|
| 3144 |
# The engagement half refusing does NOT lose the identity half β the profile and the
|
| 3145 |
# post rows still land, and the run says which part was not readable.
|
|
@@ -3160,6 +3195,23 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
|
|
| 3160 |
note = "post metrics were requested but the Posts dataset returned no rows"
|
| 3161 |
elif metric_notes:
|
| 3162 |
note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3163 |
|
| 3164 |
# IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften
|
| 3165 |
# because we are paying: a run that wrote a follower count and no posts must not paint green
|
|
@@ -3168,8 +3220,9 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
|
|
| 3168 |
# snapshot row's `source` column, so a stored measurement can always say whether it was read
|
| 3169 |
# live or taken from the vendor's pre-collected corpus. Two routes writing one indisting-
|
| 3170 |
# uishable `source` would make the series unauditable exactly where it matters most.
|
| 3171 |
-
return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts,
|
| 3172 |
-
"
|
|
|
|
| 3173 |
"note": note or ("" if posts else "profile read; no posts were returned")}
|
| 3174 |
|
| 3175 |
|
|
@@ -3513,8 +3566,8 @@ def filter_meta():
|
|
| 3513 |
#: not carry it", not "this account does not have it" β which is exactly what `enriched_at` and
|
| 3514 |
#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a
|
| 3515 |
#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make.
|
| 3516 |
-
#:
|
| 3517 |
-
#:
|
| 3518 |
#:
|
| 3519 |
#: β WAVE 26 Β· R5 β THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is
|
| 3520 |
#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`).
|
|
@@ -4855,9 +4908,9 @@ def _candidate_row(row, stamp):
|
|
| 4855 |
}
|
| 4856 |
|
| 4857 |
|
| 4858 |
-
def pull_profile(url, max_posts=DEFAULT_POSTS_PER_PULL, log=print, tier="anonymous",
|
| 4859 |
-
fallback=True,
|
| 4860 |
-
post_metrics=False):
|
| 4861 |
"""Everything readable about one public profile, cheapest-honest rung last.
|
| 4862 |
|
| 4863 |
Returns `{state, profile, posts, via, note}` with state β ok | partial | blocked | error.
|
|
@@ -4877,17 +4930,18 @@ def pull_profile(url, max_posts=DEFAULT_POSTS_PER_PULL, log=print, tier="anonymo
|
|
| 4877 |
"""
|
| 4878 |
handle = ig_handle(url)
|
| 4879 |
if not handle:
|
| 4880 |
-
return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
|
| 4881 |
-
"profile": {}, "posts": [], "via": ""}
|
| 4882 |
attempts = []
|
| 4883 |
|
| 4884 |
if clean_tier(tier) == "brightdata":
|
| 4885 |
-
paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
|
|
|
|
| 4886 |
if paid["state"] in ("ok", "partial"):
|
| 4887 |
return paid
|
| 4888 |
attempts.append(f"brightdata:{_s(paid.get('note'), 90)}")
|
| 4889 |
if not fallback:
|
| 4890 |
-
return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
|
| 4891 |
"note": "the paid rung did not answer and the anonymous fallback is turned "
|
| 4892 |
f"off for this automation ({paid.get('note') or 'blocked'})"}
|
| 4893 |
time.sleep(PACE_SECONDS)
|
|
@@ -5147,8 +5201,8 @@ def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, row
|
|
| 5147 |
return (state, summary, counts, affected, steps)
|
| 5148 |
|
| 5149 |
|
| 5150 |
-
def capture_rows(res, pulled):
|
| 5151 |
-
"""ONE pull β `(snapshot_row, post_identity_rows, post_metric_rows)`.
|
| 5152 |
|
| 5153 |
β WAVE 25 (C4) β FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT
|
| 5154 |
RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path β
|
|
@@ -5185,18 +5239,21 @@ def capture_rows(res, pulled):
|
|
| 5185 |
# Profile history declares the same percentage dialect as the current Profile row.
|
| 5186 |
# Converting only the latest projection left the append series at 0-1 while its field
|
| 5187 |
# now said pct (0-100), so the two views of one measurement disagreed by 100x.
|
| 5188 |
-
|
| 5189 |
-
|
| 5190 |
-
|
|
|
|
|
|
|
| 5191 |
for p in (res or {}).get("posts") or []:
|
| 5192 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5193 |
"url": _s(p.get("url"), 300)}
|
| 5194 |
for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
|
| 5195 |
("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
|
| 5196 |
-
("alt_text", 400), ("tagged_location", 400)
|
| 5197 |
-
|
| 5198 |
-
|
| 5199 |
-
|
|
|
|
| 5200 |
metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
|
| 5201 |
# ββ 2026-08-07 β THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
|
| 5202 |
#
|
|
@@ -5222,14 +5279,28 @@ def capture_rows(res, pulled):
|
|
| 5222 |
# no stamp it means that AND "we never looked", indistinguishably.
|
| 5223 |
ident["measured_at"] = _day(pulled)
|
| 5224 |
idents.append(ident)
|
| 5225 |
-
if any(v is not None for v in metrics.values()):
|
| 5226 |
metrics_rows.append({
|
| 5227 |
"post_snapshot_key": f"{p['shortcode']}@{pulled}",
|
| 5228 |
"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5229 |
"pulled_at": pulled,
|
| 5230 |
"likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
|
| 5231 |
-
"views": _s(metrics["views"]), "plays": _s(metrics["plays"])
|
| 5232 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5233 |
|
| 5234 |
|
| 5235 |
#: C4 β how ONE pulled profile becomes the C1 preset CELLS on the record being enriched.
|
|
@@ -5291,7 +5362,7 @@ def preset_cells(res, pulled):
|
|
| 5291 |
# straight-through copy here would write the same wrong number the discovery path
|
| 5292 |
# used to write. Both writers convert; neither is the exception.
|
| 5293 |
cells[dest] = (_pct100(v) if dest == "avg_engagement" else
|
| 5294 |
-
|
| 5295 |
# β R5 β stamped by the writer, not carried from the pull: this function only ever sees an
|
| 5296 |
# Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
|
| 5297 |
# arrived with no platform at all β and a blank half of the dedup key is how one account
|
|
@@ -5312,9 +5383,10 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 5312 |
cfg = defn.get("config") or {}
|
| 5313 |
table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
|
| 5314 |
tier = clean_tier(cfg.get("tier")) or "anonymous"
|
| 5315 |
-
paid = tier == "brightdata"
|
| 5316 |
-
fallback = not cfg.get("noFallback")
|
| 5317 |
-
post_metrics = bool(cfg.get("postMetrics"))
|
|
|
|
| 5318 |
dry = bool(cfg.get("dryRun"))
|
| 5319 |
steps = {"trigger": "ok", "source": "idle", "capture": "idle",
|
| 5320 |
"capture_paid": "idle" if paid else "skipped",
|
|
@@ -5334,27 +5406,28 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 5334 |
# the relational tables the pull lands in (R7): every row timestamped for time-range filters
|
| 5335 |
if dry:
|
| 5336 |
# The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
|
| 5337 |
-
snap_key, post_key, ps_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE,
|
| 5338 |
-
|
| 5339 |
else:
|
| 5340 |
graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
|
| 5341 |
profile_table=table_key)
|
| 5342 |
-
snap_key, post_key, ps_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
|
| 5343 |
-
|
| 5344 |
-
missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key)
|
| 5345 |
snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
|
| 5346 |
-
posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
|
| 5347 |
-
psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {})
|
|
|
|
| 5348 |
|
| 5349 |
counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0,
|
| 5350 |
-
"posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0,
|
| 5351 |
"missing_tables": len(missing)}
|
| 5352 |
cells, affected, notes = {}, [], []
|
| 5353 |
# β ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows`
|
| 5354 |
# once per POST, which is O(existing) per call β survivable only while the cap was 5000 rows.
|
| 5355 |
# `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
|
| 5356 |
# that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
|
| 5357 |
-
in_snaps, in_posts, in_psnaps = [], [], []
|
| 5358 |
paid_tried = paid_answered = anon_tried = anon_answered = False
|
| 5359 |
targets = [(rid, str(r.get(url_field, "") or "").strip())
|
| 5360 |
for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
|
|
@@ -5366,8 +5439,9 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 5366 |
# between "working through them" and "stuck".
|
| 5367 |
step(f"Capturing profile {i + 1} of {len(targets)}")
|
| 5368 |
counts["profiles"] += 1
|
| 5369 |
-
res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
|
| 5370 |
-
tier=tier, fallback=fallback, post_metrics=post_metrics
|
|
|
|
| 5371 |
pulled = _iso()
|
| 5372 |
via = res.get("via") or ""
|
| 5373 |
read_ok = res["state"] in ("ok", "partial")
|
|
@@ -5386,11 +5460,13 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 5386 |
if read_ok:
|
| 5387 |
counts["ok" if res["state"] == "ok" else "partial"] += 1
|
| 5388 |
prof = res["profile"]
|
| 5389 |
-
snap_row, ident_rows, metric_rows = capture_rows(res, pulled)
|
| 5390 |
in_snaps.append(snap_row)
|
| 5391 |
-
in_posts.extend(ident_rows)
|
| 5392 |
-
in_psnaps.extend(metric_rows)
|
| 5393 |
-
|
|
|
|
|
|
|
| 5394 |
counts["posts"] += len(res.get("posts") or [])
|
| 5395 |
detail = f"{prof.get('followers') or '?'} followers"
|
| 5396 |
if prof.get("approx"):
|
|
@@ -5415,10 +5491,13 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 5415 |
posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at")
|
| 5416 |
posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
|
| 5417 |
c_post["duplicates"] += collapsed_posts
|
| 5418 |
-
psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
|
| 5419 |
-
|
| 5420 |
-
|
| 5421 |
-
|
|
|
|
|
|
|
|
|
|
| 5422 |
counts["capped"] = sum(n for _k, n in capped)
|
| 5423 |
|
| 5424 |
# R5 β route each processed record down its winning lane, judged over the row's FINAL
|
|
@@ -5448,7 +5527,8 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 5448 |
tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
|
| 5449 |
for rid, vals in stage_writes.items():
|
| 5450 |
tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals)
|
| 5451 |
-
for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)
|
|
|
|
| 5452 |
tgt = cur.get(k)
|
| 5453 |
if tgt is not None:
|
| 5454 |
tgt["rows"] = rws
|
|
@@ -5515,12 +5595,16 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
|
|
| 5515 |
summary += f", {counts['routed']}/{len(targets)} routed to lanes"
|
| 5516 |
if counts["paid"]:
|
| 5517 |
summary += f", {counts['paid']} with exact counts"
|
| 5518 |
-
if counts["metrics"]:
|
| 5519 |
-
summary += f", {counts['metrics']} post engagement snapshots"
|
| 5520 |
elif post_metrics and counts["posts"]:
|
| 5521 |
# β ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there
|
| 5522 |
# was no engagement", which is a claim about Instagram rather than about our run.
|
| 5523 |
-
summary += ", no post engagement was readable"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5524 |
if counts["partial"]:
|
| 5525 |
summary += f", {counts['partial']} profile-only (posts not readable)"
|
| 5526 |
if counts["blocked"]:
|
|
@@ -6834,9 +6918,12 @@ def _clean_action_config(kind, cfg, depth, seen, count):
|
|
| 6834 |
# not be the thing that starts spending money (the `clean_config` rule, verbatim).
|
| 6835 |
"tier": clean_tier(cfg.get("tier")) or "anonymous",
|
| 6836 |
"noFallback": bool(cfg.get("noFallback")),
|
| 6837 |
-
# β THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
|
| 6838 |
-
"postMetrics": bool(cfg.get("postMetrics")),
|
| 6839 |
-
|
|
|
|
|
|
|
|
|
|
| 6840 |
"maxPosts": max_posts}, None
|
| 6841 |
if kind == "find_records":
|
| 6842 |
table = _s(cfg.get("table"), 60).strip()
|
|
@@ -7478,7 +7565,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 7478 |
# and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
|
| 7479 |
# entry, so calling it per record is O(existing) per record β survivable at 5,000 rows and an
|
| 7480 |
# automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
|
| 7481 |
-
enrich = {"snaps": [], "posts": [], "psnaps": [], "profiles": 0, "ok": 0, "blocked": 0,
|
| 7482 |
"notes": [], "dry": False}
|
| 7483 |
# β 2026-08-07 β THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
|
| 7484 |
# `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
|
|
@@ -7644,16 +7731,18 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
|
|
| 7644 |
res = pull_profile(handle_raw,
|
| 7645 |
max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
|
| 7646 |
log=log,
|
| 7647 |
-
tier=clean_tier(cfg.get("tier")) or "anonymous",
|
| 7648 |
-
fallback=not cfg.get("noFallback"),
|
| 7649 |
-
post_metrics=bool(cfg.get("postMetrics"))
|
|
|
|
| 7650 |
pulled = _iso()
|
| 7651 |
if res["state"] in ("ok", "partial"):
|
| 7652 |
enrich["ok"] += 1
|
| 7653 |
-
snap_row, idents, metric_rows = capture_rows(res, pulled)
|
| 7654 |
enrich["snaps"].append(snap_row)
|
| 7655 |
-
enrich["posts"].extend(idents)
|
| 7656 |
-
enrich["psnaps"].extend(metric_rows)
|
|
|
|
| 7657 |
# R3: LATEST onto the record itself. `row.update` too, so a LATER action in
|
| 7658 |
# this same flow sees the enriched values β the rule `update_record` already
|
| 7659 |
# follows, and without it "enrich, then route by follower count" would judge
|
|
@@ -7783,10 +7872,11 @@ def migrate_field_instagram(rt, defn):
|
|
| 7783 |
max_posts = 24
|
| 7784 |
act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 7785 |
"config": {"profileField": bound,
|
| 7786 |
-
"tier": clean_tier(cfg.get("tier")) or "anonymous",
|
| 7787 |
-
"noFallback": bool(cfg.get("noFallback")),
|
| 7788 |
-
"postMetrics": bool(cfg.get("postMetrics")),
|
| 7789 |
-
"
|
|
|
|
| 7790 |
"maxPosts": max(1, min(max_posts, 200))}}
|
| 7791 |
flow = dict(defn.get("flow") or {})
|
| 7792 |
out = dict(defn)
|
|
@@ -7803,8 +7893,8 @@ def migrate_field_instagram(rt, defn):
|
|
| 7803 |
return out, True
|
| 7804 |
|
| 7805 |
|
| 7806 |
-
def _enrich_flush(rt, defn, username, acc, log):
|
| 7807 |
-
"""C4: the enrich action's
|
| 7808 |
|
| 7809 |
Returns the counts the run reports. Empty when no enrich action ran, so a flow without one
|
| 7810 |
pays nothing and its run history is unchanged (`run_now` merges only non-zero keys).
|
|
@@ -7817,7 +7907,7 @@ def _enrich_flush(rt, defn, username, acc, log):
|
|
| 7817 |
if not acc.get("profiles"):
|
| 7818 |
return {}
|
| 7819 |
out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
|
| 7820 |
-
if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"]):
|
| 7821 |
# A dry run resolves nothing and writes nothing β not even `ut_ensure`, which CREATES.
|
| 7822 |
if acc.get("dry"):
|
| 7823 |
out["enrichDryRun"] = acc["profiles"]
|
|
@@ -7825,21 +7915,27 @@ def _enrich_flush(rt, defn, username, acc, log):
|
|
| 7825 |
tag = str(defn.get("id") or "")
|
| 7826 |
profile_table = str(_flow_table(defn) or "")
|
| 7827 |
graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table)
|
| 7828 |
-
snap_key, post_key, ps_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
|
| 7829 |
-
|
| 7830 |
-
missing = ut_missing(rt, snap_key, post_key, ps_key)
|
| 7831 |
snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
|
| 7832 |
acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
|
| 7833 |
old_posts, collapsed_posts = dedupe_canonical_rows(
|
| 7834 |
dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
|
| 7835 |
posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key))
|
| 7836 |
c_post["duplicates"] += collapsed_posts
|
| 7837 |
-
psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
|
| 7838 |
-
acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7839 |
|
| 7840 |
def _up(cur):
|
| 7841 |
cur = cur if isinstance(cur, dict) else {}
|
| 7842 |
-
for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)
|
|
|
|
| 7843 |
if cur.get(k) is not None:
|
| 7844 |
cur[k]["rows"] = rws
|
| 7845 |
_refresh_relations_inplace(cur, log=log)
|
|
@@ -7848,15 +7944,16 @@ def _enrich_flush(rt, defn, username, acc, log):
|
|
| 7848 |
rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
|
| 7849 |
# LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
|
| 7850 |
# is the failure a chart cannot show you.
|
| 7851 |
-
capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"]
|
| 7852 |
if capped:
|
| 7853 |
out["enrichCapped"] = capped
|
| 7854 |
log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap")
|
| 7855 |
if missing:
|
| 7856 |
out["enrichMissingTables"] = len(missing)
|
| 7857 |
log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}")
|
| 7858 |
-
out["enrichPoints"] = len(acc["snaps"])
|
| 7859 |
-
out["enrichPosts"] = c_post["inserted"]
|
|
|
|
| 7860 |
# C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated β the same
|
| 7861 |
# contract `run_field_instagram` follows, because a pooled history with silent holes is worse
|
| 7862 |
# than none.
|
|
|
|
| 1038 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1039 |
"replies": "Reply count reported for the comment.",
|
| 1040 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
| 1041 |
+
"source_payload": "Full Bright Data source response for this record.",
|
| 1042 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1043 |
"source": "Method or provider used to read this profile.",
|
| 1044 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
|
|
|
| 1231 |
"noFallback": flag("noFallback"),
|
| 1232 |
# β THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
|
| 1233 |
# for: the Profiles row carries post IDENTITY for free but NO engagement
|
| 1234 |
+
# (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
|
| 1235 |
+
"postMetrics": flag("postMetrics"),
|
| 1236 |
+
# The full Comments dataset can bill many rows per post. It is an explicit,
|
| 1237 |
+
# independent opt-in and never follows the per-post switch automatically.
|
| 1238 |
+
"commentMetrics": flag("commentMetrics"),
|
| 1239 |
+
"dryRun": flag("dryRun"),
|
| 1240 |
"lanes": lanes,
|
| 1241 |
"maxPosts": max_posts}, None
|
| 1242 |
if kind == "discover_instagram":
|
|
|
|
| 1473 |
# with only step 1 lands this at the end, which IS step 2.
|
| 1474 |
seeded.insert(1, {
|
| 1475 |
"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 1476 |
+
"config": {"tier": "anonymous", "noFallback": False, "postMetrics": False,
|
| 1477 |
+
"commentMetrics": False,
|
| 1478 |
"dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
|
| 1479 |
"fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
|
| 1480 |
"limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
|
|
|
|
| 2085 |
#: has exactly the same cost as delaying capture β for that column β and is justified by exactly
|
| 2086 |
#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to
|
| 2087 |
#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices.
|
| 2088 |
+
#: Every returned provider field is retained in Source data. Promoted columns make commonly used
|
| 2089 |
+
#: values filterable; the raw source document prevents a new or uncommon field from being lost.
|
| 2090 |
#: β A blank in any of these means NOT READ β never "they have none". The anonymous rungs expose
|
| 2091 |
#: almost none of them, which is what the `source` column is for.
|
| 2092 |
SNAPSHOT_FIELDS = [
|
|
|
|
| 2100 |
# The link in bio β the paid rung's field, and commercially the most useful single string an
|
| 2101 |
# influencer row can carry (it is the shop/affiliate destination).
|
| 2102 |
field_def("external_url", "Link in bio", "url"),
|
| 2103 |
+
# --- the rest of the vendor's Profiles schema.
|
| 2104 |
field_def("ig_id", "Instagram id"),
|
| 2105 |
field_def("profile_url", "Profile", "url"),
|
| 2106 |
field_def("avg_engagement", "Avg engagement", "pct"),
|
|
|
|
| 2135 |
# presented as the creator's location: a creator can tag a holiday, venue or brand location.
|
| 2136 |
field_def("tagged_location", "Tagged location"),
|
| 2137 |
# The per-field map below gives operational fields first; this locked JSON document retains
|
| 2138 |
+
# every other value Bright Data supplied, so a vendor schema addition is preserved
|
| 2139 |
# instead of being silently discarded while the canonical field model catches up.
|
| 2140 |
field_def("source_payload", "Source data", "json"),
|
| 2141 |
# Sponsored-post detection β Β§2c called it one of the fields worth having that the anonymous
|
|
|
|
| 2207 |
#: β WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as
|
| 2208 |
#: `config.postMetrics` and the Bright Data money switch W26/R15 kept β the relation costs
|
| 2209 |
#: nothing; the spend and the third-party ingest are a click somebody makes knowingly.
|
| 2210 |
+
#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because
|
| 2211 |
+
#: they are part of a record already paid for. The separate full Comments dataset remains OFF
|
| 2212 |
+
#: unless `commentMetrics` is explicitly enabled on the enrichment action.
|
| 2213 |
COMMENT_FIELDS = [
|
| 2214 |
field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
|
| 2215 |
field_def("influencer_key", "Influencer"),
|
|
|
|
| 2221 |
description="Post record linked to this comment.",
|
| 2222 |
link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
|
| 2223 |
"single": True}),
|
| 2224 |
+
# Author/text and every provider-specific value stay intact in Source data. The promoted
|
| 2225 |
+
# columns intentionally keep the grid concise while Likes and Replies make comment engagement
|
| 2226 |
+
# directly filterable and rollup-ready.
|
| 2227 |
+
]
|
|
|
|
| 2228 |
POST_SNAPSHOT_FIELDS = [
|
| 2229 |
field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
|
| 2230 |
field_def("influencer_key", "Influencer"),
|
|
|
|
| 2234 |
# thing to derive. Paid rung only; blank means not read.
|
| 2235 |
field_def("views", "Views", "int"),
|
| 2236 |
field_def("plays", "Plays", "int"),
|
| 2237 |
+
field_def("source_payload", "Source data", "json"),
|
| 2238 |
field_def("post_link", "Post", "link",
|
| 2239 |
description="Post record linked to this measurement.",
|
| 2240 |
link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
|
|
|
|
| 2386 |
BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram β Profiles. 36 fields, 620M records
|
| 2387 |
BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram β Posts. 43 fields
|
| 2388 |
BD_DS_REELS = "gd_lyclm20il4r5helnj" # Instagram Reels: views and play counts
|
| 2389 |
+
BD_DS_COMMENTS = "gd_ltppn085pokosxh13" # Instagram Comments: opt-in full comment engagement
|
| 2390 |
BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
|
| 2391 |
BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_β¦"}
|
| 2392 |
BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
|
|
|
|
| 2562 |
return default
|
| 2563 |
|
| 2564 |
|
| 2565 |
+
#: Source-data retention is intentionally unfiltered: if Bright Data returned it in an already
|
| 2566 |
+
#: paid Profile, Post, Reel, or Comment response, it belongs in that record's source document.
|
| 2567 |
+
#: The one cost boundary is the *request*: the full Comments dataset is only called when an owner
|
| 2568 |
+
#: enables `commentMetrics`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2569 |
|
| 2570 |
+
#: `bd_call` refuses a partial response at this same transport ceiling. A single source document
|
| 2571 |
+
#: therefore fits a JSON cell; Source data is never shortened by `_s`, which would corrupt JSON
|
| 2572 |
+
#: and silently lose fields.
|
| 2573 |
+
BD_SOURCE_PAYLOAD_MAX = BD_MAX_KB * 1024
|
| 2574 |
|
| 2575 |
#: The vendor's content vocabulary β ours. MEASURED values: profiles' `posts[].content_type` is
|
| 2576 |
#: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`.
|
| 2577 |
+
_BD_TYPES = {"image": "image", "images": "image", "photo": "image", "photos": "image",
|
| 2578 |
+
"video": "video", "videos": "video", "reel": "video", "reels": "video",
|
| 2579 |
+
"clip": "video", "clips": "video", "igtv": "video", "carousel": "carousel",
|
| 2580 |
"carousel_container": "carousel", "sidecar": "carousel", "posts": "image"}
|
| 2581 |
|
| 2582 |
|
|
|
|
| 2682 |
return ", ".join(parts)[:400] or None
|
| 2683 |
|
| 2684 |
|
| 2685 |
+
def _bd_source_payload(row):
|
| 2686 |
+
"""One complete Bright Data response row as valid JSON, without field filtering.
|
| 2687 |
|
| 2688 |
+
Source data is the lossless provider record, not a second relationship cache. A Profile's
|
| 2689 |
+
embedded ``posts`` array stays here *and* each parseable post becomes a linked Post record;
|
| 2690 |
+
keeping both preserves data while Links remain the interactive relational view. `bd_call`
|
| 2691 |
+
rejects partial bodies at ``BD_SOURCE_PAYLOAD_MAX``, and the table-layer JSON ceiling matches
|
| 2692 |
+
it, so this function never truncates a paid source response.
|
| 2693 |
"""
|
| 2694 |
if not isinstance(row, dict):
|
| 2695 |
return None
|
|
|
|
| 2696 |
try:
|
| 2697 |
+
packed = json.dumps({str(k): v for k, v in row.items()}, ensure_ascii=False,
|
| 2698 |
+
sort_keys=True, separators=(",", ":"))
|
| 2699 |
except (TypeError, ValueError):
|
| 2700 |
return None
|
| 2701 |
+
# A source row is bounded by the complete raw HTTP response. Leave this defensive assertion
|
| 2702 |
+
# explicit: should the transport ceiling and JSON-cell ceiling ever diverge, fail visibly
|
| 2703 |
+
# rather than shortening a document and pretending every paid field was retained.
|
| 2704 |
+
if len(packed.encode("utf-8")) > BD_SOURCE_PAYLOAD_MAX:
|
| 2705 |
+
raise ValueError("a Bright Data source row exceeded the guarded response ceiling")
|
| 2706 |
+
return packed
|
| 2707 |
|
| 2708 |
|
| 2709 |
+
def _bd_comment(row, influencer_key="", shortcode="", fallback_key=""):
|
| 2710 |
+
"""One Bright Data Comment response -> one linked Comment record with engagement.
|
| 2711 |
|
| 2712 |
+
The dedicated Comments dataset is invoked only by the default-off `commentMetrics` switch.
|
| 2713 |
+
This mapper is also used for comment arrays embedded in an already-paid Post/Reel response,
|
| 2714 |
+
which adds no request. The full provider object is preserved in Source data; Likes and Replies
|
| 2715 |
+
are promoted for the grid and future rollups.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2716 |
"""
|
| 2717 |
if not isinstance(row, dict):
|
| 2718 |
return None
|
| 2719 |
url = str(_first(row, "url", "post_url", default="") or "")
|
| 2720 |
code = str(shortcode or "") or ig_shortcode(url)
|
| 2721 |
+
comment_id = str(_first(row, "comment_id", "id", "pk", default="") or fallback_key or "")
|
| 2722 |
if not code or not comment_id:
|
| 2723 |
return None
|
| 2724 |
+
posted = str(_first(row, "comment_date", "date_of_comment", "created_at", "created_time",
|
| 2725 |
+
default="") or "")
|
| 2726 |
return {
|
| 2727 |
"comment_key": comment_id,
|
| 2728 |
"shortcode": code,
|
| 2729 |
"influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
|
| 2730 |
"commented_at": posted.replace("T", " ")[:16],
|
| 2731 |
+
"likes": _ig_int(_first(row, "likes_number", "likes", "like_count", "likes_count")),
|
| 2732 |
+
"replies": _ig_int(_first(row, "replies_number", "num_replies", "replies",
|
| 2733 |
+
"reply_count", "replies_count")),
|
| 2734 |
+
"source_payload": _bd_source_payload(row),
|
| 2735 |
}
|
| 2736 |
+
|
| 2737 |
+
|
| 2738 |
+
def _bd_embedded_comments(post, influencer_key=""):
|
| 2739 |
+
"""Comment rows carried inside a paid Profile/Post/Reel record, with no extra request."""
|
| 2740 |
+
if not isinstance(post, dict):
|
| 2741 |
+
return []
|
| 2742 |
+
shortcode = str(post.get("shortcode") or "") or ig_shortcode(post.get("url"))
|
| 2743 |
+
if not shortcode:
|
| 2744 |
+
return []
|
| 2745 |
+
out, seen = [], set()
|
| 2746 |
+
# Known Bright Data arrays plus any future list-valued comment field. Source data still keeps
|
| 2747 |
+
# unknown shapes even if it cannot be normalised into a Comment row.
|
| 2748 |
+
arrays = [(str(k), v) for k, v in post.items()
|
| 2749 |
+
if isinstance(v, list) and (str(k) in ("latest_comments", "top_comments")
|
| 2750 |
+
or "comment" in str(k).lower())]
|
| 2751 |
+
for source_key, values in arrays:
|
| 2752 |
+
for index, value in enumerate(values):
|
| 2753 |
+
if not isinstance(value, dict):
|
| 2754 |
+
continue
|
| 2755 |
+
try:
|
| 2756 |
+
raw = json.dumps(value, ensure_ascii=False, sort_keys=True,
|
| 2757 |
+
separators=(",", ":"))
|
| 2758 |
+
except (TypeError, ValueError):
|
| 2759 |
+
continue
|
| 2760 |
+
fallback = "embedded:" + hashlib.sha256(
|
| 2761 |
+
f"{shortcode}:{source_key}:{index}:{raw}".encode("utf-8")).hexdigest()[:24]
|
| 2762 |
+
got = _bd_comment(value, influencer_key=influencer_key, shortcode=shortcode,
|
| 2763 |
+
fallback_key=fallback)
|
| 2764 |
+
if got and got["comment_key"] not in seen:
|
| 2765 |
+
seen.add(got["comment_key"])
|
| 2766 |
+
out.append(got)
|
| 2767 |
+
return out
|
| 2768 |
|
| 2769 |
|
| 2770 |
def _bd_profile(node, handle):
|
|
|
|
| 2810 |
# `wave20-split.md`), and the first four were explicitly logged as *"populated on scrape
|
| 2811 |
# and NOT in any previous list"* β so these are not speculative names, they are fields the
|
| 2812 |
# vendor was already sending and this map was already throwing away.
|
| 2813 |
+
# Every unpromoted provider key β including contact and image fields β stays complete in
|
| 2814 |
+
# Source data. This keeps the locked grid readable without throwing away paid data.
|
|
|
|
|
|
|
|
|
|
| 2815 |
"profile_name": str(_first(node, "profile_name", default="") or ""),
|
| 2816 |
"is_joined_recently": _bd_flag(node, "is_joined_recently"),
|
| 2817 |
"has_channel": _bd_flag(node, "has_channel"),
|
|
|
|
| 2831 |
# was `None` on all five corpus rows. It is a row field regardless, and reading it costs
|
| 2832 |
# nothing; a blank here means not read, like every other blank in this map.
|
| 2833 |
"country_code": str(_first(node, "country_code", default="") or ""),
|
| 2834 |
+
"source_payload": _bd_source_payload(node),
|
| 2835 |
+
}
|
| 2836 |
+
return out
|
|
|
|
|
|
|
|
|
|
| 2837 |
|
| 2838 |
|
| 2839 |
def _bd_post_identity(p):
|
|
|
|
| 2862 |
caption = _first(p, "caption", "description", default="")
|
| 2863 |
if isinstance(caption, dict):
|
| 2864 |
caption = _first(caption, "text", default="")
|
| 2865 |
+
if caption:
|
| 2866 |
+
out["caption"] = str(caption)
|
| 2867 |
+
out["source_payload"] = _bd_source_payload(p)
|
| 2868 |
+
embedded = _bd_embedded_comments(p)
|
| 2869 |
+
if embedded:
|
| 2870 |
+
out["embedded_comments"] = embedded
|
| 2871 |
+
return out
|
| 2872 |
|
| 2873 |
|
| 2874 |
def _bd_post_metrics(row):
|
|
|
|
| 2883 |
is declared `text`, the others `number`). A candidate list may span types only when the
|
| 2884 |
MEANING is identical β never the way `like_and_view_counts_disabled`, a boolean, would have
|
| 2885 |
turned "likes are hidden" into "1 like".
|
| 2886 |
+
Embedded comment arrays are retained in Source data and converted to linked Comment rows
|
| 2887 |
+
without triggering the separate full Comments-dataset request.
|
| 2888 |
"""
|
| 2889 |
if not isinstance(row, dict):
|
| 2890 |
return None
|
|
|
|
| 2916 |
"hashtags": _bd_list(row, "hashtags"),
|
| 2917 |
"alt_text": str(_first(row, "alt_text", default="") or ""),
|
| 2918 |
"tagged_location": _bd_tagged_location(row),
|
| 2919 |
+
"source_payload": _bd_source_payload(row),
|
| 2920 |
+
}
|
| 2921 |
+
embedded = _bd_embedded_comments(row, influencer_key=out["influencer_key"])
|
| 2922 |
+
if embedded:
|
| 2923 |
+
out["embedded_comments"] = embedded
|
| 2924 |
+
return out
|
| 2925 |
|
| 2926 |
|
| 2927 |
def _bd_rows(payload):
|
|
|
|
| 3074 |
f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building β it is not lost)")
|
| 3075 |
|
| 3076 |
|
| 3077 |
+
def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False,
|
| 3078 |
+
comment_metrics=False, log=print):
|
| 3079 |
"""The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides.
|
| 3080 |
|
| 3081 |
Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner
|
| 3082 |
treats every rung identically. `blocked` here means the VENDOR rung did not answer β the
|
| 3083 |
caller decides whether to drop to the anonymous ladder.
|
| 3084 |
|
| 3085 |
+
`post_metrics=True` adds the paid Post/Reel calls. `comment_metrics=True` independently adds
|
| 3086 |
+
the separate paid Comments-dataset call. Both default off; the latter is never inferred from
|
| 3087 |
+
an enabled post-metrics switch. Embedded comment previews returned by either paid record are
|
| 3088 |
+
retained without that extra Comments request.
|
| 3089 |
"""
|
| 3090 |
handle = ig_handle(url)
|
| 3091 |
if not handle:
|
| 3092 |
+
return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
|
| 3093 |
+
"profile": {}, "posts": [], "comments": [], "via": ""}
|
| 3094 |
rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"])
|
| 3095 |
node = rows[0] if rows else {}
|
| 3096 |
profile = _bd_profile(node, handle) if node else {}
|
|
|
|
| 3125 |
f"the vendor's pre-collected profile corpus β the values are its last "
|
| 3126 |
f"capture, not a read taken just now")
|
| 3127 |
else:
|
| 3128 |
+
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 3129 |
"note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"}
|
| 3130 |
if profile.get("followers") is None and profile.get("following") is None:
|
| 3131 |
# It answered 200 with something we could not read. Say THAT β not "0 followers".
|
| 3132 |
# β `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and
|
| 3133 |
# is discarded, so a row where it is the only "readable" field is a row we cannot read.
|
| 3134 |
+
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 3135 |
"note": "the profile source answered, but no follower/following counts were readable in "
|
| 3136 |
"it (the field names may have moved β see instagram-capture.md)"}
|
| 3137 |
posts = []
|
|
|
|
| 3148 |
# without duplicating any record, so enabling post metrics still buys one row per post.
|
| 3149 |
ordinary = [p for p in posts if p.get("type") != "video"]
|
| 3150 |
videos = [p for p in posts if p.get("type") == "video"]
|
| 3151 |
+
rows2, metric_notes, ordinary_rows = [], [], []
|
| 3152 |
if ordinary:
|
| 3153 |
got_rows, got_note = bd_scrape(BD_DS_POSTS, [p["url"] for p in ordinary])
|
| 3154 |
rows2.extend(got_rows)
|
| 3155 |
+
ordinary_rows.extend(got_rows)
|
| 3156 |
if got_note:
|
| 3157 |
metric_notes.append(f"posts: {got_note}")
|
| 3158 |
if videos:
|
|
|
|
| 3160 |
rows2.extend(got_rows)
|
| 3161 |
if got_note:
|
| 3162 |
metric_notes.append(f"reels: {got_note}")
|
| 3163 |
+
# Bright Data's Profiles feed has used more than one media spelling. If one of those
|
| 3164 |
+
# unrecognised labels sent a Reel to Posts, the Posts response itself identifies the
|
| 3165 |
+
# mistake (`content_type: Reel`) but cannot supply Views. Correct it immediately with a
|
| 3166 |
+
# Reels read, then let the richer response win below. This costs a replacement record only
|
| 3167 |
+
# for a proven misroute; correctly classified images/carousels are never re-read.
|
| 3168 |
+
repair_reels = []
|
| 3169 |
+
for row in ordinary_rows:
|
| 3170 |
+
mapped = _bd_post_metrics(row)
|
| 3171 |
+
if mapped and mapped.get("type") == "video":
|
| 3172 |
+
repair_reels.append(f"https://www.instagram.com/reel/{mapped['shortcode']}/")
|
| 3173 |
+
if repair_reels:
|
| 3174 |
+
got_rows, got_note = bd_scrape(BD_DS_REELS, list(dict.fromkeys(repair_reels)))
|
| 3175 |
+
rows2.extend(got_rows)
|
| 3176 |
+
if got_note:
|
| 3177 |
+
metric_notes.append(f"reel recovery: {got_note}")
|
| 3178 |
if metric_notes and not rows2:
|
| 3179 |
# The engagement half refusing does NOT lose the identity half β the profile and the
|
| 3180 |
# post rows still land, and the run says which part was not readable.
|
|
|
|
| 3195 |
note = "post metrics were requested but the Posts dataset returned no rows"
|
| 3196 |
elif metric_notes:
|
| 3197 |
note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
|
| 3198 |
+
|
| 3199 |
+
comments = []
|
| 3200 |
+
if comment_metrics and posts:
|
| 3201 |
+
# This is the cost boundary: do not make the full comment request unless the stored
|
| 3202 |
+
# enrichment action explicitly opted in. A batched call still bills provider records, but
|
| 3203 |
+
# it prevents unnecessary HTTP fan-out and preserves every returned field per comment.
|
| 3204 |
+
time.sleep(min(PACE_SECONDS, 1.0))
|
| 3205 |
+
comment_source, comment_note = bd_scrape(BD_DS_COMMENTS, [p["url"] for p in posts])
|
| 3206 |
+
for row in comment_source:
|
| 3207 |
+
got = _bd_comment(row, influencer_key=profile.get("username") or handle)
|
| 3208 |
+
if got:
|
| 3209 |
+
comments.append(got)
|
| 3210 |
+
if comment_note:
|
| 3211 |
+
note = "; ".join(x for x in (note, f"comment engagement unavailable: {comment_note}") if x)
|
| 3212 |
+
elif not comments:
|
| 3213 |
+
note = "; ".join(x for x in (note,
|
| 3214 |
+
"comment engagement was requested but the Comments dataset returned no rows") if x)
|
| 3215 |
|
| 3216 |
# IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften
|
| 3217 |
# because we are paying: a run that wrote a follower count and no posts must not paint green
|
|
|
|
| 3220 |
# snapshot row's `source` column, so a stored measurement can always say whether it was read
|
| 3221 |
# live or taken from the vendor's pre-collected corpus. Two routes writing one indisting-
|
| 3222 |
# uishable `source` would make the series unauditable exactly where it matters most.
|
| 3223 |
+
return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts,
|
| 3224 |
+
"comments": comments,
|
| 3225 |
+
"via": via,
|
| 3226 |
"note": note or ("" if posts else "profile read; no posts were returned")}
|
| 3227 |
|
| 3228 |
|
|
|
|
| 3566 |
#: not carry it", not "this account does not have it" β which is exactly what `enriched_at` and
|
| 3567 |
#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a
|
| 3568 |
#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make.
|
| 3569 |
+
#: Source data retains every field the provider returns. The only separate decision is whether to
|
| 3570 |
+
#: request the paid full Comments dataset (`commentMetrics`, default OFF).
|
| 3571 |
#:
|
| 3572 |
#: β WAVE 26 Β· R5 β THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is
|
| 3573 |
#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`).
|
|
|
|
| 4908 |
}
|
| 4909 |
|
| 4910 |
|
| 4911 |
+
def pull_profile(url, max_posts=DEFAULT_POSTS_PER_PULL, log=print, tier="anonymous",
|
| 4912 |
+
fallback=True,
|
| 4913 |
+
post_metrics=False, comment_metrics=False):
|
| 4914 |
"""Everything readable about one public profile, cheapest-honest rung last.
|
| 4915 |
|
| 4916 |
Returns `{state, profile, posts, via, note}` with state β ok | partial | blocked | error.
|
|
|
|
| 4930 |
"""
|
| 4931 |
handle = ig_handle(url)
|
| 4932 |
if not handle:
|
| 4933 |
+
return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
|
| 4934 |
+
"profile": {}, "posts": [], "comments": [], "via": ""}
|
| 4935 |
attempts = []
|
| 4936 |
|
| 4937 |
if clean_tier(tier) == "brightdata":
|
| 4938 |
+
paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
|
| 4939 |
+
comment_metrics=comment_metrics, log=log)
|
| 4940 |
if paid["state"] in ("ok", "partial"):
|
| 4941 |
return paid
|
| 4942 |
attempts.append(f"brightdata:{_s(paid.get('note'), 90)}")
|
| 4943 |
if not fallback:
|
| 4944 |
+
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 4945 |
"note": "the paid rung did not answer and the anonymous fallback is turned "
|
| 4946 |
f"off for this automation ({paid.get('note') or 'blocked'})"}
|
| 4947 |
time.sleep(PACE_SECONDS)
|
|
|
|
| 5201 |
return (state, summary, counts, affected, steps)
|
| 5202 |
|
| 5203 |
|
| 5204 |
+
def capture_rows(res, pulled):
|
| 5205 |
+
"""ONE pull β `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`.
|
| 5206 |
|
| 5207 |
β WAVE 25 (C4) β FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT
|
| 5208 |
RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path β
|
|
|
|
| 5239 |
# Profile history declares the same percentage dialect as the current Profile row.
|
| 5240 |
# Converting only the latest projection left the append series at 0-1 while its field
|
| 5241 |
# now said pct (0-100), so the two views of one measurement disagreed by 100x.
|
| 5242 |
+
if k == "source_payload":
|
| 5243 |
+
snap_row[k] = str(value or "")
|
| 5244 |
+
else:
|
| 5245 |
+
snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400)
|
| 5246 |
+
idents, metrics_rows, comment_rows = [], [], []
|
| 5247 |
for p in (res or {}).get("posts") or []:
|
| 5248 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5249 |
"url": _s(p.get("url"), 300)}
|
| 5250 |
for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
|
| 5251 |
("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
|
| 5252 |
+
("alt_text", 400), ("tagged_location", 400)):
|
| 5253 |
+
if p.get(k):
|
| 5254 |
+
ident[k] = _s(p.get(k), n)
|
| 5255 |
+
if p.get("source_payload"):
|
| 5256 |
+
ident["source_payload"] = str(p["source_payload"])
|
| 5257 |
metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
|
| 5258 |
# ββ 2026-08-07 β THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
|
| 5259 |
#
|
|
|
|
| 5279 |
# no stamp it means that AND "we never looked", indistinguishably.
|
| 5280 |
ident["measured_at"] = _day(pulled)
|
| 5281 |
idents.append(ident)
|
| 5282 |
+
if any(v is not None for v in metrics.values()):
|
| 5283 |
metrics_rows.append({
|
| 5284 |
"post_snapshot_key": f"{p['shortcode']}@{pulled}",
|
| 5285 |
"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5286 |
"pulled_at": pulled,
|
| 5287 |
"likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
|
| 5288 |
+
"views": _s(metrics["views"]), "plays": _s(metrics["plays"]),
|
| 5289 |
+
"source_payload": str(p.get("source_payload") or "")})
|
| 5290 |
+
for comment in p.get("embedded_comments") or []:
|
| 5291 |
+
# The embedded shape often names only the commenter. Its parent Post is the
|
| 5292 |
+
# authoritative owner, so bind every preview to this profile and post explicitly.
|
| 5293 |
+
comment_rows.append({**comment, "shortcode": p["shortcode"],
|
| 5294 |
+
"influencer_key": prof.get("username") or ""})
|
| 5295 |
+
comment_rows.extend((res or {}).get("comments") or [])
|
| 5296 |
+
# The same comment can be returned as both `latest_comments` and `top_comments`, or by the
|
| 5297 |
+
# optional full endpoint after an embedded preview. The canonical key is the one row truth.
|
| 5298 |
+
unique_comments = {}
|
| 5299 |
+
for row in comment_rows:
|
| 5300 |
+
key = str((row or {}).get("comment_key") or "")
|
| 5301 |
+
if key:
|
| 5302 |
+
unique_comments[key] = {**(unique_comments.get(key) or {}), **row}
|
| 5303 |
+
return snap_row, idents, metrics_rows, list(unique_comments.values())
|
| 5304 |
|
| 5305 |
|
| 5306 |
#: C4 β how ONE pulled profile becomes the C1 preset CELLS on the record being enriched.
|
|
|
|
| 5362 |
# straight-through copy here would write the same wrong number the discovery path
|
| 5363 |
# used to write. Both writers convert; neither is the exception.
|
| 5364 |
cells[dest] = (_pct100(v) if dest == "avg_engagement" else
|
| 5365 |
+
(str(v) if dest == "source_payload" else _s(v, 500)))
|
| 5366 |
# β R5 β stamped by the writer, not carried from the pull: this function only ever sees an
|
| 5367 |
# Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
|
| 5368 |
# arrived with no platform at all β and a blank half of the dedup key is how one account
|
|
|
|
| 5383 |
cfg = defn.get("config") or {}
|
| 5384 |
table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
|
| 5385 |
tier = clean_tier(cfg.get("tier")) or "anonymous"
|
| 5386 |
+
paid = tier == "brightdata"
|
| 5387 |
+
fallback = not cfg.get("noFallback")
|
| 5388 |
+
post_metrics = bool(cfg.get("postMetrics"))
|
| 5389 |
+
comment_metrics = bool(cfg.get("commentMetrics"))
|
| 5390 |
dry = bool(cfg.get("dryRun"))
|
| 5391 |
steps = {"trigger": "ok", "source": "idle", "capture": "idle",
|
| 5392 |
"capture_paid": "idle" if paid else "skipped",
|
|
|
|
| 5406 |
# the relational tables the pull lands in (R7): every row timestamped for time-range filters
|
| 5407 |
if dry:
|
| 5408 |
# The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
|
| 5409 |
+
snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE,
|
| 5410 |
+
IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE)
|
| 5411 |
else:
|
| 5412 |
graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
|
| 5413 |
profile_table=table_key)
|
| 5414 |
+
snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
|
| 5415 |
+
graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
|
| 5416 |
+
missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key)
|
| 5417 |
snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
|
| 5418 |
+
posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
|
| 5419 |
+
psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {})
|
| 5420 |
+
comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {})
|
| 5421 |
|
| 5422 |
counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0,
|
| 5423 |
+
"posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0,
|
| 5424 |
"missing_tables": len(missing)}
|
| 5425 |
cells, affected, notes = {}, [], []
|
| 5426 |
# β ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows`
|
| 5427 |
# once per POST, which is O(existing) per call β survivable only while the cap was 5000 rows.
|
| 5428 |
# `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
|
| 5429 |
# that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
|
| 5430 |
+
in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
|
| 5431 |
paid_tried = paid_answered = anon_tried = anon_answered = False
|
| 5432 |
targets = [(rid, str(r.get(url_field, "") or "").strip())
|
| 5433 |
for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
|
|
|
|
| 5439 |
# between "working through them" and "stuck".
|
| 5440 |
step(f"Capturing profile {i + 1} of {len(targets)}")
|
| 5441 |
counts["profiles"] += 1
|
| 5442 |
+
res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
|
| 5443 |
+
tier=tier, fallback=fallback, post_metrics=post_metrics,
|
| 5444 |
+
comment_metrics=comment_metrics)
|
| 5445 |
pulled = _iso()
|
| 5446 |
via = res.get("via") or ""
|
| 5447 |
read_ok = res["state"] in ("ok", "partial")
|
|
|
|
| 5460 |
if read_ok:
|
| 5461 |
counts["ok" if res["state"] == "ok" else "partial"] += 1
|
| 5462 |
prof = res["profile"]
|
| 5463 |
+
snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled)
|
| 5464 |
in_snaps.append(snap_row)
|
| 5465 |
+
in_posts.extend(ident_rows)
|
| 5466 |
+
in_psnaps.extend(metric_rows)
|
| 5467 |
+
in_comments.extend(captured_comments)
|
| 5468 |
+
counts["metrics"] += len(metric_rows)
|
| 5469 |
+
counts["comment_rows"] += len(captured_comments)
|
| 5470 |
counts["posts"] += len(res.get("posts") or [])
|
| 5471 |
detail = f"{prof.get('followers') or '?'} followers"
|
| 5472 |
if prof.get("approx"):
|
|
|
|
| 5491 |
posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at")
|
| 5492 |
posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
|
| 5493 |
c_post["duplicates"] += collapsed_posts
|
| 5494 |
+
psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
|
| 5495 |
+
comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key")
|
| 5496 |
+
comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key))
|
| 5497 |
+
c_comments["duplicates"] += collapsed_comments
|
| 5498 |
+
counts["new_posts"] = c_post["inserted"]
|
| 5499 |
+
capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]),
|
| 5500 |
+
(ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])]
|
| 5501 |
counts["capped"] = sum(n for _k, n in capped)
|
| 5502 |
|
| 5503 |
# R5 β route each processed record down its winning lane, judged over the row's FINAL
|
|
|
|
| 5527 |
tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
|
| 5528 |
for rid, vals in stage_writes.items():
|
| 5529 |
tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals)
|
| 5530 |
+
for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
|
| 5531 |
+
(comment_key, comments)):
|
| 5532 |
tgt = cur.get(k)
|
| 5533 |
if tgt is not None:
|
| 5534 |
tgt["rows"] = rws
|
|
|
|
| 5595 |
summary += f", {counts['routed']}/{len(targets)} routed to lanes"
|
| 5596 |
if counts["paid"]:
|
| 5597 |
summary += f", {counts['paid']} with exact counts"
|
| 5598 |
+
if counts["metrics"]:
|
| 5599 |
+
summary += f", {counts['metrics']} post engagement snapshots"
|
| 5600 |
elif post_metrics and counts["posts"]:
|
| 5601 |
# β ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there
|
| 5602 |
# was no engagement", which is a claim about Instagram rather than about our run.
|
| 5603 |
+
summary += ", no post engagement was readable"
|
| 5604 |
+
if counts["comment_rows"]:
|
| 5605 |
+
summary += f", {counts['comment_rows']} comment rows captured"
|
| 5606 |
+
elif comment_metrics and counts["posts"]:
|
| 5607 |
+
summary += ", no comment engagement was readable"
|
| 5608 |
if counts["partial"]:
|
| 5609 |
summary += f", {counts['partial']} profile-only (posts not readable)"
|
| 5610 |
if counts["blocked"]:
|
|
|
|
| 6918 |
# not be the thing that starts spending money (the `clean_config` rule, verbatim).
|
| 6919 |
"tier": clean_tier(cfg.get("tier")) or "anonymous",
|
| 6920 |
"noFallback": bool(cfg.get("noFallback")),
|
| 6921 |
+
# β THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
|
| 6922 |
+
"postMetrics": bool(cfg.get("postMetrics")),
|
| 6923 |
+
# The separate Comments dataset is more granular than a post read and stays off
|
| 6924 |
+
# until the user intentionally enables it.
|
| 6925 |
+
"commentMetrics": bool(cfg.get("commentMetrics")),
|
| 6926 |
+
"dryRun": bool(cfg.get("dryRun")),
|
| 6927 |
"maxPosts": max_posts}, None
|
| 6928 |
if kind == "find_records":
|
| 6929 |
table = _s(cfg.get("table"), 60).strip()
|
|
|
|
| 7565 |
# and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
|
| 7566 |
# entry, so calling it per record is O(existing) per record β survivable at 5,000 rows and an
|
| 7567 |
# automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
|
| 7568 |
+
enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "profiles": 0, "ok": 0, "blocked": 0,
|
| 7569 |
"notes": [], "dry": False}
|
| 7570 |
# β 2026-08-07 β THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
|
| 7571 |
# `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
|
|
|
|
| 7731 |
res = pull_profile(handle_raw,
|
| 7732 |
max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
|
| 7733 |
log=log,
|
| 7734 |
+
tier=clean_tier(cfg.get("tier")) or "anonymous",
|
| 7735 |
+
fallback=not cfg.get("noFallback"),
|
| 7736 |
+
post_metrics=bool(cfg.get("postMetrics")),
|
| 7737 |
+
comment_metrics=bool(cfg.get("commentMetrics")))
|
| 7738 |
pulled = _iso()
|
| 7739 |
if res["state"] in ("ok", "partial"):
|
| 7740 |
enrich["ok"] += 1
|
| 7741 |
+
snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
|
| 7742 |
enrich["snaps"].append(snap_row)
|
| 7743 |
+
enrich["posts"].extend(idents)
|
| 7744 |
+
enrich["psnaps"].extend(metric_rows)
|
| 7745 |
+
enrich["comments"].extend(comment_rows)
|
| 7746 |
# R3: LATEST onto the record itself. `row.update` too, so a LATER action in
|
| 7747 |
# this same flow sees the enriched values β the rule `update_record` already
|
| 7748 |
# follows, and without it "enrich, then route by follower count" would judge
|
|
|
|
| 7872 |
max_posts = 24
|
| 7873 |
act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
|
| 7874 |
"config": {"profileField": bound,
|
| 7875 |
+
"tier": clean_tier(cfg.get("tier")) or "anonymous",
|
| 7876 |
+
"noFallback": bool(cfg.get("noFallback")),
|
| 7877 |
+
"postMetrics": bool(cfg.get("postMetrics")),
|
| 7878 |
+
"commentMetrics": bool(cfg.get("commentMetrics")),
|
| 7879 |
+
"dryRun": bool(cfg.get("dryRun")),
|
| 7880 |
"maxPosts": max(1, min(max_posts, 200))}}
|
| 7881 |
flow = dict(defn.get("flow") or {})
|
| 7882 |
out = dict(defn)
|
|
|
|
| 7893 |
return out, True
|
| 7894 |
|
| 7895 |
|
| 7896 |
+
def _enrich_flush(rt, defn, username, acc, log):
|
| 7897 |
+
"""C4: the enrich action's canonical IG tables, written ONCE for the whole run.
|
| 7898 |
|
| 7899 |
Returns the counts the run reports. Empty when no enrich action ran, so a flow without one
|
| 7900 |
pays nothing and its run history is unchanged (`run_now` merges only non-zero keys).
|
|
|
|
| 7907 |
if not acc.get("profiles"):
|
| 7908 |
return {}
|
| 7909 |
out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
|
| 7910 |
+
if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]):
|
| 7911 |
# A dry run resolves nothing and writes nothing β not even `ut_ensure`, which CREATES.
|
| 7912 |
if acc.get("dry"):
|
| 7913 |
out["enrichDryRun"] = acc["profiles"]
|
|
|
|
| 7915 |
tag = str(defn.get("id") or "")
|
| 7916 |
profile_table = str(_flow_table(defn) or "")
|
| 7917 |
graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table)
|
| 7918 |
+
snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
|
| 7919 |
+
graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
|
| 7920 |
+
missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key)
|
| 7921 |
snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
|
| 7922 |
acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
|
| 7923 |
old_posts, collapsed_posts = dedupe_canonical_rows(
|
| 7924 |
dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
|
| 7925 |
posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key))
|
| 7926 |
c_post["duplicates"] += collapsed_posts
|
| 7927 |
+
psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
|
| 7928 |
+
acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
|
| 7929 |
+
old_comments, collapsed_comments = dedupe_canonical_rows(
|
| 7930 |
+
dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key")
|
| 7931 |
+
comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key",
|
| 7932 |
+
cap=row_cap(comment_key))
|
| 7933 |
+
c_comments["duplicates"] += collapsed_comments
|
| 7934 |
|
| 7935 |
def _up(cur):
|
| 7936 |
cur = cur if isinstance(cur, dict) else {}
|
| 7937 |
+
for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
|
| 7938 |
+
(comment_key, comments)):
|
| 7939 |
if cur.get(k) is not None:
|
| 7940 |
cur[k]["rows"] = rws
|
| 7941 |
_refresh_relations_inplace(cur, log=log)
|
|
|
|
| 7944 |
rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
|
| 7945 |
# LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
|
| 7946 |
# is the failure a chart cannot show you.
|
| 7947 |
+
capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"]
|
| 7948 |
if capped:
|
| 7949 |
out["enrichCapped"] = capped
|
| 7950 |
log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap")
|
| 7951 |
if missing:
|
| 7952 |
out["enrichMissingTables"] = len(missing)
|
| 7953 |
log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}")
|
| 7954 |
+
out["enrichPoints"] = len(acc["snaps"])
|
| 7955 |
+
out["enrichPosts"] = c_post["inserted"]
|
| 7956 |
+
out["enrichComments"] = c_comments["inserted"]
|
| 7957 |
# C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated β the same
|
| 7958 |
# contract `run_field_instagram` follows, because a pooled history with silent holes is worse
|
| 7959 |
# than none.
|
platform/core/user_tables.py
CHANGED
|
@@ -82,10 +82,11 @@ def records_mutable(table_key, st=None):
|
|
| 82 |
UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date',
|
| 83 |
'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json',
|
| 84 |
'link', 'rollup'}
|
| 85 |
-
#:
|
| 86 |
-
#:
|
| 87 |
-
#:
|
| 88 |
-
|
|
|
|
| 89 |
|
| 90 |
# ---------------------------------------------------------------------------------------------
|
| 91 |
# THE PROFILE FLAG (wave 25, contract C3 / owner ruling R7)
|
|
|
|
| 82 |
UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date',
|
| 83 |
'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json',
|
| 84 |
'link', 'rollup'}
|
| 85 |
+
#: A cell is still a scalar on the wire (`Row` values are strings/numbers) β what this bounds is
|
| 86 |
+
#: the DOCUMENT inside it. Source data is one already-paid provider response; keep it whole so a
|
| 87 |
+
#: provider field is never silently discarded merely because the response is rich. 32 MiB matches
|
| 88 |
+
#: the automation transport's guarded maximum response size.
|
| 89 |
+
MAX_JSON_CELL = 32 * 1024 * 1024
|
| 90 |
|
| 91 |
# ---------------------------------------------------------------------------------------------
|
| 92 |
# THE PROFILE FLAG (wave 25, contract C3 / owner ruling R7)
|
web/src/automation/AutomationBuilder.tsx
CHANGED
|
@@ -575,7 +575,8 @@ export default function AutomationBuilder({
|
|
| 575 |
*/
|
| 576 |
if (kind === "enrich_instagram")
|
| 577 |
return {
|
| 578 |
-
tier: "anonymous", noFallback: false, postMetrics: false,
|
|
|
|
| 579 |
fromView: "", sortField: "first_found", sortDir: "desc",
|
| 580 |
// β `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
|
| 581 |
// control must not have two defaults depending on whether the step was added by hand or
|
|
@@ -2487,6 +2488,21 @@ function ActionProps({
|
|
| 2487 |
Billed per post, not per profile β a separate scrape for each post kept above.
|
| 2488 |
</p>
|
| 2489 |
) : null}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2490 |
<label className="auto-check">
|
| 2491 |
<input
|
| 2492 |
type="checkbox"
|
|
|
|
| 575 |
*/
|
| 576 |
if (kind === "enrich_instagram")
|
| 577 |
return {
|
| 578 |
+
tier: "anonymous", noFallback: false, postMetrics: false, commentMetrics: false,
|
| 579 |
+
dryRun: false, maxPosts: 10,
|
| 580 |
fromView: "", sortField: "first_found", sortDir: "desc",
|
| 581 |
// β `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
|
| 582 |
// control must not have two defaults depending on whether the step was added by hand or
|
|
|
|
| 2488 |
Billed per post, not per profile β a separate scrape for each post kept above.
|
| 2489 |
</p>
|
| 2490 |
) : null}
|
| 2491 |
+
<label className="auto-check">
|
| 2492 |
+
<input
|
| 2493 |
+
type="checkbox"
|
| 2494 |
+
checked={!!(cfg as { commentMetrics?: boolean }).commentMetrics}
|
| 2495 |
+
disabled={busy}
|
| 2496 |
+
onChange={(e) => setCfg({ commentMetrics: e.target.checked })}
|
| 2497 |
+
/>
|
| 2498 |
+
Also capture per-comment engagement
|
| 2499 |
+
</label>
|
| 2500 |
+
{(cfg as { commentMetrics?: boolean }).commentMetrics ? (
|
| 2501 |
+
<p className="auto-hint">
|
| 2502 |
+
Billed by the separate Comments dataset. This is off by default; embedded comments
|
| 2503 |
+
already returned with a paid post are retained without this additional request.
|
| 2504 |
+
</p>
|
| 2505 |
+
) : null}
|
| 2506 |
<label className="auto-check">
|
| 2507 |
<input
|
| 2508 |
type="checkbox"
|