fsanyoto commited on
Commit
5d4a142
Β·
verified Β·
1 Parent(s): ecb511f

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "04c2e87",
3
  "releases": [
4
  {
5
  "version": "v14",
 
1
  {
2
+ "current": "f4a0171",
3
  "releases": [
4
  {
5
  "version": "v14",
VERSION CHANGED
@@ -1 +1 @@
1
- 04c2e87
 
1
+ f4a0171
api/automation_engine.py CHANGED
@@ -1038,7 +1038,7 @@ IG_FIELD_DESCRIPTIONS = {
1038
  "related_accounts": "Accounts Instagram suggests alongside this profile.",
1039
  "replies": "Reply count reported for the comment.",
1040
  "shortcode": "Instagram shortcode that uniquely identifies the post.",
1041
- "source_payload": "Full Bright Data source response, excluding protected contact and comment data.",
1042
  "snapshot_key": "Unique ID for this profile snapshot.",
1043
  "source": "Method or provider used to read this profile.",
1044
  "tagged_location": "Location tagged on the post; it is not the creator's residence.",
@@ -1231,9 +1231,12 @@ def clean_config(kind, raw, previous=None):
1231
  "noFallback": flag("noFallback"),
1232
  # β›” THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
1233
  # for: the Profiles row carries post IDENTITY for free but NO engagement
1234
- # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
1235
- "postMetrics": flag("postMetrics"),
1236
- "dryRun": flag("dryRun"),
 
 
 
1237
  "lanes": lanes,
1238
  "maxPosts": max_posts}, None
1239
  if kind == "discover_instagram":
@@ -1470,7 +1473,8 @@ def _ensure_enrich_step(flow_raw):
1470
  # with only step 1 lands this at the end, which IS step 2.
1471
  seeded.insert(1, {
1472
  "id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
1473
- "config": {"tier": "anonymous", "noFallback": False, "postMetrics": False,
 
1474
  "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
1475
  "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
1476
  "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
@@ -2081,8 +2085,8 @@ IG_MAX_RETRIES = 2
2081
  #: has exactly the same cost as delaying capture β€” for that column β€” and is justified by exactly
2082
  #: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to
2083
  #: be usually-empty costs a blank cell; omitting one costs the months before somebody notices.
2084
- #: β›” THE ONLY EXCLUSIONS ARE RULINGS, NOT TASTE: contact PII (D-24) and third-party comment text
2085
- #: (R11), both enforced in `BD_NEVER_MAP`.
2086
  #: ⚠ A blank in any of these means NOT READ β€” never "they have none". The anonymous rungs expose
2087
  #: almost none of them, which is what the `source` column is for.
2088
  SNAPSHOT_FIELDS = [
@@ -2096,7 +2100,7 @@ SNAPSHOT_FIELDS = [
2096
  # The link in bio β€” the paid rung's field, and commercially the most useful single string an
2097
  # influencer row can carry (it is the shop/affiliate destination).
2098
  field_def("external_url", "Link in bio", "url"),
2099
- # --- the rest of the vendor's Profiles schema, minus the PII and the comment text.
2100
  field_def("ig_id", "Instagram id"),
2101
  field_def("profile_url", "Profile", "url"),
2102
  field_def("avg_engagement", "Avg engagement", "pct"),
@@ -2131,7 +2135,7 @@ POST_FIELDS = [
2131
  # presented as the creator's location: a creator can tag a holiday, venue or brand location.
2132
  field_def("tagged_location", "Tagged location"),
2133
  # The per-field map below gives operational fields first; this locked JSON document retains
2134
- # every other PUBLIC value Bright Data supplied, so a vendor schema addition is preserved
2135
  # instead of being silently discarded while the canonical field model catches up.
2136
  field_def("source_payload", "Source data", "json"),
2137
  # Sponsored-post detection β€” Β§2c called it one of the fields worth having that the anonymous
@@ -2203,9 +2207,9 @@ POST_FIELDS = [
2203
  #: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as
2204
  #: `config.postMetrics` and the Bright Data money switch W26/R15 kept β€” the relation costs
2205
  #: nothing; the spend and the third-party ingest are a click somebody makes knowingly.
2206
- #: ⚠ `BD_NEVER_MAP` STILL STRIPS `latest_comments` β€” the array of 15 comment texts a POSTS row
2207
- #: carries whether we asked or not. Turning this on does not open that door; it is a separate
2208
- #: dataset with a separate decision, and R11's enforcement point is unchanged.
2209
  COMMENT_FIELDS = [
2210
  field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
2211
  field_def("influencer_key", "Influencer"),
@@ -2217,11 +2221,10 @@ COMMENT_FIELDS = [
2217
  description="Post record linked to this comment.",
2218
  link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
2219
  "single": True}),
2220
- # ⚠ NO AUTHOR COLUMN AND NO TEXT COLUMN, and their absence is the ruling rather than an
2221
- # oversight. `comment_user` is vendor-flagged PII and the text is the payload D-22 priced.
2222
- # What is left is the SHAPE of a thread β€” how many, how recent, how engaged β€” which is the
2223
- # part that informs an influencer decision without ingesting a stranger.
2224
- ]
2225
  POST_SNAPSHOT_FIELDS = [
2226
  field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
2227
  field_def("influencer_key", "Influencer"),
@@ -2231,6 +2234,7 @@ POST_SNAPSHOT_FIELDS = [
2231
  # thing to derive. Paid rung only; blank means not read.
2232
  field_def("views", "Views", "int"),
2233
  field_def("plays", "Plays", "int"),
 
2234
  field_def("post_link", "Post", "link",
2235
  description="Post record linked to this measurement.",
2236
  link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
@@ -2382,6 +2386,7 @@ BD_BASE_DEFAULT = "https://api.brightdata.com"
2382
  BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records
2383
  BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields
2384
  BD_DS_REELS = "gd_lyclm20il4r5helnj" # Instagram Reels: views and play counts
 
2385
  BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
2386
  BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"}
2387
  BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
@@ -2557,30 +2562,21 @@ def _first(node, *names, default=None):
2557
  return default
2558
 
2559
 
2560
- #: β›” FIELDS THAT ARE NEVER MAPPED, WHATEVER THE VENDOR SENDS. Two separate reasons, one list,
2561
- #: because the enforcement is the same: a field never collected needs no erasure path (D-24).
2562
- #: PII (the vendor's own `pii` flag): contact details are a different ORDER of privacy exposure
2563
- #: than a follower count, and `email_address` + `phone_number` change the lawful-basis analysis
2564
- #: materially. `business_email`'s own description says it "no longer exists without login!".
2565
- #: R11 (owner ruling, wave 20): NO Instagram comment-text capture. ⚠ This is not merely "do not
2566
- #: buy the Comments dataset" β€” MEASURED 2026-08-05, a POSTS row arrives with `latest_comments`
2567
- #: as an array of 15 comment texts WITH their authors, whether we asked or not. The ruling is
2568
- #: enforced HERE, at the map, because that is the only place it can be.
2569
- BD_NEVER_MAP = frozenset({
2570
- "email_address", "phone_number", "business_email", "profile_image_link",
2571
- "profile_pic_url", "latest_comments", "top_comments", "comment_user",
2572
- "comment_user_url", "post_user", "comment", "user_commenting", "avatar",
2573
- })
2574
 
2575
- # `json` cells are capped by the table layer. Keep this local literal with the same boot-path
2576
- # discipline as the other user-table contracts in this module: importing `core` here would make
2577
- # the API's automation boot path depend on the whole platform package.
2578
- BD_PUBLIC_POST_PAYLOAD_MAX = 32 * 1024
2579
 
2580
  #: The vendor's content vocabulary β†’ ours. MEASURED values: profiles' `posts[].content_type` is
2581
  #: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`.
2582
- _BD_TYPES = {"image": "image", "photo": "image", "video": "video", "reel": "video",
2583
- "reels": "video", "clips": "video", "igtv": "video", "carousel": "carousel",
 
2584
  "carousel_container": "carousel", "sidecar": "carousel", "posts": "image"}
2585
 
2586
 
@@ -2686,67 +2682,89 @@ def _bd_tagged_location(node):
2686
  return ", ".join(parts)[:400] or None
2687
 
2688
 
2689
- def _bd_public_post_payload(row):
2690
- """A canonical JSON copy of every public Post/Reel response field.
2691
 
2692
- This is intentionally a *copy*, not the original mutable provider dict. The PII/comment rail
2693
- applies before serialisation, so neither `latest_comments` nor Reels' `top_comments` can slip
2694
- into a generic "all fields" document. The table's JSON cap is observed before a value reaches
2695
- the write door; an over-cap response is omitted rather than truncated into invalid JSON.
 
2696
  """
2697
  if not isinstance(row, dict):
2698
  return None
2699
- public = {str(k): v for k, v in row.items() if str(k) not in BD_NEVER_MAP}
2700
  try:
2701
- packed = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
 
2702
  except (TypeError, ValueError):
2703
  return None
2704
- return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
 
 
 
 
 
2705
 
2706
 
2707
- def _bd_public_profile_payload(node):
2708
- """The Profile counterpart to the Post source document.
2709
 
2710
- A profile scrape embeds its latest `posts` list. That list already becomes canonical linked
2711
- Post rows, so retaining it here would recreate the retired Recent Posts JSON cache. All other
2712
- public source fields are preserved, subject to the same contact/comment boundary as Posts.
2713
- """
2714
- if not isinstance(node, dict):
2715
- return None
2716
- public = {str(k): v for k, v in node.items()
2717
- if str(k) not in BD_NEVER_MAP and str(k) != "posts"}
2718
- try:
2719
- packed = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
2720
- except (TypeError, ValueError):
2721
- return None
2722
- return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
2723
-
2724
-
2725
- def _bd_comment(row, influencer_key="", shortcode=""):
2726
- """One Bright Data Comments response -> the complete permitted Comment record.
2727
-
2728
- The actual Comments endpoint remains an explicit paid opt-in. This pure mapper means that
2729
- when it is enabled, every allowed response field is preserved in `source_payload` rather than
2730
- reintroducing a partial child-table schema. Third-party author identity and comment text stay
2731
- outside both the first-class fields and the source document.
2732
  """
2733
  if not isinstance(row, dict):
2734
  return None
2735
  url = str(_first(row, "url", "post_url", default="") or "")
2736
  code = str(shortcode or "") or ig_shortcode(url)
2737
- comment_id = str(_first(row, "comment_id", "id", "pk", default="") or "")
2738
  if not code or not comment_id:
2739
  return None
2740
- posted = str(_first(row, "comment_date", "date_of_comment", default="") or "")
 
2741
  return {
2742
  "comment_key": comment_id,
2743
  "shortcode": code,
2744
  "influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
2745
  "commented_at": posted.replace("T", " ")[:16],
2746
- "likes": _ig_int(_first(row, "likes_number", "likes", "like_count")),
2747
- "replies": _ig_int(_first(row, "replies_number", "num_replies", "replies")),
2748
- "source_payload": _bd_public_post_payload(row),
 
2749
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2750
 
2751
 
2752
  def _bd_profile(node, handle):
@@ -2792,11 +2810,8 @@ def _bd_profile(node, handle):
2792
  # `wave20-split.md`), and the first four were explicitly logged as *"populated on scrape
2793
  # and NOT in any previous list"* β€” so these are not speculative names, they are fields the
2794
  # vendor was already sending and this map was already throwing away.
2795
- # β›” THE PII RAIL IS UNCHANGED AND IS WHAT BOUNDS "ALL": `BD_NEVER_MAP` still strips
2796
- # `email_address`, `phone_number`, `business_email`, `profile_image_link` (D-24) and
2797
- # `latest_comments` (R11) on the way out. "Every field" means every field two owner
2798
- # rulings have not removed β€” and the filter below is what makes that true rather than
2799
- # promised.
2800
  "profile_name": str(_first(node, "profile_name", default="") or ""),
2801
  "is_joined_recently": _bd_flag(node, "is_joined_recently"),
2802
  "has_channel": _bd_flag(node, "has_channel"),
@@ -2816,12 +2831,9 @@ def _bd_profile(node, handle):
2816
  # was `None` on all five corpus rows. It is a row field regardless, and reading it costs
2817
  # nothing; a blank here means not read, like every other blank in this map.
2818
  "country_code": str(_first(node, "country_code", default="") or ""),
2819
- "source_payload": _bd_public_profile_payload(node),
2820
- }
2821
- # β›” THE PII RAIL, ENFORCED HERE RATHER THAN TRUSTED. Nothing above reads a `BD_NEVER_MAP`
2822
- # field, and this asserts it stays that way after the next edit β€” a map is exactly the kind
2823
- # of code somebody extends in a hurry.
2824
- return {k: v for k, v in out.items() if k not in BD_NEVER_MAP}
2825
 
2826
 
2827
  def _bd_post_identity(p):
@@ -2850,9 +2862,13 @@ def _bd_post_identity(p):
2850
  caption = _first(p, "caption", "description", default="")
2851
  if isinstance(caption, dict):
2852
  caption = _first(caption, "text", default="")
2853
- if caption:
2854
- out["caption"] = str(caption)
2855
- return out
 
 
 
 
2856
 
2857
 
2858
  def _bd_post_metrics(row):
@@ -2867,7 +2883,8 @@ def _bd_post_metrics(row):
2867
  is declared `text`, the others `number`). A candidate list may span types only when the
2868
  MEANING is identical β€” never the way `like_and_view_counts_disabled`, a boolean, would have
2869
  turned "likes are hidden" into "1 like".
2870
- β›” `latest_comments` IS PRESENT ON THESE ROWS AND IS NEVER READ (R11 / `BD_NEVER_MAP`).
 
2871
  """
2872
  if not isinstance(row, dict):
2873
  return None
@@ -2899,12 +2916,12 @@ def _bd_post_metrics(row):
2899
  "hashtags": _bd_list(row, "hashtags"),
2900
  "alt_text": str(_first(row, "alt_text", default="") or ""),
2901
  "tagged_location": _bd_tagged_location(row),
2902
- "source_payload": _bd_public_post_payload(row),
2903
- }
2904
- # β›” SAME RAIL AS THE PROFILE MAP β€” and it matters MORE here, because a Posts row genuinely
2905
- # arrives carrying `latest_comments`: fifteen third-party comment texts with their authors,
2906
- # sent whether we asked or not (R11).
2907
- return {k: v for k, v in out.items() if k not in BD_NEVER_MAP}
2908
 
2909
 
2910
  def _bd_rows(payload):
@@ -3057,21 +3074,23 @@ def bd_corpus_profile(handle, log=print):
3057
  f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building β€” it is not lost)")
3058
 
3059
 
3060
- def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, log=print):
 
3061
  """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides.
3062
 
3063
  Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner
3064
  treats every rung identically. `blocked` here means the VENDOR rung did not answer β€” the
3065
  caller decides whether to drop to the anonymous ladder.
3066
 
3067
- `post_metrics=True` adds the second, PAID-PER-POST call against the Posts dataset, which is
3068
- the ONLY source of likes/comments (see `_bd_post_identity`). It is off by default because it
3069
- multiplies the bill by roughly the post count.
 
3070
  """
3071
  handle = ig_handle(url)
3072
  if not handle:
3073
- return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
3074
- "profile": {}, "posts": [], "via": ""}
3075
  rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"])
3076
  node = rows[0] if rows else {}
3077
  profile = _bd_profile(node, handle) if node else {}
@@ -3106,13 +3125,13 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
3106
  f"the vendor's pre-collected profile corpus β€” the values are its last "
3107
  f"capture, not a read taken just now")
3108
  else:
3109
- return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
3110
  "note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"}
3111
  if profile.get("followers") is None and profile.get("following") is None:
3112
  # It answered 200 with something we could not read. Say THAT β€” not "0 followers".
3113
  # ⚠ `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and
3114
  # is discarded, so a row where it is the only "readable" field is a row we cannot read.
3115
- return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
3116
  "note": "the profile source answered, but no follower/following counts were readable in "
3117
  "it (the field names may have moved β€” see instagram-capture.md)"}
3118
  posts = []
@@ -3129,10 +3148,11 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
3129
  # without duplicating any record, so enabling post metrics still buys one row per post.
3130
  ordinary = [p for p in posts if p.get("type") != "video"]
3131
  videos = [p for p in posts if p.get("type") == "video"]
3132
- rows2, metric_notes = [], []
3133
  if ordinary:
3134
  got_rows, got_note = bd_scrape(BD_DS_POSTS, [p["url"] for p in ordinary])
3135
  rows2.extend(got_rows)
 
3136
  if got_note:
3137
  metric_notes.append(f"posts: {got_note}")
3138
  if videos:
@@ -3140,6 +3160,21 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
3140
  rows2.extend(got_rows)
3141
  if got_note:
3142
  metric_notes.append(f"reels: {got_note}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3143
  if metric_notes and not rows2:
3144
  # The engagement half refusing does NOT lose the identity half β€” the profile and the
3145
  # post rows still land, and the run says which part was not readable.
@@ -3160,6 +3195,23 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
3160
  note = "post metrics were requested but the Posts dataset returned no rows"
3161
  elif metric_notes:
3162
  note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3163
 
3164
  # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften
3165
  # because we are paying: a run that wrote a follower count and no posts must not paint green
@@ -3168,8 +3220,9 @@ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, l
3168
  # snapshot row's `source` column, so a stored measurement can always say whether it was read
3169
  # live or taken from the vendor's pre-collected corpus. Two routes writing one indisting-
3170
  # uishable `source` would make the series unauditable exactly where it matters most.
3171
- return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts,
3172
- "via": via,
 
3173
  "note": note or ("" if posts else "profile read; no posts were returned")}
3174
 
3175
 
@@ -3513,8 +3566,8 @@ def filter_meta():
3513
  #: not carry it", not "this account does not have it" β€” which is exactly what `enriched_at` and
3514
  #: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a
3515
  #: sometimes-blank column earn a place on the grid", and that was always the owner's call to make.
3516
- #: β›” WHAT DID *NOT* CHANGE: `BD_NEVER_MAP`. Contact PII (D-24) and third-party comment text (R11)
3517
- #: are owner RULINGS, not fill-rate judgements, and "all fields" does not reach them.
3518
  #:
3519
  #: ⭐ WAVE 26 Β· R5 β€” THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is
3520
  #: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`).
@@ -4855,9 +4908,9 @@ def _candidate_row(row, stamp):
4855
  }
4856
 
4857
 
4858
- def pull_profile(url, max_posts=DEFAULT_POSTS_PER_PULL, log=print, tier="anonymous",
4859
- fallback=True,
4860
- post_metrics=False):
4861
  """Everything readable about one public profile, cheapest-honest rung last.
4862
 
4863
  Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error.
@@ -4877,17 +4930,18 @@ def pull_profile(url, max_posts=DEFAULT_POSTS_PER_PULL, log=print, tier="anonymo
4877
  """
4878
  handle = ig_handle(url)
4879
  if not handle:
4880
- return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
4881
- "profile": {}, "posts": [], "via": ""}
4882
  attempts = []
4883
 
4884
  if clean_tier(tier) == "brightdata":
4885
- paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics, log=log)
 
4886
  if paid["state"] in ("ok", "partial"):
4887
  return paid
4888
  attempts.append(f"brightdata:{_s(paid.get('note'), 90)}")
4889
  if not fallback:
4890
- return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata",
4891
  "note": "the paid rung did not answer and the anonymous fallback is turned "
4892
  f"off for this automation ({paid.get('note') or 'blocked'})"}
4893
  time.sleep(PACE_SECONDS)
@@ -5147,8 +5201,8 @@ def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, row
5147
  return (state, summary, counts, affected, steps)
5148
 
5149
 
5150
- def capture_rows(res, pulled):
5151
- """ONE pull β†’ `(snapshot_row, post_identity_rows, post_metric_rows)`.
5152
 
5153
  ⭐ WAVE 25 (C4) β€” FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT
5154
  RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path β€”
@@ -5185,18 +5239,21 @@ def capture_rows(res, pulled):
5185
  # Profile history declares the same percentage dialect as the current Profile row.
5186
  # Converting only the latest projection left the append series at 0-1 while its field
5187
  # now said pct (0-100), so the two views of one measurement disagreed by 100x.
5188
- n = BD_PUBLIC_POST_PAYLOAD_MAX if k == "source_payload" else 400
5189
- snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, n)
5190
- idents, metrics_rows = [], []
 
 
5191
  for p in (res or {}).get("posts") or []:
5192
  ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
5193
  "url": _s(p.get("url"), 300)}
5194
  for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
5195
  ("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
5196
- ("alt_text", 400), ("tagged_location", 400),
5197
- ("source_payload", BD_PUBLIC_POST_PAYLOAD_MAX)):
5198
- if p.get(k):
5199
- ident[k] = _s(p.get(k), n)
 
5200
  metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
5201
  # ⭐⭐ 2026-08-07 β€” THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
5202
  #
@@ -5222,14 +5279,28 @@ def capture_rows(res, pulled):
5222
  # no stamp it means that AND "we never looked", indistinguishably.
5223
  ident["measured_at"] = _day(pulled)
5224
  idents.append(ident)
5225
- if any(v is not None for v in metrics.values()):
5226
  metrics_rows.append({
5227
  "post_snapshot_key": f"{p['shortcode']}@{pulled}",
5228
  "shortcode": p["shortcode"], "influencer_key": prof.get("username"),
5229
  "pulled_at": pulled,
5230
  "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
5231
- "views": _s(metrics["views"]), "plays": _s(metrics["plays"])})
5232
- return snap_row, idents, metrics_rows
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5233
 
5234
 
5235
  #: C4 β€” how ONE pulled profile becomes the C1 preset CELLS on the record being enriched.
@@ -5291,7 +5362,7 @@ def preset_cells(res, pulled):
5291
  # straight-through copy here would write the same wrong number the discovery path
5292
  # used to write. Both writers convert; neither is the exception.
5293
  cells[dest] = (_pct100(v) if dest == "avg_engagement" else
5294
- _s(v, BD_PUBLIC_POST_PAYLOAD_MAX if dest == "source_payload" else 500))
5295
  # ⭐ R5 β€” stamped by the writer, not carried from the pull: this function only ever sees an
5296
  # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
5297
  # arrived with no platform at all β€” and a blank half of the dedup key is how one account
@@ -5312,9 +5383,10 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
5312
  cfg = defn.get("config") or {}
5313
  table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
5314
  tier = clean_tier(cfg.get("tier")) or "anonymous"
5315
- paid = tier == "brightdata"
5316
- fallback = not cfg.get("noFallback")
5317
- post_metrics = bool(cfg.get("postMetrics"))
 
5318
  dry = bool(cfg.get("dryRun"))
5319
  steps = {"trigger": "ok", "source": "idle", "capture": "idle",
5320
  "capture_paid": "idle" if paid else "skipped",
@@ -5334,27 +5406,28 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
5334
  # the relational tables the pull lands in (R7): every row timestamped for time-range filters
5335
  if dry:
5336
  # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
5337
- snap_key, post_key, ps_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE,
5338
- IG_POST_SNAPSHOTS_TABLE)
5339
  else:
5340
  graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
5341
  profile_table=table_key)
5342
- snap_key, post_key, ps_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
5343
- graph[IG_POST_SNAPSHOTS_TABLE])
5344
- missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key)
5345
  snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
5346
- posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
5347
- psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {})
 
5348
 
5349
  counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0,
5350
- "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0,
5351
  "missing_tables": len(missing)}
5352
  cells, affected, notes = {}, [], []
5353
  # β›” ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows`
5354
  # once per POST, which is O(existing) per call β€” survivable only while the cap was 5000 rows.
5355
  # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
5356
  # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
5357
- in_snaps, in_posts, in_psnaps = [], [], []
5358
  paid_tried = paid_answered = anon_tried = anon_answered = False
5359
  targets = [(rid, str(r.get(url_field, "") or "").strip())
5360
  for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
@@ -5366,8 +5439,9 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
5366
  # between "working through them" and "stuck".
5367
  step(f"Capturing profile {i + 1} of {len(targets)}")
5368
  counts["profiles"] += 1
5369
- res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
5370
- tier=tier, fallback=fallback, post_metrics=post_metrics)
 
5371
  pulled = _iso()
5372
  via = res.get("via") or ""
5373
  read_ok = res["state"] in ("ok", "partial")
@@ -5386,11 +5460,13 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
5386
  if read_ok:
5387
  counts["ok" if res["state"] == "ok" else "partial"] += 1
5388
  prof = res["profile"]
5389
- snap_row, ident_rows, metric_rows = capture_rows(res, pulled)
5390
  in_snaps.append(snap_row)
5391
- in_posts.extend(ident_rows)
5392
- in_psnaps.extend(metric_rows)
5393
- counts["metrics"] += len(metric_rows)
 
 
5394
  counts["posts"] += len(res.get("posts") or [])
5395
  detail = f"{prof.get('followers') or '?'} followers"
5396
  if prof.get("approx"):
@@ -5415,10 +5491,13 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
5415
  posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at")
5416
  posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
5417
  c_post["duplicates"] += collapsed_posts
5418
- psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
5419
- counts["new_posts"] = c_post["inserted"]
5420
- capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]),
5421
- (ps_key, c_ps["capped"])]
 
 
 
5422
  counts["capped"] = sum(n for _k, n in capped)
5423
 
5424
  # R5 β€” route each processed record down its winning lane, judged over the row's FINAL
@@ -5448,7 +5527,8 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
5448
  tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
5449
  for rid, vals in stage_writes.items():
5450
  tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals)
5451
- for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)):
 
5452
  tgt = cur.get(k)
5453
  if tgt is not None:
5454
  tgt["rows"] = rws
@@ -5515,12 +5595,16 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste
5515
  summary += f", {counts['routed']}/{len(targets)} routed to lanes"
5516
  if counts["paid"]:
5517
  summary += f", {counts['paid']} with exact counts"
5518
- if counts["metrics"]:
5519
- summary += f", {counts['metrics']} post engagement snapshots"
5520
  elif post_metrics and counts["posts"]:
5521
  # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there
5522
  # was no engagement", which is a claim about Instagram rather than about our run.
5523
- summary += ", no post engagement was readable"
 
 
 
 
5524
  if counts["partial"]:
5525
  summary += f", {counts['partial']} profile-only (posts not readable)"
5526
  if counts["blocked"]:
@@ -6834,9 +6918,12 @@ def _clean_action_config(kind, cfg, depth, seen, count):
6834
  # not be the thing that starts spending money (the `clean_config` rule, verbatim).
6835
  "tier": clean_tier(cfg.get("tier")) or "anonymous",
6836
  "noFallback": bool(cfg.get("noFallback")),
6837
- # β›” THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
6838
- "postMetrics": bool(cfg.get("postMetrics")),
6839
- "dryRun": bool(cfg.get("dryRun")),
 
 
 
6840
  "maxPosts": max_posts}, None
6841
  if kind == "find_records":
6842
  table = _s(cfg.get("table"), 60).strip()
@@ -7478,7 +7565,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
7478
  # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
7479
  # entry, so calling it per record is O(existing) per record β€” survivable at 5,000 rows and an
7480
  # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
7481
- enrich = {"snaps": [], "posts": [], "psnaps": [], "profiles": 0, "ok": 0, "blocked": 0,
7482
  "notes": [], "dry": False}
7483
  # ⭐ 2026-08-07 β€” THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
7484
  # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
@@ -7644,16 +7731,18 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
7644
  res = pull_profile(handle_raw,
7645
  max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
7646
  log=log,
7647
- tier=clean_tier(cfg.get("tier")) or "anonymous",
7648
- fallback=not cfg.get("noFallback"),
7649
- post_metrics=bool(cfg.get("postMetrics")))
 
7650
  pulled = _iso()
7651
  if res["state"] in ("ok", "partial"):
7652
  enrich["ok"] += 1
7653
- snap_row, idents, metric_rows = capture_rows(res, pulled)
7654
  enrich["snaps"].append(snap_row)
7655
- enrich["posts"].extend(idents)
7656
- enrich["psnaps"].extend(metric_rows)
 
7657
  # R3: LATEST onto the record itself. `row.update` too, so a LATER action in
7658
  # this same flow sees the enriched values β€” the rule `update_record` already
7659
  # follows, and without it "enrich, then route by follower count" would judge
@@ -7783,10 +7872,11 @@ def migrate_field_instagram(rt, defn):
7783
  max_posts = 24
7784
  act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
7785
  "config": {"profileField": bound,
7786
- "tier": clean_tier(cfg.get("tier")) or "anonymous",
7787
- "noFallback": bool(cfg.get("noFallback")),
7788
- "postMetrics": bool(cfg.get("postMetrics")),
7789
- "dryRun": bool(cfg.get("dryRun")),
 
7790
  "maxPosts": max(1, min(max_posts, 200))}}
7791
  flow = dict(defn.get("flow") or {})
7792
  out = dict(defn)
@@ -7803,8 +7893,8 @@ def migrate_field_instagram(rt, defn):
7803
  return out, True
7804
 
7805
 
7806
- def _enrich_flush(rt, defn, username, acc, log):
7807
- """C4: the enrich action's THREE append tables, written ONCE for the whole run.
7808
 
7809
  Returns the counts the run reports. Empty when no enrich action ran, so a flow without one
7810
  pays nothing and its run history is unchanged (`run_now` merges only non-zero keys).
@@ -7817,7 +7907,7 @@ def _enrich_flush(rt, defn, username, acc, log):
7817
  if not acc.get("profiles"):
7818
  return {}
7819
  out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
7820
- if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"]):
7821
  # A dry run resolves nothing and writes nothing β€” not even `ut_ensure`, which CREATES.
7822
  if acc.get("dry"):
7823
  out["enrichDryRun"] = acc["profiles"]
@@ -7825,21 +7915,27 @@ def _enrich_flush(rt, defn, username, acc, log):
7825
  tag = str(defn.get("id") or "")
7826
  profile_table = str(_flow_table(defn) or "")
7827
  graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table)
7828
- snap_key, post_key, ps_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
7829
- graph[IG_POST_SNAPSHOTS_TABLE])
7830
- missing = ut_missing(rt, snap_key, post_key, ps_key)
7831
  snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
7832
  acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
7833
  old_posts, collapsed_posts = dedupe_canonical_rows(
7834
  dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
7835
  posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key))
7836
  c_post["duplicates"] += collapsed_posts
7837
- psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
7838
- acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
 
 
 
 
 
7839
 
7840
  def _up(cur):
7841
  cur = cur if isinstance(cur, dict) else {}
7842
- for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)):
 
7843
  if cur.get(k) is not None:
7844
  cur[k]["rows"] = rws
7845
  _refresh_relations_inplace(cur, log=log)
@@ -7848,15 +7944,16 @@ def _enrich_flush(rt, defn, username, acc, log):
7848
  rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
7849
  # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
7850
  # is the failure a chart cannot show you.
7851
- capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"]
7852
  if capped:
7853
  out["enrichCapped"] = capped
7854
  log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap")
7855
  if missing:
7856
  out["enrichMissingTables"] = len(missing)
7857
  log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}")
7858
- out["enrichPoints"] = len(acc["snaps"])
7859
- out["enrichPosts"] = c_post["inserted"]
 
7860
  # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated β€” the same
7861
  # contract `run_field_instagram` follows, because a pooled history with silent holes is worse
7862
  # than none.
 
1038
  "related_accounts": "Accounts Instagram suggests alongside this profile.",
1039
  "replies": "Reply count reported for the comment.",
1040
  "shortcode": "Instagram shortcode that uniquely identifies the post.",
1041
+ "source_payload": "Full Bright Data source response for this record.",
1042
  "snapshot_key": "Unique ID for this profile snapshot.",
1043
  "source": "Method or provider used to read this profile.",
1044
  "tagged_location": "Location tagged on the post; it is not the creator's residence.",
 
1231
  "noFallback": flag("noFallback"),
1232
  # β›” THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
1233
  # for: the Profiles row carries post IDENTITY for free but NO engagement
1234
+ # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
1235
+ "postMetrics": flag("postMetrics"),
1236
+ # The full Comments dataset can bill many rows per post. It is an explicit,
1237
+ # independent opt-in and never follows the per-post switch automatically.
1238
+ "commentMetrics": flag("commentMetrics"),
1239
+ "dryRun": flag("dryRun"),
1240
  "lanes": lanes,
1241
  "maxPosts": max_posts}, None
1242
  if kind == "discover_instagram":
 
1473
  # with only step 1 lands this at the end, which IS step 2.
1474
  seeded.insert(1, {
1475
  "id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
1476
+ "config": {"tier": "anonymous", "noFallback": False, "postMetrics": False,
1477
+ "commentMetrics": False,
1478
  "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
1479
  "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
1480
  "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
 
2085
  #: has exactly the same cost as delaying capture β€” for that column β€” and is justified by exactly
2086
  #: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to
2087
  #: be usually-empty costs a blank cell; omitting one costs the months before somebody notices.
2088
+ #: Every returned provider field is retained in Source data. Promoted columns make commonly used
2089
+ #: values filterable; the raw source document prevents a new or uncommon field from being lost.
2090
  #: ⚠ A blank in any of these means NOT READ β€” never "they have none". The anonymous rungs expose
2091
  #: almost none of them, which is what the `source` column is for.
2092
  SNAPSHOT_FIELDS = [
 
2100
  # The link in bio β€” the paid rung's field, and commercially the most useful single string an
2101
  # influencer row can carry (it is the shop/affiliate destination).
2102
  field_def("external_url", "Link in bio", "url"),
2103
+ # --- the rest of the vendor's Profiles schema.
2104
  field_def("ig_id", "Instagram id"),
2105
  field_def("profile_url", "Profile", "url"),
2106
  field_def("avg_engagement", "Avg engagement", "pct"),
 
2135
  # presented as the creator's location: a creator can tag a holiday, venue or brand location.
2136
  field_def("tagged_location", "Tagged location"),
2137
  # The per-field map below gives operational fields first; this locked JSON document retains
2138
+ # every other value Bright Data supplied, so a vendor schema addition is preserved
2139
  # instead of being silently discarded while the canonical field model catches up.
2140
  field_def("source_payload", "Source data", "json"),
2141
  # Sponsored-post detection β€” Β§2c called it one of the fields worth having that the anonymous
 
2207
  #: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as
2208
  #: `config.postMetrics` and the Bright Data money switch W26/R15 kept β€” the relation costs
2209
  #: nothing; the spend and the third-party ingest are a click somebody makes knowingly.
2210
+ #: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because
2211
+ #: they are part of a record already paid for. The separate full Comments dataset remains OFF
2212
+ #: unless `commentMetrics` is explicitly enabled on the enrichment action.
2213
  COMMENT_FIELDS = [
2214
  field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
2215
  field_def("influencer_key", "Influencer"),
 
2221
  description="Post record linked to this comment.",
2222
  link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
2223
  "single": True}),
2224
+ # Author/text and every provider-specific value stay intact in Source data. The promoted
2225
+ # columns intentionally keep the grid concise while Likes and Replies make comment engagement
2226
+ # directly filterable and rollup-ready.
2227
+ ]
 
2228
  POST_SNAPSHOT_FIELDS = [
2229
  field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
2230
  field_def("influencer_key", "Influencer"),
 
2234
  # thing to derive. Paid rung only; blank means not read.
2235
  field_def("views", "Views", "int"),
2236
  field_def("plays", "Plays", "int"),
2237
+ field_def("source_payload", "Source data", "json"),
2238
  field_def("post_link", "Post", "link",
2239
  description="Post record linked to this measurement.",
2240
  link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
 
2386
  BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records
2387
  BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields
2388
  BD_DS_REELS = "gd_lyclm20il4r5helnj" # Instagram Reels: views and play counts
2389
+ BD_DS_COMMENTS = "gd_ltppn085pokosxh13" # Instagram Comments: opt-in full comment engagement
2390
  BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
2391
  BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"}
2392
  BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
 
2562
  return default
2563
 
2564
 
2565
+ #: Source-data retention is intentionally unfiltered: if Bright Data returned it in an already
2566
+ #: paid Profile, Post, Reel, or Comment response, it belongs in that record's source document.
2567
+ #: The one cost boundary is the *request*: the full Comments dataset is only called when an owner
2568
+ #: enables `commentMetrics`.
 
 
 
 
 
 
 
 
 
 
2569
 
2570
+ #: `bd_call` refuses a partial response at this same transport ceiling. A single source document
2571
+ #: therefore fits a JSON cell; Source data is never shortened by `_s`, which would corrupt JSON
2572
+ #: and silently lose fields.
2573
+ BD_SOURCE_PAYLOAD_MAX = BD_MAX_KB * 1024
2574
 
2575
  #: The vendor's content vocabulary β†’ ours. MEASURED values: profiles' `posts[].content_type` is
2576
  #: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`.
2577
+ _BD_TYPES = {"image": "image", "images": "image", "photo": "image", "photos": "image",
2578
+ "video": "video", "videos": "video", "reel": "video", "reels": "video",
2579
+ "clip": "video", "clips": "video", "igtv": "video", "carousel": "carousel",
2580
  "carousel_container": "carousel", "sidecar": "carousel", "posts": "image"}
2581
 
2582
 
 
2682
  return ", ".join(parts)[:400] or None
2683
 
2684
 
2685
+ def _bd_source_payload(row):
2686
+ """One complete Bright Data response row as valid JSON, without field filtering.
2687
 
2688
+ Source data is the lossless provider record, not a second relationship cache. A Profile's
2689
+ embedded ``posts`` array stays here *and* each parseable post becomes a linked Post record;
2690
+ keeping both preserves data while Links remain the interactive relational view. `bd_call`
2691
+ rejects partial bodies at ``BD_SOURCE_PAYLOAD_MAX``, and the table-layer JSON ceiling matches
2692
+ it, so this function never truncates a paid source response.
2693
  """
2694
  if not isinstance(row, dict):
2695
  return None
 
2696
  try:
2697
+ packed = json.dumps({str(k): v for k, v in row.items()}, ensure_ascii=False,
2698
+ sort_keys=True, separators=(",", ":"))
2699
  except (TypeError, ValueError):
2700
  return None
2701
+ # A source row is bounded by the complete raw HTTP response. Leave this defensive assertion
2702
+ # explicit: should the transport ceiling and JSON-cell ceiling ever diverge, fail visibly
2703
+ # rather than shortening a document and pretending every paid field was retained.
2704
+ if len(packed.encode("utf-8")) > BD_SOURCE_PAYLOAD_MAX:
2705
+ raise ValueError("a Bright Data source row exceeded the guarded response ceiling")
2706
+ return packed
2707
 
2708
 
2709
+ def _bd_comment(row, influencer_key="", shortcode="", fallback_key=""):
2710
+ """One Bright Data Comment response -> one linked Comment record with engagement.
2711
 
2712
+ The dedicated Comments dataset is invoked only by the default-off `commentMetrics` switch.
2713
+ This mapper is also used for comment arrays embedded in an already-paid Post/Reel response,
2714
+ which adds no request. The full provider object is preserved in Source data; Likes and Replies
2715
+ are promoted for the grid and future rollups.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2716
  """
2717
  if not isinstance(row, dict):
2718
  return None
2719
  url = str(_first(row, "url", "post_url", default="") or "")
2720
  code = str(shortcode or "") or ig_shortcode(url)
2721
+ comment_id = str(_first(row, "comment_id", "id", "pk", default="") or fallback_key or "")
2722
  if not code or not comment_id:
2723
  return None
2724
+ posted = str(_first(row, "comment_date", "date_of_comment", "created_at", "created_time",
2725
+ default="") or "")
2726
  return {
2727
  "comment_key": comment_id,
2728
  "shortcode": code,
2729
  "influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
2730
  "commented_at": posted.replace("T", " ")[:16],
2731
+ "likes": _ig_int(_first(row, "likes_number", "likes", "like_count", "likes_count")),
2732
+ "replies": _ig_int(_first(row, "replies_number", "num_replies", "replies",
2733
+ "reply_count", "replies_count")),
2734
+ "source_payload": _bd_source_payload(row),
2735
  }
2736
+
2737
+
2738
+ def _bd_embedded_comments(post, influencer_key=""):
2739
+ """Comment rows carried inside a paid Profile/Post/Reel record, with no extra request."""
2740
+ if not isinstance(post, dict):
2741
+ return []
2742
+ shortcode = str(post.get("shortcode") or "") or ig_shortcode(post.get("url"))
2743
+ if not shortcode:
2744
+ return []
2745
+ out, seen = [], set()
2746
+ # Known Bright Data arrays plus any future list-valued comment field. Source data still keeps
2747
+ # unknown shapes even if it cannot be normalised into a Comment row.
2748
+ arrays = [(str(k), v) for k, v in post.items()
2749
+ if isinstance(v, list) and (str(k) in ("latest_comments", "top_comments")
2750
+ or "comment" in str(k).lower())]
2751
+ for source_key, values in arrays:
2752
+ for index, value in enumerate(values):
2753
+ if not isinstance(value, dict):
2754
+ continue
2755
+ try:
2756
+ raw = json.dumps(value, ensure_ascii=False, sort_keys=True,
2757
+ separators=(",", ":"))
2758
+ except (TypeError, ValueError):
2759
+ continue
2760
+ fallback = "embedded:" + hashlib.sha256(
2761
+ f"{shortcode}:{source_key}:{index}:{raw}".encode("utf-8")).hexdigest()[:24]
2762
+ got = _bd_comment(value, influencer_key=influencer_key, shortcode=shortcode,
2763
+ fallback_key=fallback)
2764
+ if got and got["comment_key"] not in seen:
2765
+ seen.add(got["comment_key"])
2766
+ out.append(got)
2767
+ return out
2768
 
2769
 
2770
  def _bd_profile(node, handle):
 
2810
  # `wave20-split.md`), and the first four were explicitly logged as *"populated on scrape
2811
  # and NOT in any previous list"* β€” so these are not speculative names, they are fields the
2812
  # vendor was already sending and this map was already throwing away.
2813
+ # Every unpromoted provider key β€” including contact and image fields β€” stays complete in
2814
+ # Source data. This keeps the locked grid readable without throwing away paid data.
 
 
 
2815
  "profile_name": str(_first(node, "profile_name", default="") or ""),
2816
  "is_joined_recently": _bd_flag(node, "is_joined_recently"),
2817
  "has_channel": _bd_flag(node, "has_channel"),
 
2831
  # was `None` on all five corpus rows. It is a row field regardless, and reading it costs
2832
  # nothing; a blank here means not read, like every other blank in this map.
2833
  "country_code": str(_first(node, "country_code", default="") or ""),
2834
+ "source_payload": _bd_source_payload(node),
2835
+ }
2836
+ return out
 
 
 
2837
 
2838
 
2839
  def _bd_post_identity(p):
 
2862
  caption = _first(p, "caption", "description", default="")
2863
  if isinstance(caption, dict):
2864
  caption = _first(caption, "text", default="")
2865
+ if caption:
2866
+ out["caption"] = str(caption)
2867
+ out["source_payload"] = _bd_source_payload(p)
2868
+ embedded = _bd_embedded_comments(p)
2869
+ if embedded:
2870
+ out["embedded_comments"] = embedded
2871
+ return out
2872
 
2873
 
2874
  def _bd_post_metrics(row):
 
2883
  is declared `text`, the others `number`). A candidate list may span types only when the
2884
  MEANING is identical β€” never the way `like_and_view_counts_disabled`, a boolean, would have
2885
  turned "likes are hidden" into "1 like".
2886
+ Embedded comment arrays are retained in Source data and converted to linked Comment rows
2887
+ without triggering the separate full Comments-dataset request.
2888
  """
2889
  if not isinstance(row, dict):
2890
  return None
 
2916
  "hashtags": _bd_list(row, "hashtags"),
2917
  "alt_text": str(_first(row, "alt_text", default="") or ""),
2918
  "tagged_location": _bd_tagged_location(row),
2919
+ "source_payload": _bd_source_payload(row),
2920
+ }
2921
+ embedded = _bd_embedded_comments(row, influencer_key=out["influencer_key"])
2922
+ if embedded:
2923
+ out["embedded_comments"] = embedded
2924
+ return out
2925
 
2926
 
2927
  def _bd_rows(payload):
 
3074
  f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building β€” it is not lost)")
3075
 
3076
 
3077
+ def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False,
3078
+ comment_metrics=False, log=print):
3079
  """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides.
3080
 
3081
  Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner
3082
  treats every rung identically. `blocked` here means the VENDOR rung did not answer β€” the
3083
  caller decides whether to drop to the anonymous ladder.
3084
 
3085
+ `post_metrics=True` adds the paid Post/Reel calls. `comment_metrics=True` independently adds
3086
+ the separate paid Comments-dataset call. Both default off; the latter is never inferred from
3087
+ an enabled post-metrics switch. Embedded comment previews returned by either paid record are
3088
+ retained without that extra Comments request.
3089
  """
3090
  handle = ig_handle(url)
3091
  if not handle:
3092
+ return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
3093
+ "profile": {}, "posts": [], "comments": [], "via": ""}
3094
  rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"])
3095
  node = rows[0] if rows else {}
3096
  profile = _bd_profile(node, handle) if node else {}
 
3125
  f"the vendor's pre-collected profile corpus β€” the values are its last "
3126
  f"capture, not a read taken just now")
3127
  else:
3128
+ return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
3129
  "note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"}
3130
  if profile.get("followers") is None and profile.get("following") is None:
3131
  # It answered 200 with something we could not read. Say THAT β€” not "0 followers".
3132
  # ⚠ `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and
3133
  # is discarded, so a row where it is the only "readable" field is a row we cannot read.
3134
+ return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
3135
  "note": "the profile source answered, but no follower/following counts were readable in "
3136
  "it (the field names may have moved β€” see instagram-capture.md)"}
3137
  posts = []
 
3148
  # without duplicating any record, so enabling post metrics still buys one row per post.
3149
  ordinary = [p for p in posts if p.get("type") != "video"]
3150
  videos = [p for p in posts if p.get("type") == "video"]
3151
+ rows2, metric_notes, ordinary_rows = [], [], []
3152
  if ordinary:
3153
  got_rows, got_note = bd_scrape(BD_DS_POSTS, [p["url"] for p in ordinary])
3154
  rows2.extend(got_rows)
3155
+ ordinary_rows.extend(got_rows)
3156
  if got_note:
3157
  metric_notes.append(f"posts: {got_note}")
3158
  if videos:
 
3160
  rows2.extend(got_rows)
3161
  if got_note:
3162
  metric_notes.append(f"reels: {got_note}")
3163
+ # Bright Data's Profiles feed has used more than one media spelling. If one of those
3164
+ # unrecognised labels sent a Reel to Posts, the Posts response itself identifies the
3165
+ # mistake (`content_type: Reel`) but cannot supply Views. Correct it immediately with a
3166
+ # Reels read, then let the richer response win below. This costs a replacement record only
3167
+ # for a proven misroute; correctly classified images/carousels are never re-read.
3168
+ repair_reels = []
3169
+ for row in ordinary_rows:
3170
+ mapped = _bd_post_metrics(row)
3171
+ if mapped and mapped.get("type") == "video":
3172
+ repair_reels.append(f"https://www.instagram.com/reel/{mapped['shortcode']}/")
3173
+ if repair_reels:
3174
+ got_rows, got_note = bd_scrape(BD_DS_REELS, list(dict.fromkeys(repair_reels)))
3175
+ rows2.extend(got_rows)
3176
+ if got_note:
3177
+ metric_notes.append(f"reel recovery: {got_note}")
3178
  if metric_notes and not rows2:
3179
  # The engagement half refusing does NOT lose the identity half β€” the profile and the
3180
  # post rows still land, and the run says which part was not readable.
 
3195
  note = "post metrics were requested but the Posts dataset returned no rows"
3196
  elif metric_notes:
3197
  note = f"some post metrics were unavailable: {'; '.join(metric_notes)}"
3198
+
3199
+ comments = []
3200
+ if comment_metrics and posts:
3201
+ # This is the cost boundary: do not make the full comment request unless the stored
3202
+ # enrichment action explicitly opted in. A batched call still bills provider records, but
3203
+ # it prevents unnecessary HTTP fan-out and preserves every returned field per comment.
3204
+ time.sleep(min(PACE_SECONDS, 1.0))
3205
+ comment_source, comment_note = bd_scrape(BD_DS_COMMENTS, [p["url"] for p in posts])
3206
+ for row in comment_source:
3207
+ got = _bd_comment(row, influencer_key=profile.get("username") or handle)
3208
+ if got:
3209
+ comments.append(got)
3210
+ if comment_note:
3211
+ note = "; ".join(x for x in (note, f"comment engagement unavailable: {comment_note}") if x)
3212
+ elif not comments:
3213
+ note = "; ".join(x for x in (note,
3214
+ "comment engagement was requested but the Comments dataset returned no rows") if x)
3215
 
3216
  # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften
3217
  # because we are paying: a run that wrote a follower count and no posts must not paint green
 
3220
  # snapshot row's `source` column, so a stored measurement can always say whether it was read
3221
  # live or taken from the vendor's pre-collected corpus. Two routes writing one indisting-
3222
  # uishable `source` would make the series unauditable exactly where it matters most.
3223
+ return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts,
3224
+ "comments": comments,
3225
+ "via": via,
3226
  "note": note or ("" if posts else "profile read; no posts were returned")}
3227
 
3228
 
 
3566
  #: not carry it", not "this account does not have it" β€” which is exactly what `enriched_at` and
3567
  #: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a
3568
  #: sometimes-blank column earn a place on the grid", and that was always the owner's call to make.
3569
+ #: Source data retains every field the provider returns. The only separate decision is whether to
3570
+ #: request the paid full Comments dataset (`commentMetrics`, default OFF).
3571
  #:
3572
  #: ⭐ WAVE 26 Β· R5 β€” THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is
3573
  #: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`).
 
4908
  }
4909
 
4910
 
4911
+ def pull_profile(url, max_posts=DEFAULT_POSTS_PER_PULL, log=print, tier="anonymous",
4912
+ fallback=True,
4913
+ post_metrics=False, comment_metrics=False):
4914
  """Everything readable about one public profile, cheapest-honest rung last.
4915
 
4916
  Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error.
 
4930
  """
4931
  handle = ig_handle(url)
4932
  if not handle:
4933
+ return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
4934
+ "profile": {}, "posts": [], "comments": [], "via": ""}
4935
  attempts = []
4936
 
4937
  if clean_tier(tier) == "brightdata":
4938
+ paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
4939
+ comment_metrics=comment_metrics, log=log)
4940
  if paid["state"] in ("ok", "partial"):
4941
  return paid
4942
  attempts.append(f"brightdata:{_s(paid.get('note'), 90)}")
4943
  if not fallback:
4944
+ return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
4945
  "note": "the paid rung did not answer and the anonymous fallback is turned "
4946
  f"off for this automation ({paid.get('note') or 'blocked'})"}
4947
  time.sleep(PACE_SECONDS)
 
5201
  return (state, summary, counts, affected, steps)
5202
 
5203
 
5204
+ def capture_rows(res, pulled):
5205
+ """ONE pull β†’ `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`.
5206
 
5207
  ⭐ WAVE 25 (C4) β€” FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT
5208
  RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path β€”
 
5239
  # Profile history declares the same percentage dialect as the current Profile row.
5240
  # Converting only the latest projection left the append series at 0-1 while its field
5241
  # now said pct (0-100), so the two views of one measurement disagreed by 100x.
5242
+ if k == "source_payload":
5243
+ snap_row[k] = str(value or "")
5244
+ else:
5245
+ snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400)
5246
+ idents, metrics_rows, comment_rows = [], [], []
5247
  for p in (res or {}).get("posts") or []:
5248
  ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
5249
  "url": _s(p.get("url"), 300)}
5250
  for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
5251
  ("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
5252
+ ("alt_text", 400), ("tagged_location", 400)):
5253
+ if p.get(k):
5254
+ ident[k] = _s(p.get(k), n)
5255
+ if p.get("source_payload"):
5256
+ ident["source_payload"] = str(p["source_payload"])
5257
  metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
5258
  # ⭐⭐ 2026-08-07 β€” THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
5259
  #
 
5279
  # no stamp it means that AND "we never looked", indistinguishably.
5280
  ident["measured_at"] = _day(pulled)
5281
  idents.append(ident)
5282
+ if any(v is not None for v in metrics.values()):
5283
  metrics_rows.append({
5284
  "post_snapshot_key": f"{p['shortcode']}@{pulled}",
5285
  "shortcode": p["shortcode"], "influencer_key": prof.get("username"),
5286
  "pulled_at": pulled,
5287
  "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
5288
+ "views": _s(metrics["views"]), "plays": _s(metrics["plays"]),
5289
+ "source_payload": str(p.get("source_payload") or "")})
5290
+ for comment in p.get("embedded_comments") or []:
5291
+ # The embedded shape often names only the commenter. Its parent Post is the
5292
+ # authoritative owner, so bind every preview to this profile and post explicitly.
5293
+ comment_rows.append({**comment, "shortcode": p["shortcode"],
5294
+ "influencer_key": prof.get("username") or ""})
5295
+ comment_rows.extend((res or {}).get("comments") or [])
5296
+ # The same comment can be returned as both `latest_comments` and `top_comments`, or by the
5297
+ # optional full endpoint after an embedded preview. The canonical key is the one row truth.
5298
+ unique_comments = {}
5299
+ for row in comment_rows:
5300
+ key = str((row or {}).get("comment_key") or "")
5301
+ if key:
5302
+ unique_comments[key] = {**(unique_comments.get(key) or {}), **row}
5303
+ return snap_row, idents, metrics_rows, list(unique_comments.values())
5304
 
5305
 
5306
  #: C4 β€” how ONE pulled profile becomes the C1 preset CELLS on the record being enriched.
 
5362
  # straight-through copy here would write the same wrong number the discovery path
5363
  # used to write. Both writers convert; neither is the exception.
5364
  cells[dest] = (_pct100(v) if dest == "avg_engagement" else
5365
+ (str(v) if dest == "source_payload" else _s(v, 500)))
5366
  # ⭐ R5 β€” stamped by the writer, not carried from the pull: this function only ever sees an
5367
  # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
5368
  # arrived with no platform at all β€” and a blank half of the dedup key is how one account
 
5383
  cfg = defn.get("config") or {}
5384
  table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
5385
  tier = clean_tier(cfg.get("tier")) or "anonymous"
5386
+ paid = tier == "brightdata"
5387
+ fallback = not cfg.get("noFallback")
5388
+ post_metrics = bool(cfg.get("postMetrics"))
5389
+ comment_metrics = bool(cfg.get("commentMetrics"))
5390
  dry = bool(cfg.get("dryRun"))
5391
  steps = {"trigger": "ok", "source": "idle", "capture": "idle",
5392
  "capture_paid": "idle" if paid else "skipped",
 
5406
  # the relational tables the pull lands in (R7): every row timestamped for time-range filters
5407
  if dry:
5408
  # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
5409
+ snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE,
5410
+ IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE)
5411
  else:
5412
  graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
5413
  profile_table=table_key)
5414
+ snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
5415
+ graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
5416
+ missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key)
5417
  snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
5418
+ posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
5419
+ psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {})
5420
+ comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {})
5421
 
5422
  counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0,
5423
+ "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0,
5424
  "missing_tables": len(missing)}
5425
  cells, affected, notes = {}, [], []
5426
  # β›” ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows`
5427
  # once per POST, which is O(existing) per call β€” survivable only while the cap was 5000 rows.
5428
  # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
5429
  # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
5430
+ in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
5431
  paid_tried = paid_answered = anon_tried = anon_answered = False
5432
  targets = [(rid, str(r.get(url_field, "") or "").strip())
5433
  for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
 
5439
  # between "working through them" and "stuck".
5440
  step(f"Capturing profile {i + 1} of {len(targets)}")
5441
  counts["profiles"] += 1
5442
+ res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
5443
+ tier=tier, fallback=fallback, post_metrics=post_metrics,
5444
+ comment_metrics=comment_metrics)
5445
  pulled = _iso()
5446
  via = res.get("via") or ""
5447
  read_ok = res["state"] in ("ok", "partial")
 
5460
  if read_ok:
5461
  counts["ok" if res["state"] == "ok" else "partial"] += 1
5462
  prof = res["profile"]
5463
+ snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled)
5464
  in_snaps.append(snap_row)
5465
+ in_posts.extend(ident_rows)
5466
+ in_psnaps.extend(metric_rows)
5467
+ in_comments.extend(captured_comments)
5468
+ counts["metrics"] += len(metric_rows)
5469
+ counts["comment_rows"] += len(captured_comments)
5470
  counts["posts"] += len(res.get("posts") or [])
5471
  detail = f"{prof.get('followers') or '?'} followers"
5472
  if prof.get("approx"):
 
5491
  posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at")
5492
  posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
5493
  c_post["duplicates"] += collapsed_posts
5494
+ psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
5495
+ comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key")
5496
+ comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key))
5497
+ c_comments["duplicates"] += collapsed_comments
5498
+ counts["new_posts"] = c_post["inserted"]
5499
+ capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]),
5500
+ (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])]
5501
  counts["capped"] = sum(n for _k, n in capped)
5502
 
5503
  # R5 β€” route each processed record down its winning lane, judged over the row's FINAL
 
5527
  tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
5528
  for rid, vals in stage_writes.items():
5529
  tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals)
5530
+ for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
5531
+ (comment_key, comments)):
5532
  tgt = cur.get(k)
5533
  if tgt is not None:
5534
  tgt["rows"] = rws
 
5595
  summary += f", {counts['routed']}/{len(targets)} routed to lanes"
5596
  if counts["paid"]:
5597
  summary += f", {counts['paid']} with exact counts"
5598
+ if counts["metrics"]:
5599
+ summary += f", {counts['metrics']} post engagement snapshots"
5600
  elif post_metrics and counts["posts"]:
5601
  # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there
5602
  # was no engagement", which is a claim about Instagram rather than about our run.
5603
+ summary += ", no post engagement was readable"
5604
+ if counts["comment_rows"]:
5605
+ summary += f", {counts['comment_rows']} comment rows captured"
5606
+ elif comment_metrics and counts["posts"]:
5607
+ summary += ", no comment engagement was readable"
5608
  if counts["partial"]:
5609
  summary += f", {counts['partial']} profile-only (posts not readable)"
5610
  if counts["blocked"]:
 
6918
  # not be the thing that starts spending money (the `clean_config` rule, verbatim).
6919
  "tier": clean_tier(cfg.get("tier")) or "anonymous",
6920
  "noFallback": bool(cfg.get("noFallback")),
6921
+ # β›” THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
6922
+ "postMetrics": bool(cfg.get("postMetrics")),
6923
+ # The separate Comments dataset is more granular than a post read and stays off
6924
+ # until the user intentionally enables it.
6925
+ "commentMetrics": bool(cfg.get("commentMetrics")),
6926
+ "dryRun": bool(cfg.get("dryRun")),
6927
  "maxPosts": max_posts}, None
6928
  if kind == "find_records":
6929
  table = _s(cfg.get("table"), 60).strip()
 
7565
  # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
7566
  # entry, so calling it per record is O(existing) per record β€” survivable at 5,000 rows and an
7567
  # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
7568
+ enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "profiles": 0, "ok": 0, "blocked": 0,
7569
  "notes": [], "dry": False}
7570
  # ⭐ 2026-08-07 β€” THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
7571
  # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
 
7731
  res = pull_profile(handle_raw,
7732
  max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
7733
  log=log,
7734
+ tier=clean_tier(cfg.get("tier")) or "anonymous",
7735
+ fallback=not cfg.get("noFallback"),
7736
+ post_metrics=bool(cfg.get("postMetrics")),
7737
+ comment_metrics=bool(cfg.get("commentMetrics")))
7738
  pulled = _iso()
7739
  if res["state"] in ("ok", "partial"):
7740
  enrich["ok"] += 1
7741
+ snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
7742
  enrich["snaps"].append(snap_row)
7743
+ enrich["posts"].extend(idents)
7744
+ enrich["psnaps"].extend(metric_rows)
7745
+ enrich["comments"].extend(comment_rows)
7746
  # R3: LATEST onto the record itself. `row.update` too, so a LATER action in
7747
  # this same flow sees the enriched values β€” the rule `update_record` already
7748
  # follows, and without it "enrich, then route by follower count" would judge
 
7872
  max_posts = 24
7873
  act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
7874
  "config": {"profileField": bound,
7875
+ "tier": clean_tier(cfg.get("tier")) or "anonymous",
7876
+ "noFallback": bool(cfg.get("noFallback")),
7877
+ "postMetrics": bool(cfg.get("postMetrics")),
7878
+ "commentMetrics": bool(cfg.get("commentMetrics")),
7879
+ "dryRun": bool(cfg.get("dryRun")),
7880
  "maxPosts": max(1, min(max_posts, 200))}}
7881
  flow = dict(defn.get("flow") or {})
7882
  out = dict(defn)
 
7893
  return out, True
7894
 
7895
 
7896
+ def _enrich_flush(rt, defn, username, acc, log):
7897
+ """C4: the enrich action's canonical IG tables, written ONCE for the whole run.
7898
 
7899
  Returns the counts the run reports. Empty when no enrich action ran, so a flow without one
7900
  pays nothing and its run history is unchanged (`run_now` merges only non-zero keys).
 
7907
  if not acc.get("profiles"):
7908
  return {}
7909
  out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
7910
+ if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]):
7911
  # A dry run resolves nothing and writes nothing β€” not even `ut_ensure`, which CREATES.
7912
  if acc.get("dry"):
7913
  out["enrichDryRun"] = acc["profiles"]
 
7915
  tag = str(defn.get("id") or "")
7916
  profile_table = str(_flow_table(defn) or "")
7917
  graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table)
7918
+ snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
7919
+ graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
7920
+ missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key)
7921
  snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
7922
  acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
7923
  old_posts, collapsed_posts = dedupe_canonical_rows(
7924
  dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
7925
  posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key))
7926
  c_post["duplicates"] += collapsed_posts
7927
+ psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
7928
+ acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
7929
+ old_comments, collapsed_comments = dedupe_canonical_rows(
7930
+ dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key")
7931
+ comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key",
7932
+ cap=row_cap(comment_key))
7933
+ c_comments["duplicates"] += collapsed_comments
7934
 
7935
  def _up(cur):
7936
  cur = cur if isinstance(cur, dict) else {}
7937
+ for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
7938
+ (comment_key, comments)):
7939
  if cur.get(k) is not None:
7940
  cur[k]["rows"] = rws
7941
  _refresh_relations_inplace(cur, log=log)
 
7944
  rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
7945
  # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
7946
  # is the failure a chart cannot show you.
7947
+ capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"]
7948
  if capped:
7949
  out["enrichCapped"] = capped
7950
  log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap")
7951
  if missing:
7952
  out["enrichMissingTables"] = len(missing)
7953
  log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}")
7954
+ out["enrichPoints"] = len(acc["snaps"])
7955
+ out["enrichPosts"] = c_post["inserted"]
7956
+ out["enrichComments"] = c_comments["inserted"]
7957
  # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated β€” the same
7958
  # contract `run_field_instagram` follows, because a pooled history with silent holes is worse
7959
  # than none.
platform/core/user_tables.py CHANGED
@@ -82,10 +82,11 @@ def records_mutable(table_key, st=None):
82
  UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date',
83
  'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json',
84
  'link', 'rollup'}
85
- #: C7's ceiling. A cell is still a scalar on the wire (`Row` values are strings/numbers) β€” what
86
- #: this bounds is the DOCUMENT inside it. 32 KB holds a long comment thread and refuses a payload
87
- #: somebody meant to store as a file.
88
- MAX_JSON_CELL = 32 * 1024
 
89
 
90
  # ---------------------------------------------------------------------------------------------
91
  # THE PROFILE FLAG (wave 25, contract C3 / owner ruling R7)
 
82
  UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date',
83
  'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json',
84
  'link', 'rollup'}
85
+ #: A cell is still a scalar on the wire (`Row` values are strings/numbers) β€” what this bounds is
86
+ #: the DOCUMENT inside it. Source data is one already-paid provider response; keep it whole so a
87
+ #: provider field is never silently discarded merely because the response is rich. 32 MiB matches
88
+ #: the automation transport's guarded maximum response size.
89
+ MAX_JSON_CELL = 32 * 1024 * 1024
90
 
91
  # ---------------------------------------------------------------------------------------------
92
  # THE PROFILE FLAG (wave 25, contract C3 / owner ruling R7)
web/src/automation/AutomationBuilder.tsx CHANGED
@@ -575,7 +575,8 @@ export default function AutomationBuilder({
575
  */
576
  if (kind === "enrich_instagram")
577
  return {
578
- tier: "anonymous", noFallback: false, postMetrics: false, dryRun: false, maxPosts: 10,
 
579
  fromView: "", sortField: "first_found", sortDir: "desc",
580
  // ⚠ `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
581
  // control must not have two defaults depending on whether the step was added by hand or
@@ -2487,6 +2488,21 @@ function ActionProps({
2487
  Billed per post, not per profile β€” a separate scrape for each post kept above.
2488
  </p>
2489
  ) : null}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2490
  <label className="auto-check">
2491
  <input
2492
  type="checkbox"
 
575
  */
576
  if (kind === "enrich_instagram")
577
  return {
578
+ tier: "anonymous", noFallback: false, postMetrics: false, commentMetrics: false,
579
+ dryRun: false, maxPosts: 10,
580
  fromView: "", sortField: "first_found", sortDir: "desc",
581
  // ⚠ `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One
582
  // control must not have two defaults depending on whether the step was added by hand or
 
2488
  Billed per post, not per profile β€” a separate scrape for each post kept above.
2489
  </p>
2490
  ) : null}
2491
+ <label className="auto-check">
2492
+ <input
2493
+ type="checkbox"
2494
+ checked={!!(cfg as { commentMetrics?: boolean }).commentMetrics}
2495
+ disabled={busy}
2496
+ onChange={(e) => setCfg({ commentMetrics: e.target.checked })}
2497
+ />
2498
+ Also capture per-comment engagement
2499
+ </label>
2500
+ {(cfg as { commentMetrics?: boolean }).commentMetrics ? (
2501
+ <p className="auto-hint">
2502
+ Billed by the separate Comments dataset. This is off by default; embedded comments
2503
+ already returned with a paid post are retained without this additional request.
2504
+ </p>
2505
+ ) : null}
2506
  <label className="auto-check">
2507
  <input
2508
  type="checkbox"