fsanyoto commited on
Commit
ecb511f
Β·
verified Β·
1 Parent(s): 95867b2

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "2e88845",
3
  "releases": [
4
  {
5
  "version": "v14",
 
1
  {
2
+ "current": "04c2e87",
3
  "releases": [
4
  {
5
  "version": "v14",
VERSION CHANGED
@@ -1 +1 @@
1
- 2e88845
 
1
+ 04c2e87
api/automation_engine.py CHANGED
@@ -1038,7 +1038,7 @@ IG_FIELD_DESCRIPTIONS = {
1038
  "related_accounts": "Accounts Instagram suggests alongside this profile.",
1039
  "replies": "Reply count reported for the comment.",
1040
  "shortcode": "Instagram shortcode that uniquely identifies the post.",
1041
- "source_payload": "Full Bright Data post response, excluding protected contact and comment data.",
1042
  "snapshot_key": "Unique ID for this profile snapshot.",
1043
  "source": "Method or provider used to read this profile.",
1044
  "tagged_location": "Location tagged on the post; it is not the creator's residence.",
@@ -2118,9 +2118,10 @@ SNAPSHOT_FIELDS = [
2118
  field_def("partner_id", "Partner id"),
2119
  field_def("external_url_title", "Link title"),
2120
  field_def("fbid", "Facebook id"),
2121
- field_def("related_accounts", "Related accounts"),
2122
- field_def("country_code", "Country"),
2123
- ]
 
2124
  POST_FIELDS = [
2125
  field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
2126
  field_def("posted_at", "Posted at", "date"),
@@ -2205,12 +2206,13 @@ POST_FIELDS = [
2205
  #: ⚠ `BD_NEVER_MAP` STILL STRIPS `latest_comments` β€” the array of 15 comment texts a POSTS row
2206
  #: carries whether we asked or not. Turning this on does not open that door; it is a separate
2207
  #: dataset with a separate decision, and R11's enforcement point is unchanged.
2208
- COMMENT_FIELDS = [
2209
  field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
2210
  field_def("influencer_key", "Influencer"),
2211
  field_def("commented_at", "Commented at", "date"),
2212
  field_def("likes", "Likes", "int"),
2213
  field_def("replies", "Replies", "int"),
 
2214
  field_def("post_link", "Post", "link",
2215
  description="Post record linked to this comment.",
2216
  link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
@@ -2566,7 +2568,8 @@ def _first(node, *names, default=None):
2566
  #: enforced HERE, at the map, because that is the only place it can be.
2567
  BD_NEVER_MAP = frozenset({
2568
  "email_address", "phone_number", "business_email", "profile_image_link",
2569
- "latest_comments", "top_comments", "comment_user", "post_user",
 
2570
  })
2571
 
2572
  # `json` cells are capped by the table layer. Keep this local literal with the same boot-path
@@ -2699,6 +2702,51 @@ def _bd_public_post_payload(row):
2699
  except (TypeError, ValueError):
2700
  return None
2701
  return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2702
 
2703
 
2704
  def _bd_profile(node, handle):
@@ -2763,11 +2811,12 @@ def _bd_profile(node, handle):
2763
  # ⚠ MEASURED `None` (not `[]`) on the real row β€” the graph-walk discovery seed is dead
2764
  # (Β§2c). Mapped anyway: history is unbuyable, so a field not captured today is lost for
2765
  # today, and n=1 is not evidence enough to drop a column permanently.
2766
- "related_accounts": _bd_list(node, "related_accounts"),
2767
  # ⚠ `country_code` is the ONE field the corpus filter API REJECTS as a predicate, and it
2768
  # was `None` on all five corpus rows. It is a row field regardless, and reading it costs
2769
  # nothing; a blank here means not read, like every other blank in this map.
2770
- "country_code": str(_first(node, "country_code", default="") or ""),
 
2771
  }
2772
  # β›” THE PII RAIL, ENFORCED HERE RATHER THAN TRUSTED. Nothing above reads a `BD_NEVER_MAP`
2773
  # field, and this asserts it stays that way after the next edit β€” a map is exactly the kind
@@ -3569,8 +3618,9 @@ PRESET_PROFILE_FIELDS = [
3569
  field_def("external_url_title", "Link title"),
3570
  # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity.
3571
  field_def("fbid", "Facebook id"),
3572
- field_def("related_accounts", "Related accounts"),
3573
- field_def("country_code", "Country"),
 
3574
  # ⭐⭐ 2026-08-07 (owner instruction) β€” THE RELATION AND THE ROLLUPS OVER IT.
3575
  #
3576
  # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is
@@ -5135,7 +5185,8 @@ def capture_rows(res, pulled):
5135
  # Profile history declares the same percentage dialect as the current Profile row.
5136
  # Converting only the latest projection left the append series at 0-1 while its field
5137
  # now said pct (0-100), so the two views of one measurement disagreed by 100x.
5138
- snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400)
 
5139
  idents, metrics_rows = [], []
5140
  for p in (res or {}).get("posts") or []:
5141
  ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
@@ -5204,8 +5255,8 @@ PRESET_FROM_PROFILE = {
5204
  "pronouns": "pronouns", "profile_name": "profile_name",
5205
  "is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
5206
  "partner_id": "partner_id", "external_url_title": "external_url_title",
5207
- "fbid": "fbid", "related_accounts": "related_accounts",
5208
- "country_code": "country_code",
5209
  }
5210
 
5211
  #: ⭐ 2026-08-07 β€” the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a
@@ -5239,7 +5290,8 @@ def preset_cells(res, pulled):
5239
  # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a
5240
  # straight-through copy here would write the same wrong number the discovery path
5241
  # used to write. Both writers convert; neither is the exception.
5242
- cells[dest] = _pct100(v) if dest == "avg_engagement" else _s(v, 500)
 
5243
  # ⭐ R5 β€” stamped by the writer, not carried from the pull: this function only ever sees an
5244
  # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
5245
  # arrived with no platform at all β€” and a blank half of the dedup key is how one account
 
1038
  "related_accounts": "Accounts Instagram suggests alongside this profile.",
1039
  "replies": "Reply count reported for the comment.",
1040
  "shortcode": "Instagram shortcode that uniquely identifies the post.",
1041
+ "source_payload": "Full Bright Data source response, excluding protected contact and comment data.",
1042
  "snapshot_key": "Unique ID for this profile snapshot.",
1043
  "source": "Method or provider used to read this profile.",
1044
  "tagged_location": "Location tagged on the post; it is not the creator's residence.",
 
2118
  field_def("partner_id", "Partner id"),
2119
  field_def("external_url_title", "Link title"),
2120
  field_def("fbid", "Facebook id"),
2121
+ field_def("related_accounts", "Related accounts"),
2122
+ field_def("country_code", "Country"),
2123
+ field_def("source_payload", "Source data", "json"),
2124
+ ]
2125
  POST_FIELDS = [
2126
  field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
2127
  field_def("posted_at", "Posted at", "date"),
 
2206
  #: ⚠ `BD_NEVER_MAP` STILL STRIPS `latest_comments` β€” the array of 15 comment texts a POSTS row
2207
  #: carries whether we asked or not. Turning this on does not open that door; it is a separate
2208
  #: dataset with a separate decision, and R11's enforcement point is unchanged.
2209
+ COMMENT_FIELDS = [
2210
  field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
2211
  field_def("influencer_key", "Influencer"),
2212
  field_def("commented_at", "Commented at", "date"),
2213
  field_def("likes", "Likes", "int"),
2214
  field_def("replies", "Replies", "int"),
2215
+ field_def("source_payload", "Source data", "json"),
2216
  field_def("post_link", "Post", "link",
2217
  description="Post record linked to this comment.",
2218
  link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
 
2568
  #: enforced HERE, at the map, because that is the only place it can be.
2569
  BD_NEVER_MAP = frozenset({
2570
  "email_address", "phone_number", "business_email", "profile_image_link",
2571
+ "profile_pic_url", "latest_comments", "top_comments", "comment_user",
2572
+ "comment_user_url", "post_user", "comment", "user_commenting", "avatar",
2573
  })
2574
 
2575
  # `json` cells are capped by the table layer. Keep this local literal with the same boot-path
 
2702
  except (TypeError, ValueError):
2703
  return None
2704
  return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
2705
+
2706
+
2707
+ def _bd_public_profile_payload(node):
2708
+ """The Profile counterpart to the Post source document.
2709
+
2710
+ A profile scrape embeds its latest `posts` list. That list already becomes canonical linked
2711
+ Post rows, so retaining it here would recreate the retired Recent Posts JSON cache. All other
2712
+ public source fields are preserved, subject to the same contact/comment boundary as Posts.
2713
+ """
2714
+ if not isinstance(node, dict):
2715
+ return None
2716
+ public = {str(k): v for k, v in node.items()
2717
+ if str(k) not in BD_NEVER_MAP and str(k) != "posts"}
2718
+ try:
2719
+ packed = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
2720
+ except (TypeError, ValueError):
2721
+ return None
2722
+ return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
2723
+
2724
+
2725
+ def _bd_comment(row, influencer_key="", shortcode=""):
2726
+ """One Bright Data Comments response -> the complete permitted Comment record.
2727
+
2728
+ The actual Comments endpoint remains an explicit paid opt-in. This pure mapper means that
2729
+ when it is enabled, every allowed response field is preserved in `source_payload` rather than
2730
+ reintroducing a partial child-table schema. Third-party author identity and comment text stay
2731
+ outside both the first-class fields and the source document.
2732
+ """
2733
+ if not isinstance(row, dict):
2734
+ return None
2735
+ url = str(_first(row, "url", "post_url", default="") or "")
2736
+ code = str(shortcode or "") or ig_shortcode(url)
2737
+ comment_id = str(_first(row, "comment_id", "id", "pk", default="") or "")
2738
+ if not code or not comment_id:
2739
+ return None
2740
+ posted = str(_first(row, "comment_date", "date_of_comment", default="") or "")
2741
+ return {
2742
+ "comment_key": comment_id,
2743
+ "shortcode": code,
2744
+ "influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
2745
+ "commented_at": posted.replace("T", " ")[:16],
2746
+ "likes": _ig_int(_first(row, "likes_number", "likes", "like_count")),
2747
+ "replies": _ig_int(_first(row, "replies_number", "num_replies", "replies")),
2748
+ "source_payload": _bd_public_post_payload(row),
2749
+ }
2750
 
2751
 
2752
  def _bd_profile(node, handle):
 
2811
  # ⚠ MEASURED `None` (not `[]`) on the real row β€” the graph-walk discovery seed is dead
2812
  # (Β§2c). Mapped anyway: history is unbuyable, so a field not captured today is lost for
2813
  # today, and n=1 is not evidence enough to drop a column permanently.
2814
+ "related_accounts": _bd_list(node, "related_accounts"),
2815
  # ⚠ `country_code` is the ONE field the corpus filter API REJECTS as a predicate, and it
2816
  # was `None` on all five corpus rows. It is a row field regardless, and reading it costs
2817
  # nothing; a blank here means not read, like every other blank in this map.
2818
+ "country_code": str(_first(node, "country_code", default="") or ""),
2819
+ "source_payload": _bd_public_profile_payload(node),
2820
  }
2821
  # β›” THE PII RAIL, ENFORCED HERE RATHER THAN TRUSTED. Nothing above reads a `BD_NEVER_MAP`
2822
  # field, and this asserts it stays that way after the next edit β€” a map is exactly the kind
 
3618
  field_def("external_url_title", "Link title"),
3619
  # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity.
3620
  field_def("fbid", "Facebook id"),
3621
+ field_def("related_accounts", "Related accounts"),
3622
+ field_def("country_code", "Country"),
3623
+ field_def("source_payload", "Source data", "json"),
3624
  # ⭐⭐ 2026-08-07 (owner instruction) β€” THE RELATION AND THE ROLLUPS OVER IT.
3625
  #
3626
  # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is
 
5185
  # Profile history declares the same percentage dialect as the current Profile row.
5186
  # Converting only the latest projection left the append series at 0-1 while its field
5187
  # now said pct (0-100), so the two views of one measurement disagreed by 100x.
5188
+ n = BD_PUBLIC_POST_PAYLOAD_MAX if k == "source_payload" else 400
5189
+ snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, n)
5190
  idents, metrics_rows = [], []
5191
  for p in (res or {}).get("posts") or []:
5192
  ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
 
5255
  "pronouns": "pronouns", "profile_name": "profile_name",
5256
  "is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
5257
  "partner_id": "partner_id", "external_url_title": "external_url_title",
5258
+ "fbid": "fbid", "related_accounts": "related_accounts",
5259
+ "country_code": "country_code", "source_payload": "source_payload",
5260
  }
5261
 
5262
  #: ⭐ 2026-08-07 β€” the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a
 
5290
  # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a
5291
  # straight-through copy here would write the same wrong number the discovery path
5292
  # used to write. Both writers convert; neither is the exception.
5293
+ cells[dest] = (_pct100(v) if dest == "avg_engagement" else
5294
+ _s(v, BD_PUBLIC_POST_PAYLOAD_MAX if dest == "source_payload" else 500))
5295
  # ⭐ R5 β€” stamped by the writer, not carried from the pull: this function only ever sees an
5296
  # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
5297
  # arrived with no platform at all β€” and a blank half of the dedup key is how one account
platform/core/user_tables.py CHANGED
@@ -139,7 +139,7 @@ PROFILE_PRESET_KEYS = (
139
  # owner instruction. They are profile FACTS like the nineteen above, so blanking the handle
140
  # clears them for the same reason (R6): they describe an account this row no longer names.
141
  'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title',
142
- 'fbid', 'related_accounts', 'country_code',
143
  # relational projections and summaries owned by the Instagram graph
144
  'posts_link', 'profile_snapshots_link', 'post_snapshots_link', 'comments_link',
145
  'avg_views_12', 'avg_plays_12', 'avg_likes_12', 'avg_comments_12',
 
139
  # owner instruction. They are profile FACTS like the nineteen above, so blanking the handle
140
  # clears them for the same reason (R6): they describe an account this row no longer names.
141
  'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title',
142
+ 'fbid', 'related_accounts', 'country_code', 'source_payload',
143
  # relational projections and summaries owned by the Instagram graph
144
  'posts_link', 'profile_snapshots_link', 'post_snapshots_link', 'comments_link',
145
  'avg_views_12', 'avg_plays_12', 'avg_likes_12', 'avg_comments_12',