Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +66 -14
- platform/core/user_tables.py +1 -1
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "04c2e87",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
04c2e87
|
api/automation_engine.py
CHANGED
|
@@ -1038,7 +1038,7 @@ IG_FIELD_DESCRIPTIONS = {
|
|
| 1038 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1039 |
"replies": "Reply count reported for the comment.",
|
| 1040 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
| 1041 |
-
"source_payload": "Full Bright Data
|
| 1042 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1043 |
"source": "Method or provider used to read this profile.",
|
| 1044 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
|
@@ -2118,9 +2118,10 @@ SNAPSHOT_FIELDS = [
|
|
| 2118 |
field_def("partner_id", "Partner id"),
|
| 2119 |
field_def("external_url_title", "Link title"),
|
| 2120 |
field_def("fbid", "Facebook id"),
|
| 2121 |
-
field_def("related_accounts", "Related accounts"),
|
| 2122 |
-
field_def("country_code", "Country"),
|
| 2123 |
-
|
|
|
|
| 2124 |
POST_FIELDS = [
|
| 2125 |
field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
|
| 2126 |
field_def("posted_at", "Posted at", "date"),
|
|
@@ -2205,12 +2206,13 @@ POST_FIELDS = [
|
|
| 2205 |
#: β `BD_NEVER_MAP` STILL STRIPS `latest_comments` β the array of 15 comment texts a POSTS row
|
| 2206 |
#: carries whether we asked or not. Turning this on does not open that door; it is a separate
|
| 2207 |
#: dataset with a separate decision, and R11's enforcement point is unchanged.
|
| 2208 |
-
COMMENT_FIELDS = [
|
| 2209 |
field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
|
| 2210 |
field_def("influencer_key", "Influencer"),
|
| 2211 |
field_def("commented_at", "Commented at", "date"),
|
| 2212 |
field_def("likes", "Likes", "int"),
|
| 2213 |
field_def("replies", "Replies", "int"),
|
|
|
|
| 2214 |
field_def("post_link", "Post", "link",
|
| 2215 |
description="Post record linked to this comment.",
|
| 2216 |
link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
|
|
@@ -2566,7 +2568,8 @@ def _first(node, *names, default=None):
|
|
| 2566 |
#: enforced HERE, at the map, because that is the only place it can be.
|
| 2567 |
BD_NEVER_MAP = frozenset({
|
| 2568 |
"email_address", "phone_number", "business_email", "profile_image_link",
|
| 2569 |
-
"
|
|
|
|
| 2570 |
})
|
| 2571 |
|
| 2572 |
# `json` cells are capped by the table layer. Keep this local literal with the same boot-path
|
|
@@ -2699,6 +2702,51 @@ def _bd_public_post_payload(row):
|
|
| 2699 |
except (TypeError, ValueError):
|
| 2700 |
return None
|
| 2701 |
return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2702 |
|
| 2703 |
|
| 2704 |
def _bd_profile(node, handle):
|
|
@@ -2763,11 +2811,12 @@ def _bd_profile(node, handle):
|
|
| 2763 |
# β MEASURED `None` (not `[]`) on the real row β the graph-walk discovery seed is dead
|
| 2764 |
# (Β§2c). Mapped anyway: history is unbuyable, so a field not captured today is lost for
|
| 2765 |
# today, and n=1 is not evidence enough to drop a column permanently.
|
| 2766 |
-
"related_accounts": _bd_list(node, "related_accounts"),
|
| 2767 |
# β `country_code` is the ONE field the corpus filter API REJECTS as a predicate, and it
|
| 2768 |
# was `None` on all five corpus rows. It is a row field regardless, and reading it costs
|
| 2769 |
# nothing; a blank here means not read, like every other blank in this map.
|
| 2770 |
-
"country_code": str(_first(node, "country_code", default="") or ""),
|
|
|
|
| 2771 |
}
|
| 2772 |
# β THE PII RAIL, ENFORCED HERE RATHER THAN TRUSTED. Nothing above reads a `BD_NEVER_MAP`
|
| 2773 |
# field, and this asserts it stays that way after the next edit β a map is exactly the kind
|
|
@@ -3569,8 +3618,9 @@ PRESET_PROFILE_FIELDS = [
|
|
| 3569 |
field_def("external_url_title", "Link title"),
|
| 3570 |
# β `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity.
|
| 3571 |
field_def("fbid", "Facebook id"),
|
| 3572 |
-
field_def("related_accounts", "Related accounts"),
|
| 3573 |
-
field_def("country_code", "Country"),
|
|
|
|
| 3574 |
# ββ 2026-08-07 (owner instruction) β THE RELATION AND THE ROLLUPS OVER IT.
|
| 3575 |
#
|
| 3576 |
# Owner: *"an enrichment automation should spawn relevant Post/Comment database that is
|
|
@@ -5135,7 +5185,8 @@ def capture_rows(res, pulled):
|
|
| 5135 |
# Profile history declares the same percentage dialect as the current Profile row.
|
| 5136 |
# Converting only the latest projection left the append series at 0-1 while its field
|
| 5137 |
# now said pct (0-100), so the two views of one measurement disagreed by 100x.
|
| 5138 |
-
|
|
|
|
| 5139 |
idents, metrics_rows = [], []
|
| 5140 |
for p in (res or {}).get("posts") or []:
|
| 5141 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
|
@@ -5204,8 +5255,8 @@ PRESET_FROM_PROFILE = {
|
|
| 5204 |
"pronouns": "pronouns", "profile_name": "profile_name",
|
| 5205 |
"is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
|
| 5206 |
"partner_id": "partner_id", "external_url_title": "external_url_title",
|
| 5207 |
-
"fbid": "fbid", "related_accounts": "related_accounts",
|
| 5208 |
-
"country_code": "country_code",
|
| 5209 |
}
|
| 5210 |
|
| 5211 |
#: β 2026-08-07 β the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a
|
|
@@ -5239,7 +5290,8 @@ def preset_cells(res, pulled):
|
|
| 5239 |
# vendor's engagement is a 0β1 fraction and our `pct` column is 0β100, so a
|
| 5240 |
# straight-through copy here would write the same wrong number the discovery path
|
| 5241 |
# used to write. Both writers convert; neither is the exception.
|
| 5242 |
-
cells[dest] = _pct100(v) if dest == "avg_engagement" else
|
|
|
|
| 5243 |
# β R5 β stamped by the writer, not carried from the pull: this function only ever sees an
|
| 5244 |
# Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
|
| 5245 |
# arrived with no platform at all β and a blank half of the dedup key is how one account
|
|
|
|
| 1038 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1039 |
"replies": "Reply count reported for the comment.",
|
| 1040 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
| 1041 |
+
"source_payload": "Full Bright Data source response, excluding protected contact and comment data.",
|
| 1042 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1043 |
"source": "Method or provider used to read this profile.",
|
| 1044 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
|
|
|
| 2118 |
field_def("partner_id", "Partner id"),
|
| 2119 |
field_def("external_url_title", "Link title"),
|
| 2120 |
field_def("fbid", "Facebook id"),
|
| 2121 |
+
field_def("related_accounts", "Related accounts"),
|
| 2122 |
+
field_def("country_code", "Country"),
|
| 2123 |
+
field_def("source_payload", "Source data", "json"),
|
| 2124 |
+
]
|
| 2125 |
POST_FIELDS = [
|
| 2126 |
field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
|
| 2127 |
field_def("posted_at", "Posted at", "date"),
|
|
|
|
| 2206 |
#: β `BD_NEVER_MAP` STILL STRIPS `latest_comments` β the array of 15 comment texts a POSTS row
|
| 2207 |
#: carries whether we asked or not. Turning this on does not open that door; it is a separate
|
| 2208 |
#: dataset with a separate decision, and R11's enforcement point is unchanged.
|
| 2209 |
+
COMMENT_FIELDS = [
|
| 2210 |
field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
|
| 2211 |
field_def("influencer_key", "Influencer"),
|
| 2212 |
field_def("commented_at", "Commented at", "date"),
|
| 2213 |
field_def("likes", "Likes", "int"),
|
| 2214 |
field_def("replies", "Replies", "int"),
|
| 2215 |
+
field_def("source_payload", "Source data", "json"),
|
| 2216 |
field_def("post_link", "Post", "link",
|
| 2217 |
description="Post record linked to this comment.",
|
| 2218 |
link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
|
|
|
|
| 2568 |
#: enforced HERE, at the map, because that is the only place it can be.
|
| 2569 |
BD_NEVER_MAP = frozenset({
|
| 2570 |
"email_address", "phone_number", "business_email", "profile_image_link",
|
| 2571 |
+
"profile_pic_url", "latest_comments", "top_comments", "comment_user",
|
| 2572 |
+
"comment_user_url", "post_user", "comment", "user_commenting", "avatar",
|
| 2573 |
})
|
| 2574 |
|
| 2575 |
# `json` cells are capped by the table layer. Keep this local literal with the same boot-path
|
|
|
|
| 2702 |
except (TypeError, ValueError):
|
| 2703 |
return None
|
| 2704 |
return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
|
| 2705 |
+
|
| 2706 |
+
|
| 2707 |
+
def _bd_public_profile_payload(node):
|
| 2708 |
+
"""The Profile counterpart to the Post source document.
|
| 2709 |
+
|
| 2710 |
+
A profile scrape embeds its latest `posts` list. That list already becomes canonical linked
|
| 2711 |
+
Post rows, so retaining it here would recreate the retired Recent Posts JSON cache. All other
|
| 2712 |
+
public source fields are preserved, subject to the same contact/comment boundary as Posts.
|
| 2713 |
+
"""
|
| 2714 |
+
if not isinstance(node, dict):
|
| 2715 |
+
return None
|
| 2716 |
+
public = {str(k): v for k, v in node.items()
|
| 2717 |
+
if str(k) not in BD_NEVER_MAP and str(k) != "posts"}
|
| 2718 |
+
try:
|
| 2719 |
+
packed = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
| 2720 |
+
except (TypeError, ValueError):
|
| 2721 |
+
return None
|
| 2722 |
+
return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
|
| 2723 |
+
|
| 2724 |
+
|
| 2725 |
+
def _bd_comment(row, influencer_key="", shortcode=""):
|
| 2726 |
+
"""One Bright Data Comments response -> the complete permitted Comment record.
|
| 2727 |
+
|
| 2728 |
+
The actual Comments endpoint remains an explicit paid opt-in. This pure mapper means that
|
| 2729 |
+
when it is enabled, every allowed response field is preserved in `source_payload` rather than
|
| 2730 |
+
reintroducing a partial child-table schema. Third-party author identity and comment text stay
|
| 2731 |
+
outside both the first-class fields and the source document.
|
| 2732 |
+
"""
|
| 2733 |
+
if not isinstance(row, dict):
|
| 2734 |
+
return None
|
| 2735 |
+
url = str(_first(row, "url", "post_url", default="") or "")
|
| 2736 |
+
code = str(shortcode or "") or ig_shortcode(url)
|
| 2737 |
+
comment_id = str(_first(row, "comment_id", "id", "pk", default="") or "")
|
| 2738 |
+
if not code or not comment_id:
|
| 2739 |
+
return None
|
| 2740 |
+
posted = str(_first(row, "comment_date", "date_of_comment", default="") or "")
|
| 2741 |
+
return {
|
| 2742 |
+
"comment_key": comment_id,
|
| 2743 |
+
"shortcode": code,
|
| 2744 |
+
"influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
|
| 2745 |
+
"commented_at": posted.replace("T", " ")[:16],
|
| 2746 |
+
"likes": _ig_int(_first(row, "likes_number", "likes", "like_count")),
|
| 2747 |
+
"replies": _ig_int(_first(row, "replies_number", "num_replies", "replies")),
|
| 2748 |
+
"source_payload": _bd_public_post_payload(row),
|
| 2749 |
+
}
|
| 2750 |
|
| 2751 |
|
| 2752 |
def _bd_profile(node, handle):
|
|
|
|
| 2811 |
# β MEASURED `None` (not `[]`) on the real row β the graph-walk discovery seed is dead
|
| 2812 |
# (Β§2c). Mapped anyway: history is unbuyable, so a field not captured today is lost for
|
| 2813 |
# today, and n=1 is not evidence enough to drop a column permanently.
|
| 2814 |
+
"related_accounts": _bd_list(node, "related_accounts"),
|
| 2815 |
# β `country_code` is the ONE field the corpus filter API REJECTS as a predicate, and it
|
| 2816 |
# was `None` on all five corpus rows. It is a row field regardless, and reading it costs
|
| 2817 |
# nothing; a blank here means not read, like every other blank in this map.
|
| 2818 |
+
"country_code": str(_first(node, "country_code", default="") or ""),
|
| 2819 |
+
"source_payload": _bd_public_profile_payload(node),
|
| 2820 |
}
|
| 2821 |
# β THE PII RAIL, ENFORCED HERE RATHER THAN TRUSTED. Nothing above reads a `BD_NEVER_MAP`
|
| 2822 |
# field, and this asserts it stays that way after the next edit β a map is exactly the kind
|
|
|
|
| 3618 |
field_def("external_url_title", "Link title"),
|
| 3619 |
# β `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity.
|
| 3620 |
field_def("fbid", "Facebook id"),
|
| 3621 |
+
field_def("related_accounts", "Related accounts"),
|
| 3622 |
+
field_def("country_code", "Country"),
|
| 3623 |
+
field_def("source_payload", "Source data", "json"),
|
| 3624 |
# ββ 2026-08-07 (owner instruction) β THE RELATION AND THE ROLLUPS OVER IT.
|
| 3625 |
#
|
| 3626 |
# Owner: *"an enrichment automation should spawn relevant Post/Comment database that is
|
|
|
|
| 5185 |
# Profile history declares the same percentage dialect as the current Profile row.
|
| 5186 |
# Converting only the latest projection left the append series at 0-1 while its field
|
| 5187 |
# now said pct (0-100), so the two views of one measurement disagreed by 100x.
|
| 5188 |
+
n = BD_PUBLIC_POST_PAYLOAD_MAX if k == "source_payload" else 400
|
| 5189 |
+
snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, n)
|
| 5190 |
idents, metrics_rows = [], []
|
| 5191 |
for p in (res or {}).get("posts") or []:
|
| 5192 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
|
|
|
| 5255 |
"pronouns": "pronouns", "profile_name": "profile_name",
|
| 5256 |
"is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
|
| 5257 |
"partner_id": "partner_id", "external_url_title": "external_url_title",
|
| 5258 |
+
"fbid": "fbid", "related_accounts": "related_accounts",
|
| 5259 |
+
"country_code": "country_code", "source_payload": "source_payload",
|
| 5260 |
}
|
| 5261 |
|
| 5262 |
#: β 2026-08-07 β the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a
|
|
|
|
| 5290 |
# vendor's engagement is a 0β1 fraction and our `pct` column is 0β100, so a
|
| 5291 |
# straight-through copy here would write the same wrong number the discovery path
|
| 5292 |
# used to write. Both writers convert; neither is the exception.
|
| 5293 |
+
cells[dest] = (_pct100(v) if dest == "avg_engagement" else
|
| 5294 |
+
_s(v, BD_PUBLIC_POST_PAYLOAD_MAX if dest == "source_payload" else 500))
|
| 5295 |
# β R5 β stamped by the writer, not carried from the pull: this function only ever sees an
|
| 5296 |
# Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
|
| 5297 |
# arrived with no platform at all β and a blank half of the dedup key is how one account
|
platform/core/user_tables.py
CHANGED
|
@@ -139,7 +139,7 @@ PROFILE_PRESET_KEYS = (
|
|
| 139 |
# owner instruction. They are profile FACTS like the nineteen above, so blanking the handle
|
| 140 |
# clears them for the same reason (R6): they describe an account this row no longer names.
|
| 141 |
'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title',
|
| 142 |
-
'fbid', 'related_accounts', 'country_code',
|
| 143 |
# relational projections and summaries owned by the Instagram graph
|
| 144 |
'posts_link', 'profile_snapshots_link', 'post_snapshots_link', 'comments_link',
|
| 145 |
'avg_views_12', 'avg_plays_12', 'avg_likes_12', 'avg_comments_12',
|
|
|
|
| 139 |
# owner instruction. They are profile FACTS like the nineteen above, so blanking the handle
|
| 140 |
# clears them for the same reason (R6): they describe an account this row no longer names.
|
| 141 |
'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title',
|
| 142 |
+
'fbid', 'related_accounts', 'country_code', 'source_payload',
|
| 143 |
# relational projections and summaries owned by the Instagram graph
|
| 144 |
'posts_link', 'profile_snapshots_link', 'post_snapshots_link', 'comments_link',
|
| 145 |
'avg_views_12', 'avg_plays_12', 'avg_likes_12', 'avg_comments_12',
|