Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- api/automation_engine.py +17 -0
- api/connectors_ig.py +8 -0
- api/connectors_tt.py +5 -0
api/automation_engine.py
CHANGED
|
@@ -1231,6 +1231,7 @@ IG_FIELD_DESCRIPTIONS = {
|
|
| 1231 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1232 |
"source": "Method or provider used to read this profile.",
|
| 1233 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
|
|
|
| 1234 |
"type": "Instagram media type: image, video, or carousel.",
|
| 1235 |
"url": "Direct URL to the Instagram post.",
|
| 1236 |
"verified": "Checked when Instagram marks the profile as verified.",
|
|
@@ -1283,6 +1284,7 @@ TT_FIELD_DESCRIPTIONS = {
|
|
| 1283 |
"influencer_key": "Handle of the account this record belongs to.",
|
| 1284 |
"comment_key": "Unique ID for this comment.",
|
| 1285 |
"commented_at": "When the comment was posted.",
|
|
|
|
| 1286 |
"replies": "Reply count reported for the comment.",
|
| 1287 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1288 |
"post_snapshot_key": "Unique ID for this post measurement.",
|
|
@@ -3018,6 +3020,12 @@ POST_FIELDS = [
|
|
| 3018 |
COMMENT_FIELDS = [
|
| 3019 |
field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
|
| 3020 |
field_def("influencer_key", "Influencer"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3021 |
field_def("commented_at", "Commented at", "date"),
|
| 3022 |
field_def("likes", "Likes", "int"),
|
| 3023 |
field_def("replies", "Replies", "int"),
|
|
@@ -3286,6 +3294,15 @@ TT_COMMENT_FIELDS = [
|
|
| 3286 |
tt_field_def("comment_key", "Comment"),
|
| 3287 |
tt_field_def("shortcode", "Post id"),
|
| 3288 |
tt_field_def("influencer_key", "Influencer"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3289 |
tt_field_def("commented_at", "Commented at", "date"),
|
| 3290 |
tt_field_def("likes", "Likes", "int"),
|
| 3291 |
tt_field_def("replies", "Replies", "int"),
|
|
|
|
| 1231 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1232 |
"source": "Method or provider used to read this profile.",
|
| 1233 |
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
| 1234 |
+
"text": "What the comment says. The commenter's name is not stored as a column.",
|
| 1235 |
"type": "Instagram media type: image, video, or carousel.",
|
| 1236 |
"url": "Direct URL to the Instagram post.",
|
| 1237 |
"verified": "Checked when Instagram marks the profile as verified.",
|
|
|
|
| 1284 |
"influencer_key": "Handle of the account this record belongs to.",
|
| 1285 |
"comment_key": "Unique ID for this comment.",
|
| 1286 |
"commented_at": "When the comment was posted.",
|
| 1287 |
+
"text": "What the comment says. The commenter's name is not stored as a column.",
|
| 1288 |
"replies": "Reply count reported for the comment.",
|
| 1289 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1290 |
"post_snapshot_key": "Unique ID for this post measurement.",
|
|
|
|
| 3020 |
COMMENT_FIELDS = [
|
| 3021 |
field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
|
| 3022 |
field_def("influencer_key", "Influencer"),
|
| 3023 |
+
# ββ OWNER RULING 2026-08-12 β the same one that put `text` on the TikTok comment schema, and
|
| 3024 |
+
# it lands on BOTH networks in the same change on purpose: the two comment tables are read side
|
| 3025 |
+
# by side, and one carrying the content while the other does not is the divergence this repo
|
| 3026 |
+
# keeps paying for. β TEXT ONLY β `comment_user` / `comment_user_url` are vendor-FLAGGED PII
|
| 3027 |
+
# and stay in `source_payload`, uncolumned.
|
| 3028 |
+
field_def("text", "Comment"),
|
| 3029 |
field_def("commented_at", "Commented at", "date"),
|
| 3030 |
field_def("likes", "Likes", "int"),
|
| 3031 |
field_def("replies", "Replies", "int"),
|
|
|
|
| 3294 |
tt_field_def("comment_key", "Comment"),
|
| 3295 |
tt_field_def("shortcode", "Post id"),
|
| 3296 |
tt_field_def("influencer_key", "Influencer"),
|
| 3297 |
+
# ββ OWNER RULING 2026-08-12, verbatim: *"why isn't any of the comments column actually HAS
|
| 3298 |
+
# the comments content, fix it."* The text was bought, stored and INVISIBLE β retained whole in
|
| 3299 |
+
# `source_payload` under the 2026-08-07 superseding ruling (`instagram-capture.md` Β§4b), but
|
| 3300 |
+
# promoted to no column, so a person paying per comment record saw only Likes and Replies.
|
| 3301 |
+
# β THE COMMENT TEXT ONLY β the commenter's IDENTITY (`commenter_user_name`, `commenter_id`,
|
| 3302 |
+
# `commenter_url`; the vendor FLAGS the first as PII) stays in `source_payload` and gets no
|
| 3303 |
+
# column. The ruling names the comments' CONTENT, and promoting a third party's name and
|
| 3304 |
+
# profile URL into a filterable, exportable column is a different decision that nobody made.
|
| 3305 |
+
tt_field_def("text", "Comment"),
|
| 3306 |
tt_field_def("commented_at", "Commented at", "date"),
|
| 3307 |
tt_field_def("likes", "Likes", "int"),
|
| 3308 |
tt_field_def("replies", "Replies", "int"),
|
api/connectors_ig.py
CHANGED
|
@@ -277,6 +277,14 @@ def _bd_comment(row, influencer_key="", shortcode="", fallback_key=""):
|
|
| 277 |
"comment_key": comment_id,
|
| 278 |
"shortcode": code,
|
| 279 |
"influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
"commented_at": posted.replace("T", " ")[:16],
|
| 281 |
"likes": _ig_int(_first(row, "likes_number", "likes", "like_count", "likes_count")),
|
| 282 |
"replies": _ig_int(_first(row, "replies_number", "num_replies", "replies",
|
|
|
|
| 277 |
"comment_key": comment_id,
|
| 278 |
"shortcode": code,
|
| 279 |
"influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""),
|
| 280 |
+
# ββ OWNER RULING 2026-08-12 β the comment's CONTENT is a column on BOTH networks now.
|
| 281 |
+
# `comment` is the key Bright Data's Comments dataset sends (and the one this module's own
|
| 282 |
+
# fixture has always carried); the rest are defensive, in the house `_first` style, because
|
| 283 |
+
# the EMBEDDED comment arrays on a paid Post/Reel response are a different shape from the
|
| 284 |
+
# dedicated dataset and this mapper serves both.
|
| 285 |
+
# β TEXT ONLY β `comment_user` / `comment_user_url` are vendor-flagged PII and stay in
|
| 286 |
+
# `source_payload`, which the 2026-08-07 retention ruling keeps whole.
|
| 287 |
+
"text": str(_first(row, "comment", "comment_text", "text", default="") or ""),
|
| 288 |
"commented_at": posted.replace("T", " ")[:16],
|
| 289 |
"likes": _ig_int(_first(row, "likes_number", "likes", "like_count", "likes_count")),
|
| 290 |
"replies": _ig_int(_first(row, "replies_number", "num_replies", "replies",
|
api/connectors_tt.py
CHANGED
|
@@ -291,6 +291,11 @@ def normalize_comment(node):
|
|
| 291 |
"platform": engine.PLATFORM_TIKTOK,
|
| 292 |
"comment_key": comment_key,
|
| 293 |
"shortcode": _tt_str(node, "post_id"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
"commented_at": _tt_day(node, "date_created"),
|
| 295 |
"likes": _ig_int(_first(node, "num_likes")),
|
| 296 |
"replies": _ig_int(_first(node, "num_replies")),
|
|
|
|
| 291 |
"platform": engine.PLATFORM_TIKTOK,
|
| 292 |
"comment_key": comment_key,
|
| 293 |
"shortcode": _tt_str(node, "post_id"),
|
| 294 |
+
# ββ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
|
| 295 |
+
# vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
|
| 296 |
+
# primary first, so a row carrying the rich form is not silently served the plain one.
|
| 297 |
+
# β The commenter's identity is deliberately NOT promoted β see `TT_COMMENT_FIELDS`.
|
| 298 |
+
"text": _tt_str(node, "comment_text", "comment_text_only"),
|
| 299 |
"commented_at": _tt_day(node, "date_created"),
|
| 300 |
"likes": _ig_int(_first(node, "num_likes")),
|
| 301 |
"replies": _ig_int(_first(node, "num_replies")),
|