Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +71 -14
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "e34bc0e",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
e34bc0e
|
api/automation_engine.py
CHANGED
|
@@ -1038,8 +1038,10 @@ IG_FIELD_DESCRIPTIONS = {
|
|
| 1038 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1039 |
"replies": "Reply count reported for the comment.",
|
| 1040 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
|
|
|
| 1041 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1042 |
"source": "Method or provider used to read this profile.",
|
|
|
|
| 1043 |
"type": "Instagram media type: image, video, or carousel.",
|
| 1044 |
"url": "Direct URL to the Instagram post.",
|
| 1045 |
"verified": "Checked when Instagram marks the profile as verified.",
|
|
@@ -2124,6 +2126,13 @@ POST_FIELDS = [
|
|
| 2124 |
field_def("posted_at", "Posted at", "date"),
|
| 2125 |
field_def("type", "Type", "select", options=["image", "video", "carousel"]),
|
| 2126 |
field_def("caption", "Caption"), field_def("url", "URL", "url"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2127 |
# Sponsored-post detection β Β§2c called it one of the fields worth having that the anonymous
|
| 2128 |
# ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
|
| 2129 |
field_def("paid_partnership", "Paid partnership", "checkbox"),
|
|
@@ -2555,10 +2564,15 @@ def _first(node, *names, default=None):
|
|
| 2555 |
#: buy the Comments dataset" β MEASURED 2026-08-05, a POSTS row arrives with `latest_comments`
|
| 2556 |
#: as an array of 15 comment texts WITH their authors, whether we asked or not. The ruling is
|
| 2557 |
#: enforced HERE, at the map, because that is the only place it can be.
|
| 2558 |
-
BD_NEVER_MAP = frozenset({
|
| 2559 |
-
"email_address", "phone_number", "business_email", "profile_image_link",
|
| 2560 |
-
"latest_comments", "comment_user", "post_user",
|
| 2561 |
-
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2562 |
|
| 2563 |
#: The vendor's content vocabulary β ours. MEASURED values: profiles' `posts[].content_type` is
|
| 2564 |
#: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`.
|
|
@@ -2638,13 +2652,53 @@ def _bd_flag(node, *names):
|
|
| 2638 |
return None
|
| 2639 |
|
| 2640 |
|
| 2641 |
-
def _bd_list(node, *names):
|
| 2642 |
"""An array field β a comma-joined string, or None when it is absent/empty."""
|
| 2643 |
for n in names:
|
| 2644 |
v = node.get(n) if isinstance(node, dict) else None
|
| 2645 |
if isinstance(v, list) and v:
|
| 2646 |
return ", ".join(str(x) for x in v if x)[:400]
|
| 2647 |
-
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2648 |
|
| 2649 |
|
| 2650 |
def _bd_profile(node, handle):
|
|
@@ -2790,11 +2844,13 @@ def _bd_post_metrics(row):
|
|
| 2790 |
"plays": _ig_int(_first(row, "video_play_count", "play_count")),
|
| 2791 |
# Sponsored-post detection. MEASURED populated (`True`, with the brand alongside) β and
|
| 2792 |
# it is the one field here that answers a commercial question the counts cannot.
|
| 2793 |
-
"paid_partnership": _bd_flag(row, "is_paid_partnership"),
|
| 2794 |
-
"partner": (str(_first(partner, "username", "profile_id", default="") or "")
|
| 2795 |
-
if isinstance(partner, dict) else ""),
|
| 2796 |
-
"hashtags": _bd_list(row, "hashtags"),
|
| 2797 |
-
"alt_text": str(_first(row, "alt_text", default="") or ""),
|
|
|
|
|
|
|
| 2798 |
}
|
| 2799 |
# β SAME RAIL AS THE PROFILE MAP β and it matters MORE here, because a Posts row genuinely
|
| 2800 |
# arrives carrying `latest_comments`: fifteen third-party comment texts with their authors,
|
|
@@ -5084,9 +5140,10 @@ def capture_rows(res, pulled):
|
|
| 5084 |
for p in (res or {}).get("posts") or []:
|
| 5085 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5086 |
"url": _s(p.get("url"), 300)}
|
| 5087 |
-
for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
|
| 5088 |
-
("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
|
| 5089 |
-
("alt_text", 400))
|
|
|
|
| 5090 |
if p.get(k):
|
| 5091 |
ident[k] = _s(p.get(k), n)
|
| 5092 |
metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
|
|
|
|
| 1038 |
"related_accounts": "Accounts Instagram suggests alongside this profile.",
|
| 1039 |
"replies": "Reply count reported for the comment.",
|
| 1040 |
"shortcode": "Instagram shortcode that uniquely identifies the post.",
|
| 1041 |
+
"source_payload": "Full Bright Data post response, excluding protected contact and comment data.",
|
| 1042 |
"snapshot_key": "Unique ID for this profile snapshot.",
|
| 1043 |
"source": "Method or provider used to read this profile.",
|
| 1044 |
+
"tagged_location": "Location tagged on the post; it is not the creator's residence.",
|
| 1045 |
"type": "Instagram media type: image, video, or carousel.",
|
| 1046 |
"url": "Direct URL to the Instagram post.",
|
| 1047 |
"verified": "Checked when Instagram marks the profile as verified.",
|
|
|
|
| 2126 |
field_def("posted_at", "Posted at", "date"),
|
| 2127 |
field_def("type", "Type", "select", options=["image", "video", "carousel"]),
|
| 2128 |
field_def("caption", "Caption"), field_def("url", "URL", "url"),
|
| 2129 |
+
# A tagged place is useful content context and can be filtered as ordinary text. It is never
|
| 2130 |
+
# presented as the creator's location: a creator can tag a holiday, venue or brand location.
|
| 2131 |
+
field_def("tagged_location", "Tagged location"),
|
| 2132 |
+
# The per-field map below gives operational fields first; this locked JSON document retains
|
| 2133 |
+
# every other PUBLIC value Bright Data supplied, so a vendor schema addition is preserved
|
| 2134 |
+
# instead of being silently discarded while the canonical field model catches up.
|
| 2135 |
+
field_def("source_payload", "Source data", "json"),
|
| 2136 |
# Sponsored-post detection β Β§2c called it one of the fields worth having that the anonymous
|
| 2137 |
# ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
|
| 2138 |
field_def("paid_partnership", "Paid partnership", "checkbox"),
|
|
|
|
| 2564 |
#: buy the Comments dataset" β MEASURED 2026-08-05, a POSTS row arrives with `latest_comments`
|
| 2565 |
#: as an array of 15 comment texts WITH their authors, whether we asked or not. The ruling is
|
| 2566 |
#: enforced HERE, at the map, because that is the only place it can be.
|
| 2567 |
+
BD_NEVER_MAP = frozenset({
|
| 2568 |
+
"email_address", "phone_number", "business_email", "profile_image_link",
|
| 2569 |
+
"latest_comments", "top_comments", "comment_user", "post_user",
|
| 2570 |
+
})
|
| 2571 |
+
|
| 2572 |
+
# `json` cells are capped by the table layer. Keep this local literal with the same boot-path
|
| 2573 |
+
# discipline as the other user-table contracts in this module: importing `core` here would make
|
| 2574 |
+
# the API's automation boot path depend on the whole platform package.
|
| 2575 |
+
BD_PUBLIC_POST_PAYLOAD_MAX = 32 * 1024
|
| 2576 |
|
| 2577 |
#: The vendor's content vocabulary β ours. MEASURED values: profiles' `posts[].content_type` is
|
| 2578 |
#: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`.
|
|
|
|
| 2652 |
return None
|
| 2653 |
|
| 2654 |
|
| 2655 |
+
def _bd_list(node, *names):
|
| 2656 |
"""An array field β a comma-joined string, or None when it is absent/empty."""
|
| 2657 |
for n in names:
|
| 2658 |
v = node.get(n) if isinstance(node, dict) else None
|
| 2659 |
if isinstance(v, list) and v:
|
| 2660 |
return ", ".join(str(x) for x in v if x)[:400]
|
| 2661 |
+
return None
|
| 2662 |
+
|
| 2663 |
+
|
| 2664 |
+
def _bd_tagged_location(node):
|
| 2665 |
+
"""Bright Data's optional Post `location` -> one filterable content-context string.
|
| 2666 |
+
|
| 2667 |
+
The documented Posts response is an ordered array such as
|
| 2668 |
+
`["Capanema", "ParΓ‘", "Brasil"]`; retain the order rather than guessing which component is
|
| 2669 |
+
a city or country. Reels do not promise this field, so a blank means it was not returned.
|
| 2670 |
+
"""
|
| 2671 |
+
raw = _first(node, "location", "location_details")
|
| 2672 |
+
if isinstance(raw, dict):
|
| 2673 |
+
raw = [raw.get(k) for k in ("name", "city", "region", "state", "country", "slug")]
|
| 2674 |
+
if not isinstance(raw, list):
|
| 2675 |
+
raw = [raw]
|
| 2676 |
+
parts = []
|
| 2677 |
+
for value in raw:
|
| 2678 |
+
if isinstance(value, dict):
|
| 2679 |
+
value = _first(value, "name", "city", "region", "state", "country", "slug")
|
| 2680 |
+
text = str(value or "").strip()
|
| 2681 |
+
if text and text not in parts:
|
| 2682 |
+
parts.append(text)
|
| 2683 |
+
return ", ".join(parts)[:400] or None
|
| 2684 |
+
|
| 2685 |
+
|
| 2686 |
+
def _bd_public_post_payload(row):
|
| 2687 |
+
"""A canonical JSON copy of every public Post/Reel response field.
|
| 2688 |
+
|
| 2689 |
+
This is intentionally a *copy*, not the original mutable provider dict. The PII/comment rail
|
| 2690 |
+
applies before serialisation, so neither `latest_comments` nor Reels' `top_comments` can slip
|
| 2691 |
+
into a generic "all fields" document. The table's JSON cap is observed before a value reaches
|
| 2692 |
+
the write door; an over-cap response is omitted rather than truncated into invalid JSON.
|
| 2693 |
+
"""
|
| 2694 |
+
if not isinstance(row, dict):
|
| 2695 |
+
return None
|
| 2696 |
+
public = {str(k): v for k, v in row.items() if str(k) not in BD_NEVER_MAP}
|
| 2697 |
+
try:
|
| 2698 |
+
packed = json.dumps(public, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
| 2699 |
+
except (TypeError, ValueError):
|
| 2700 |
+
return None
|
| 2701 |
+
return packed if len(packed.encode("utf-8")) <= BD_PUBLIC_POST_PAYLOAD_MAX else None
|
| 2702 |
|
| 2703 |
|
| 2704 |
def _bd_profile(node, handle):
|
|
|
|
| 2844 |
"plays": _ig_int(_first(row, "video_play_count", "play_count")),
|
| 2845 |
# Sponsored-post detection. MEASURED populated (`True`, with the brand alongside) β and
|
| 2846 |
# it is the one field here that answers a commercial question the counts cannot.
|
| 2847 |
+
"paid_partnership": _bd_flag(row, "is_paid_partnership"),
|
| 2848 |
+
"partner": (str(_first(partner, "username", "profile_id", default="") or "")
|
| 2849 |
+
if isinstance(partner, dict) else ""),
|
| 2850 |
+
"hashtags": _bd_list(row, "hashtags"),
|
| 2851 |
+
"alt_text": str(_first(row, "alt_text", default="") or ""),
|
| 2852 |
+
"tagged_location": _bd_tagged_location(row),
|
| 2853 |
+
"source_payload": _bd_public_post_payload(row),
|
| 2854 |
}
|
| 2855 |
# β SAME RAIL AS THE PROFILE MAP β and it matters MORE here, because a Posts row genuinely
|
| 2856 |
# arrives carrying `latest_comments`: fifteen third-party comment texts with their authors,
|
|
|
|
| 5140 |
for p in (res or {}).get("posts") or []:
|
| 5141 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
| 5142 |
"url": _s(p.get("url"), 300)}
|
| 5143 |
+
for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
|
| 5144 |
+
("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
|
| 5145 |
+
("alt_text", 400), ("tagged_location", 400),
|
| 5146 |
+
("source_payload", BD_PUBLIC_POST_PAYLOAD_MAX)):
|
| 5147 |
if p.get(k):
|
| 5148 |
ident[k] = _s(p.get(k), n)
|
| 5149 |
metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
|