Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +159 -52
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "8b81d13",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v14",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
8b81d13
|
api/automation_engine.py
CHANGED
|
@@ -2003,26 +2003,26 @@ IG_MAX_RETRIES = 2
|
|
| 2003 |
#: (R11), both enforced in `BD_NEVER_MAP`.
|
| 2004 |
#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose
|
| 2005 |
#: almost none of them, which is what the `source` column is for.
|
| 2006 |
-
SNAPSHOT_FIELDS = [
|
| 2007 |
-
field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"),
|
| 2008 |
-
field_def("pulled_at", "Pulled at"), field_def("followers", "Followers"),
|
| 2009 |
-
field_def("following", "Following"), field_def("posts_count", "Posts"),
|
| 2010 |
-
field_def("full_name", "Name"), field_def("bio", "Bio"),
|
| 2011 |
-
field_def("verified", "Verified"), field_def("source", "Read via"),
|
| 2012 |
-
field_def("approx", "Counts are approximate"),
|
| 2013 |
# The link in bio — the paid rung's field, and commercially the most useful single string an
|
| 2014 |
# influencer row can carry (it is the shop/affiliate destination).
|
| 2015 |
field_def("external_url", "Link in bio", "url"),
|
| 2016 |
# --- the rest of the vendor's Profiles schema, minus the PII and the comment text.
|
| 2017 |
field_def("ig_id", "Instagram id"),
|
| 2018 |
field_def("profile_url", "Profile", "url"),
|
| 2019 |
-
field_def("avg_engagement", "Avg engagement"),
|
| 2020 |
-
field_def("category", "Category"),
|
| 2021 |
-
field_def("business_category", "Business category"),
|
| 2022 |
-
field_def("is_business", "Business account"),
|
| 2023 |
-
field_def("is_professional", "Professional account"),
|
| 2024 |
-
field_def("is_private", "Private"),
|
| 2025 |
-
field_def("highlights_count", "
|
| 2026 |
field_def("bio_hashtags", "Bio hashtags"),
|
| 2027 |
field_def("pronouns", "Pronouns"),
|
| 2028 |
# ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
|
|
@@ -2030,21 +2030,22 @@ SNAPSHOT_FIELDS = [
|
|
| 2030 |
# leaving them out would be the "captured but unrecorded" half of the same loss the note at
|
| 2031 |
# the top of this list is about.
|
| 2032 |
field_def("profile_name", "Profile name"),
|
| 2033 |
-
field_def("is_joined_recently", "Joined recently"),
|
| 2034 |
-
field_def("has_channel", "Has channel"),
|
| 2035 |
field_def("partner_id", "Partner id"),
|
| 2036 |
field_def("external_url_title", "Link title"),
|
| 2037 |
field_def("fbid", "Facebook id"),
|
| 2038 |
field_def("related_accounts", "Related accounts"),
|
| 2039 |
field_def("country_code", "Country"),
|
| 2040 |
]
|
| 2041 |
-
POST_FIELDS = [
|
| 2042 |
-
field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
|
| 2043 |
-
field_def("posted_at", "Posted at"
|
| 2044 |
-
field_def("
|
|
|
|
| 2045 |
# Sponsored-post detection — §2c called it one of the fields worth having that the anonymous
|
| 2046 |
# ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
|
| 2047 |
-
field_def("paid_partnership", "Paid partnership"),
|
| 2048 |
field_def("partner", "Partner brand"),
|
| 2049 |
field_def("hashtags", "Hashtags"),
|
| 2050 |
field_def("alt_text", "Alt text"),
|
|
@@ -2126,11 +2127,11 @@ COMMENT_FIELDS = [
|
|
| 2126 |
# What is left is the SHAPE of a thread — how many, how recent, how engaged — which is the
|
| 2127 |
# part that informs an influencer decision without ingesting a stranger.
|
| 2128 |
]
|
| 2129 |
-
POST_SNAPSHOT_FIELDS = [
|
| 2130 |
field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
|
| 2131 |
field_def("influencer_key", "Influencer"),
|
| 2132 |
-
field_def("pulled_at", "Pulled at"), field_def("likes", "Likes"),
|
| 2133 |
-
field_def("comments", "Comments"),
|
| 2134 |
# Plays/views move independently of likes on video, so it is its own series rather than a
|
| 2135 |
# thing to derive. Paid rung only; blank means not read.
|
| 2136 |
field_def("views", "Views", "int"),
|
|
@@ -3571,11 +3572,70 @@ def _profile_schema_for(bound_key):
|
|
| 3571 |
item = dict(field)
|
| 3572 |
if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY:
|
| 3573 |
item.pop("profile", None)
|
| 3574 |
-
item.pop("pinned", None)
|
| 3575 |
out.append(item)
|
| 3576 |
return out
|
| 3577 |
|
| 3578 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3579 |
def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
|
| 3580 |
"""Repair machine-owned IG field declarations in one coalesced store update.
|
| 3581 |
|
|
@@ -3585,6 +3645,10 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
|
|
| 3585 |
unrelated/user fields, and removes only explicitly retired preset columns plus their cells.
|
| 3586 |
"""
|
| 3587 |
drop_by_table = drop_by_table or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3588 |
changed = False
|
| 3589 |
|
| 3590 |
def _up(cur):
|
|
@@ -3607,12 +3671,28 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
|
|
| 3607 |
fields.append(stored)
|
| 3608 |
continue
|
| 3609 |
repaired = dict(stored)
|
| 3610 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3611 |
# These bags define behaviour, not decoration. If the desired field does not use
|
| 3612 |
# one, a stale bag from an earlier type must not remain attached to it.
|
| 3613 |
-
for bag in ("link", "rollup"):
|
| 3614 |
if bag not in desired:
|
| 3615 |
repaired.pop(bag, None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3616 |
if repaired != stored:
|
| 3617 |
changed = True
|
| 3618 |
fields.append(repaired)
|
|
@@ -3634,6 +3714,8 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
|
|
| 3634 |
snapshot = ut_all(rt)
|
| 3635 |
needs = False
|
| 3636 |
for table_key, wanted_fields in wanted_by_table.items():
|
|
|
|
|
|
|
| 3637 |
table = snapshot.get(table_key) or {}
|
| 3638 |
by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []}
|
| 3639 |
drops = set(drop_by_table.get(table_key) or ())
|
|
@@ -3642,7 +3724,24 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
|
|
| 3642 |
break
|
| 3643 |
for desired in wanted_fields:
|
| 3644 |
stored = by_key.get(str(desired.get("key") or ""))
|
| 3645 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3646 |
needs = True
|
| 3647 |
break
|
| 3648 |
if needs:
|
|
@@ -3679,19 +3778,14 @@ def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", pr
|
|
| 3679 |
keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag,
|
| 3680 |
record_mode=AUTOMATION_RECORD_MODE, lock_fields=True)
|
| 3681 |
|
| 3682 |
-
# `ut_ensure` stamps newly created fields with the first flow that introduced them. Do not
|
| 3683 |
-
# put that tag into the reconciliation template: these canonical tables are shared by many
|
| 3684 |
-
# Profile databases, and otherwise every run would rewrite their provenance from flow A to
|
| 3685 |
-
# flow B and back again.
|
| 3686 |
-
wanted = {key: [dict(f) for f in fields]
|
| 3687 |
-
for key, (_label, fields) in graph.items()}
|
| 3688 |
-
drops = {IG_SNAPSHOTS_TABLE: {"post_hashtags"}}
|
| 3689 |
if profile_table and profile is not None and bound:
|
| 3690 |
profile_fields = _profile_schema_for(bound)
|
| 3691 |
ut_ensure(rt, profile_label or profile_table, profile_fields, username,
|
| 3692 |
key=profile_table, flow_tag=flow_tag, lock_fields=True)
|
| 3693 |
-
|
| 3694 |
-
|
|
|
|
|
|
|
| 3695 |
_reconcile_ig_graph_fields(rt, wanted, drops)
|
| 3696 |
return keys
|
| 3697 |
|
|
@@ -3829,11 +3923,13 @@ def _merge_candidates(rows):
|
|
| 3829 |
return out, merged
|
| 3830 |
|
| 3831 |
|
| 3832 |
-
def migrate_ig_tables(rt, log=print, only=""):
|
| 3833 |
-
"""
|
| 3834 |
-
|
| 3835 |
-
Returns
|
| 3836 |
-
|
|
|
|
|
|
|
| 3837 |
|
| 3838 |
`only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides
|
| 3839 |
the WRITE PATH, so a table is brought forward immediately before anything appends to it and no
|
|
@@ -3842,9 +3938,10 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 3842 |
stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0,
|
| 3843 |
# ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from
|
| 3844 |
# `stamped` because they answer different questions: that one is "how many ROWS got a
|
| 3845 |
-
# platform", these are "how many TABLES changed shape".
|
| 3846 |
"pinned": 0, "flagged": 0, "droppedName": 0,
|
| 3847 |
-
"droppedRecentPosts": 0, "droppedPostHashtags": 0
|
|
|
|
| 3848 |
want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
|
| 3849 |
# ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red
|
| 3850 |
# checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
|
|
@@ -3889,7 +3986,7 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 3889 |
want_flag = (flag_f is not None and other_profile is None
|
| 3890 |
and not isinstance(flag_f.get("profile"), dict))
|
| 3891 |
vestigial = _vestigial_name_field(fields, rows)
|
| 3892 |
-
retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"}
|
| 3893 |
if (not stale_keys and has_platform and not needs_stamp
|
| 3894 |
and not want_pin and not want_flag and vestigial is None and not retired):
|
| 3895 |
continue
|
|
@@ -3915,6 +4012,7 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 3915 |
r.pop(key, None)
|
| 3916 |
stats["droppedRecentPosts"] += int("posts" in retired)
|
| 3917 |
stats["droppedPostHashtags"] += int("post_hashtags" in retired)
|
|
|
|
| 3918 |
|
| 3919 |
for key in stale_keys:
|
| 3920 |
conv = _MIGRATE_CONVERT.get(key)
|
|
@@ -3945,9 +4043,14 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 3945 |
cur[table_key] = t
|
| 3946 |
return cur
|
| 3947 |
|
| 3948 |
-
rt.update(UT_STORE_KEY, _up)
|
| 3949 |
-
|
| 3950 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3951 |
return stats
|
| 3952 |
|
| 3953 |
|
|
@@ -4879,10 +4982,14 @@ def capture_rows(res, pulled):
|
|
| 4879 |
"bio": _s(prof.get("bio"), 500),
|
| 4880 |
"external_url": _s(prof.get("external_url"), 300),
|
| 4881 |
}
|
| 4882 |
-
for fd in SNAPSHOT_FIELDS:
|
| 4883 |
-
k = fd["key"]
|
| 4884 |
-
if k not in snap_row:
|
| 4885 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4886 |
idents, metrics_rows = [], []
|
| 4887 |
for p in (res or {}).get("posts") or []:
|
| 4888 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|
|
|
|
| 2003 |
#: (R11), both enforced in `BD_NEVER_MAP`.
|
| 2004 |
#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose
|
| 2005 |
#: almost none of them, which is what the `source` column is for.
|
| 2006 |
+
SNAPSHOT_FIELDS = [
|
| 2007 |
+
field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"),
|
| 2008 |
+
field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"),
|
| 2009 |
+
field_def("following", "Following", "int"), field_def("posts_count", "Posts", "int"),
|
| 2010 |
+
field_def("full_name", "Name"), field_def("bio", "Bio"),
|
| 2011 |
+
field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"),
|
| 2012 |
+
field_def("approx", "Counts are approximate", "checkbox"),
|
| 2013 |
# The link in bio — the paid rung's field, and commercially the most useful single string an
|
| 2014 |
# influencer row can carry (it is the shop/affiliate destination).
|
| 2015 |
field_def("external_url", "Link in bio", "url"),
|
| 2016 |
# --- the rest of the vendor's Profiles schema, minus the PII and the comment text.
|
| 2017 |
field_def("ig_id", "Instagram id"),
|
| 2018 |
field_def("profile_url", "Profile", "url"),
|
| 2019 |
+
field_def("avg_engagement", "Avg engagement", "pct"),
|
| 2020 |
+
field_def("category", "Category"),
|
| 2021 |
+
field_def("business_category", "Business category"),
|
| 2022 |
+
field_def("is_business", "Business account", "checkbox"),
|
| 2023 |
+
field_def("is_professional", "Professional account", "checkbox"),
|
| 2024 |
+
field_def("is_private", "Private", "checkbox"),
|
| 2025 |
+
field_def("highlights_count", "Story highlights", "int"),
|
| 2026 |
field_def("bio_hashtags", "Bio hashtags"),
|
| 2027 |
field_def("pronouns", "Pronouns"),
|
| 2028 |
# ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
|
|
|
|
| 2030 |
# leaving them out would be the "captured but unrecorded" half of the same loss the note at
|
| 2031 |
# the top of this list is about.
|
| 2032 |
field_def("profile_name", "Profile name"),
|
| 2033 |
+
field_def("is_joined_recently", "Joined recently", "checkbox"),
|
| 2034 |
+
field_def("has_channel", "Has channel", "checkbox"),
|
| 2035 |
field_def("partner_id", "Partner id"),
|
| 2036 |
field_def("external_url_title", "Link title"),
|
| 2037 |
field_def("fbid", "Facebook id"),
|
| 2038 |
field_def("related_accounts", "Related accounts"),
|
| 2039 |
field_def("country_code", "Country"),
|
| 2040 |
]
|
| 2041 |
+
POST_FIELDS = [
|
| 2042 |
+
field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
|
| 2043 |
+
field_def("posted_at", "Posted at", "date"),
|
| 2044 |
+
field_def("type", "Type", "select", options=["image", "video", "carousel"]),
|
| 2045 |
+
field_def("caption", "Caption"), field_def("url", "URL", "url"),
|
| 2046 |
# Sponsored-post detection — §2c called it one of the fields worth having that the anonymous
|
| 2047 |
# ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
|
| 2048 |
+
field_def("paid_partnership", "Paid partnership", "checkbox"),
|
| 2049 |
field_def("partner", "Partner brand"),
|
| 2050 |
field_def("hashtags", "Hashtags"),
|
| 2051 |
field_def("alt_text", "Alt text"),
|
|
|
|
| 2127 |
# What is left is the SHAPE of a thread — how many, how recent, how engaged — which is the
|
| 2128 |
# part that informs an influencer decision without ingesting a stranger.
|
| 2129 |
]
|
| 2130 |
+
POST_SNAPSHOT_FIELDS = [
|
| 2131 |
field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
|
| 2132 |
field_def("influencer_key", "Influencer"),
|
| 2133 |
+
field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"),
|
| 2134 |
+
field_def("comments", "Comments", "int"),
|
| 2135 |
# Plays/views move independently of likes on video, so it is its own series rather than a
|
| 2136 |
# thing to derive. Paid rung only; blank means not read.
|
| 2137 |
field_def("views", "Views", "int"),
|
|
|
|
| 3572 |
item = dict(field)
|
| 3573 |
if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY:
|
| 3574 |
item.pop("profile", None)
|
|
|
|
| 3575 |
out.append(item)
|
| 3576 |
return out
|
| 3577 |
|
| 3578 |
|
| 3579 |
+
def _locked_ig_field(field):
|
| 3580 |
+
"""One canonical Instagram field with the immutable preset declaration attached."""
|
| 3581 |
+
item = dict(field)
|
| 3582 |
+
automation = dict(item.get("automation") or {})
|
| 3583 |
+
automation["preset"] = True
|
| 3584 |
+
item["automation"] = automation
|
| 3585 |
+
return item
|
| 3586 |
+
|
| 3587 |
+
|
| 3588 |
+
def _profile_binding(table):
|
| 3589 |
+
"""The declared Profile identity field, falling back only to the canonical handle."""
|
| 3590 |
+
fields = list((table or {}).get("fields") or [])
|
| 3591 |
+
flagged = next((str(f.get("key") or "") for f in fields
|
| 3592 |
+
if isinstance(f.get("profile"), dict)), "")
|
| 3593 |
+
if flagged:
|
| 3594 |
+
return flagged
|
| 3595 |
+
return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else ""
|
| 3596 |
+
|
| 3597 |
+
|
| 3598 |
+
def _ig_schema_contract(rt, profile_tables=None):
|
| 3599 |
+
"""The complete per-tenant Instagram graph contract, including every Profile backlink.
|
| 3600 |
+
|
| 3601 |
+
The fixed child tables are shared. Profile tables are discovered structurally, so an older
|
| 3602 |
+
user-named target and the built-in candidate database receive the same preset columns, Links,
|
| 3603 |
+
Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted.
|
| 3604 |
+
"""
|
| 3605 |
+
tables = ut_all(rt)
|
| 3606 |
+
profile_keys = sorted(set(profile_tables if profile_tables is not None
|
| 3607 |
+
else _ig_profile_tables(rt)))
|
| 3608 |
+
backlinks, wanted, drops = [], {}, {
|
| 3609 |
+
IG_SNAPSHOTS_TABLE: {"post_hashtags"},
|
| 3610 |
+
}
|
| 3611 |
+
for table_key in profile_keys:
|
| 3612 |
+
table = tables.get(table_key) or {}
|
| 3613 |
+
bound = _profile_binding(table)
|
| 3614 |
+
if not bound:
|
| 3615 |
+
continue
|
| 3616 |
+
label = str(table.get("label") or table_key)
|
| 3617 |
+
backlinks.append(_profile_backlink_field(table_key, label, bound))
|
| 3618 |
+
wanted[table_key] = _profile_schema_for(bound)
|
| 3619 |
+
# `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
|
| 3620 |
+
# to disagree, so it is retired by key just like the old nested Posts JSON.
|
| 3621 |
+
drops[table_key] = {"posts", "post_hashtags", "tracked"}
|
| 3622 |
+
wanted.update({
|
| 3623 |
+
IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
|
| 3624 |
+
IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
|
| 3625 |
+
IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks],
|
| 3626 |
+
IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks],
|
| 3627 |
+
})
|
| 3628 |
+
return wanted, drops
|
| 3629 |
+
|
| 3630 |
+
|
| 3631 |
+
# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use
|
| 3632 |
+
# the scalar strings their renderers expect; engagement is the exception because Bright Data's
|
| 3633 |
+
# 0-1 fraction becomes this product's 0-100 percentage.
|
| 3634 |
+
_IG_RETYPE_CONVERTERS = {
|
| 3635 |
+
(IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100,
|
| 3636 |
+
}
|
| 3637 |
+
|
| 3638 |
+
|
| 3639 |
def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
|
| 3640 |
"""Repair machine-owned IG field declarations in one coalesced store update.
|
| 3641 |
|
|
|
|
| 3645 |
unrelated/user fields, and removes only explicitly retired preset columns plus their cells.
|
| 3646 |
"""
|
| 3647 |
drop_by_table = drop_by_table or {}
|
| 3648 |
+
wanted_by_table = {
|
| 3649 |
+
key: [_locked_ig_field(f) for f in fields]
|
| 3650 |
+
for key, fields in wanted_by_table.items()
|
| 3651 |
+
}
|
| 3652 |
changed = False
|
| 3653 |
|
| 3654 |
def _up(cur):
|
|
|
|
| 3671 |
fields.append(stored)
|
| 3672 |
continue
|
| 3673 |
repaired = dict(stored)
|
| 3674 |
+
for dkey, value in desired.items():
|
| 3675 |
+
if dkey != "automation":
|
| 3676 |
+
repaired[dkey] = value
|
| 3677 |
+
automation = dict(stored.get("automation") or {})
|
| 3678 |
+
automation.update(desired.get("automation") or {})
|
| 3679 |
+
repaired["automation"] = automation
|
| 3680 |
# These bags define behaviour, not decoration. If the desired field does not use
|
| 3681 |
# one, a stale bag from an earlier type must not remain attached to it.
|
| 3682 |
+
for bag in ("link", "rollup", "profile", "pinned"):
|
| 3683 |
if bag not in desired:
|
| 3684 |
repaired.pop(bag, None)
|
| 3685 |
+
if desired.get("type") not in ("select", "multiselect"):
|
| 3686 |
+
repaired.pop("options", None)
|
| 3687 |
+
converter = _IG_RETYPE_CONVERTERS.get((table_key, key))
|
| 3688 |
+
if (converter and stored.get("type") in ("text", "", None)
|
| 3689 |
+
and repaired.get("type") != stored.get("type")):
|
| 3690 |
+
for row in (table.get("rows") or {}).values():
|
| 3691 |
+
if str(row.get(key) or "").strip():
|
| 3692 |
+
converted = converter(row.get(key))
|
| 3693 |
+
if converted:
|
| 3694 |
+
row[key] = converted
|
| 3695 |
+
changed = True
|
| 3696 |
if repaired != stored:
|
| 3697 |
changed = True
|
| 3698 |
fields.append(repaired)
|
|
|
|
| 3714 |
snapshot = ut_all(rt)
|
| 3715 |
needs = False
|
| 3716 |
for table_key, wanted_fields in wanted_by_table.items():
|
| 3717 |
+
if table_key not in snapshot:
|
| 3718 |
+
continue
|
| 3719 |
table = snapshot.get(table_key) or {}
|
| 3720 |
by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []}
|
| 3721 |
drops = set(drop_by_table.get(table_key) or ())
|
|
|
|
| 3724 |
break
|
| 3725 |
for desired in wanted_fields:
|
| 3726 |
stored = by_key.get(str(desired.get("key") or ""))
|
| 3727 |
+
mismatch = stored is None
|
| 3728 |
+
if stored is not None:
|
| 3729 |
+
for key, value in desired.items():
|
| 3730 |
+
if key == "automation":
|
| 3731 |
+
if any((stored.get("automation") or {}).get(k) != v
|
| 3732 |
+
for k, v in value.items()):
|
| 3733 |
+
mismatch = True
|
| 3734 |
+
break
|
| 3735 |
+
elif stored.get(key) != value:
|
| 3736 |
+
mismatch = True
|
| 3737 |
+
break
|
| 3738 |
+
if not mismatch:
|
| 3739 |
+
stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired)
|
| 3740 |
+
mismatch = any(k in stored for k in stale_bags)
|
| 3741 |
+
if (not mismatch and desired.get("type") not in ("select", "multiselect")
|
| 3742 |
+
and "options" in stored):
|
| 3743 |
+
mismatch = True
|
| 3744 |
+
if mismatch:
|
| 3745 |
needs = True
|
| 3746 |
break
|
| 3747 |
if needs:
|
|
|
|
| 3778 |
keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag,
|
| 3779 |
record_mode=AUTOMATION_RECORD_MODE, lock_fields=True)
|
| 3780 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3781 |
if profile_table and profile is not None and bound:
|
| 3782 |
profile_fields = _profile_schema_for(bound)
|
| 3783 |
ut_ensure(rt, profile_label or profile_table, profile_fields, username,
|
| 3784 |
key=profile_table, flow_tag=flow_tag, lock_fields=True)
|
| 3785 |
+
# Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older
|
| 3786 |
+
# Profile database and IG candidates on the same universal schema while all of them point to
|
| 3787 |
+
# the same four children. Existing flow provenance is merged, never replaced.
|
| 3788 |
+
wanted, drops = _ig_schema_contract(rt)
|
| 3789 |
_reconcile_ig_graph_fields(rt, wanted, drops)
|
| 3790 |
return keys
|
| 3791 |
|
|
|
|
| 3923 |
return out, merged
|
| 3924 |
|
| 3925 |
|
| 3926 |
+
def migrate_ig_tables(rt, log=print, only=""):
|
| 3927 |
+
"""Bring every Instagram Profile and canonical child table onto one current contract.
|
| 3928 |
+
|
| 3929 |
+
Returns counted changes rather than a reassuring line. Profile cells are converted before
|
| 3930 |
+
their type declarations change; then the graph reconciliation adds any missing preset
|
| 3931 |
+
Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes
|
| 3932 |
+
only retired machine fields.
|
| 3933 |
|
| 3934 |
`only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides
|
| 3935 |
the WRITE PATH, so a table is brought forward immediately before anything appends to it and no
|
|
|
|
| 3938 |
stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0,
|
| 3939 |
# ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from
|
| 3940 |
# `stamped` because they answer different questions: that one is "how many ROWS got a
|
| 3941 |
+
# platform", these are "how many TABLES changed shape".
|
| 3942 |
"pinned": 0, "flagged": 0, "droppedName": 0,
|
| 3943 |
+
"droppedRecentPosts": 0, "droppedPostHashtags": 0,
|
| 3944 |
+
"droppedTracked": 0, "schema": 0}
|
| 3945 |
want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
|
| 3946 |
# ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red
|
| 3947 |
# checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
|
|
|
|
| 3986 |
want_flag = (flag_f is not None and other_profile is None
|
| 3987 |
and not isinstance(flag_f.get("profile"), dict))
|
| 3988 |
vestigial = _vestigial_name_field(fields, rows)
|
| 3989 |
+
retired = {f.get("key") for f in fields} & {"posts", "post_hashtags", "tracked"}
|
| 3990 |
if (not stale_keys and has_platform and not needs_stamp
|
| 3991 |
and not want_pin and not want_flag and vestigial is None and not retired):
|
| 3992 |
continue
|
|
|
|
| 4012 |
r.pop(key, None)
|
| 4013 |
stats["droppedRecentPosts"] += int("posts" in retired)
|
| 4014 |
stats["droppedPostHashtags"] += int("post_hashtags" in retired)
|
| 4015 |
+
stats["droppedTracked"] += int("tracked" in retired)
|
| 4016 |
|
| 4017 |
for key in stale_keys:
|
| 4018 |
conv = _MIGRATE_CONVERT.get(key)
|
|
|
|
| 4043 |
cur[table_key] = t
|
| 4044 |
return cur
|
| 4045 |
|
| 4046 |
+
rt.update(UT_STORE_KEY, _up)
|
| 4047 |
+
# The profile loop owns value conversions and deduplication. The schema pass owns the
|
| 4048 |
+
# declarations across ALL related databases and is safe only after those conversions, because
|
| 4049 |
+
# the stored old type is the idempotency key for unit-changing values such as engagement.
|
| 4050 |
+
wanted, drops = _ig_schema_contract(rt)
|
| 4051 |
+
stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
|
| 4052 |
+
if stats["tables"] or stats["schema"]:
|
| 4053 |
+
log(f"[ig-migrate] {stats}")
|
| 4054 |
return stats
|
| 4055 |
|
| 4056 |
|
|
|
|
| 4982 |
"bio": _s(prof.get("bio"), 500),
|
| 4983 |
"external_url": _s(prof.get("external_url"), 300),
|
| 4984 |
}
|
| 4985 |
+
for fd in SNAPSHOT_FIELDS:
|
| 4986 |
+
k = fd["key"]
|
| 4987 |
+
if k not in snap_row:
|
| 4988 |
+
value = prof.get(k)
|
| 4989 |
+
# Profile history declares the same percentage dialect as the current Profile row.
|
| 4990 |
+
# Converting only the latest projection left the append series at 0-1 while its field
|
| 4991 |
+
# now said pct (0-100), so the two views of one measurement disagreed by 100x.
|
| 4992 |
+
snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400)
|
| 4993 |
idents, metrics_rows = [], []
|
| 4994 |
for p in (res or {}).get("posts") or []:
|
| 4995 |
ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
|