fsanyoto commited on
Commit
ef448b2
·
verified ·
1 Parent(s): e4d9f48

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
Files changed (3) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/automation_engine.py +159 -52
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "139bf21",
3
  "releases": [
4
  {
5
  "version": "v14",
 
1
  {
2
+ "current": "8b81d13",
3
  "releases": [
4
  {
5
  "version": "v14",
VERSION CHANGED
@@ -1 +1 @@
1
- 139bf21
 
1
+ 8b81d13
api/automation_engine.py CHANGED
@@ -2003,26 +2003,26 @@ IG_MAX_RETRIES = 2
2003
  #: (R11), both enforced in `BD_NEVER_MAP`.
2004
  #: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose
2005
  #: almost none of them, which is what the `source` column is for.
2006
- SNAPSHOT_FIELDS = [
2007
- field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"),
2008
- field_def("pulled_at", "Pulled at"), field_def("followers", "Followers"),
2009
- field_def("following", "Following"), field_def("posts_count", "Posts"),
2010
- field_def("full_name", "Name"), field_def("bio", "Bio"),
2011
- field_def("verified", "Verified"), field_def("source", "Read via"),
2012
- field_def("approx", "Counts are approximate"),
2013
  # The link in bio — the paid rung's field, and commercially the most useful single string an
2014
  # influencer row can carry (it is the shop/affiliate destination).
2015
  field_def("external_url", "Link in bio", "url"),
2016
  # --- the rest of the vendor's Profiles schema, minus the PII and the comment text.
2017
  field_def("ig_id", "Instagram id"),
2018
  field_def("profile_url", "Profile", "url"),
2019
- field_def("avg_engagement", "Avg engagement"),
2020
- field_def("category", "Category"),
2021
- field_def("business_category", "Business category"),
2022
- field_def("is_business", "Business account"),
2023
- field_def("is_professional", "Professional account"),
2024
- field_def("is_private", "Private"),
2025
- field_def("highlights_count", "Highlights"),
2026
  field_def("bio_hashtags", "Bio hashtags"),
2027
  field_def("pronouns", "Pronouns"),
2028
  # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
@@ -2030,21 +2030,22 @@ SNAPSHOT_FIELDS = [
2030
  # leaving them out would be the "captured but unrecorded" half of the same loss the note at
2031
  # the top of this list is about.
2032
  field_def("profile_name", "Profile name"),
2033
- field_def("is_joined_recently", "Joined recently"),
2034
- field_def("has_channel", "Has channel"),
2035
  field_def("partner_id", "Partner id"),
2036
  field_def("external_url_title", "Link title"),
2037
  field_def("fbid", "Facebook id"),
2038
  field_def("related_accounts", "Related accounts"),
2039
  field_def("country_code", "Country"),
2040
  ]
2041
- POST_FIELDS = [
2042
- field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
2043
- field_def("posted_at", "Posted at"), field_def("type", "Type"),
2044
- field_def("caption", "Caption"), field_def("url", "URL", "url"),
 
2045
  # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous
2046
  # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
2047
- field_def("paid_partnership", "Paid partnership"),
2048
  field_def("partner", "Partner brand"),
2049
  field_def("hashtags", "Hashtags"),
2050
  field_def("alt_text", "Alt text"),
@@ -2126,11 +2127,11 @@ COMMENT_FIELDS = [
2126
  # What is left is the SHAPE of a thread — how many, how recent, how engaged — which is the
2127
  # part that informs an influencer decision without ingesting a stranger.
2128
  ]
2129
- POST_SNAPSHOT_FIELDS = [
2130
  field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
2131
  field_def("influencer_key", "Influencer"),
2132
- field_def("pulled_at", "Pulled at"), field_def("likes", "Likes"),
2133
- field_def("comments", "Comments"),
2134
  # Plays/views move independently of likes on video, so it is its own series rather than a
2135
  # thing to derive. Paid rung only; blank means not read.
2136
  field_def("views", "Views", "int"),
@@ -3571,11 +3572,70 @@ def _profile_schema_for(bound_key):
3571
  item = dict(field)
3572
  if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY:
3573
  item.pop("profile", None)
3574
- item.pop("pinned", None)
3575
  out.append(item)
3576
  return out
3577
 
3578
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3579
  def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
3580
  """Repair machine-owned IG field declarations in one coalesced store update.
3581
 
@@ -3585,6 +3645,10 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
3585
  unrelated/user fields, and removes only explicitly retired preset columns plus their cells.
3586
  """
3587
  drop_by_table = drop_by_table or {}
 
 
 
 
3588
  changed = False
3589
 
3590
  def _up(cur):
@@ -3607,12 +3671,28 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
3607
  fields.append(stored)
3608
  continue
3609
  repaired = dict(stored)
3610
- repaired.update(desired)
 
 
 
 
 
3611
  # These bags define behaviour, not decoration. If the desired field does not use
3612
  # one, a stale bag from an earlier type must not remain attached to it.
3613
- for bag in ("link", "rollup"):
3614
  if bag not in desired:
3615
  repaired.pop(bag, None)
 
 
 
 
 
 
 
 
 
 
 
3616
  if repaired != stored:
3617
  changed = True
3618
  fields.append(repaired)
@@ -3634,6 +3714,8 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
3634
  snapshot = ut_all(rt)
3635
  needs = False
3636
  for table_key, wanted_fields in wanted_by_table.items():
 
 
3637
  table = snapshot.get(table_key) or {}
3638
  by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []}
3639
  drops = set(drop_by_table.get(table_key) or ())
@@ -3642,7 +3724,24 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
3642
  break
3643
  for desired in wanted_fields:
3644
  stored = by_key.get(str(desired.get("key") or ""))
3645
- if stored is None or any(stored.get(k) != v for k, v in desired.items()):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3646
  needs = True
3647
  break
3648
  if needs:
@@ -3679,19 +3778,14 @@ def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", pr
3679
  keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag,
3680
  record_mode=AUTOMATION_RECORD_MODE, lock_fields=True)
3681
 
3682
- # `ut_ensure` stamps newly created fields with the first flow that introduced them. Do not
3683
- # put that tag into the reconciliation template: these canonical tables are shared by many
3684
- # Profile databases, and otherwise every run would rewrite their provenance from flow A to
3685
- # flow B and back again.
3686
- wanted = {key: [dict(f) for f in fields]
3687
- for key, (_label, fields) in graph.items()}
3688
- drops = {IG_SNAPSHOTS_TABLE: {"post_hashtags"}}
3689
  if profile_table and profile is not None and bound:
3690
  profile_fields = _profile_schema_for(bound)
3691
  ut_ensure(rt, profile_label or profile_table, profile_fields, username,
3692
  key=profile_table, flow_tag=flow_tag, lock_fields=True)
3693
- wanted[profile_table] = [dict(f) for f in profile_fields]
3694
- drops[profile_table] = {"posts", "post_hashtags"}
 
 
3695
  _reconcile_ig_graph_fields(rt, wanted, drops)
3696
  return keys
3697
 
@@ -3829,11 +3923,13 @@ def _merge_candidates(rows):
3829
  return out, merged
3830
 
3831
 
3832
- def migrate_ig_tables(rt, log=print, only=""):
3833
- """R3 + R4: bring every Instagram profile table onto the honest types and the new identity.
3834
-
3835
- Returns `{tables, retyped, converted, stamped, merged}` — counted rather than announced, so a
3836
- boot that migrated nothing says so instead of printing a reassuring line.
 
 
3837
 
3838
  `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides
3839
  the WRITE PATH, so a table is brought forward immediately before anything appends to it and no
@@ -3842,9 +3938,10 @@ def migrate_ig_tables(rt, log=print, only=""):
3842
  stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0,
3843
  # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from
3844
  # `stamped` because they answer different questions: that one is "how many ROWS got a
3845
- # platform", these are "how many TABLES changed shape".
3846
  "pinned": 0, "flagged": 0, "droppedName": 0,
3847
- "droppedRecentPosts": 0, "droppedPostHashtags": 0}
 
3848
  want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
3849
  # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red
3850
  # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
@@ -3889,7 +3986,7 @@ def migrate_ig_tables(rt, log=print, only=""):
3889
  want_flag = (flag_f is not None and other_profile is None
3890
  and not isinstance(flag_f.get("profile"), dict))
3891
  vestigial = _vestigial_name_field(fields, rows)
3892
- retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"}
3893
  if (not stale_keys and has_platform and not needs_stamp
3894
  and not want_pin and not want_flag and vestigial is None and not retired):
3895
  continue
@@ -3915,6 +4012,7 @@ def migrate_ig_tables(rt, log=print, only=""):
3915
  r.pop(key, None)
3916
  stats["droppedRecentPosts"] += int("posts" in retired)
3917
  stats["droppedPostHashtags"] += int("post_hashtags" in retired)
 
3918
 
3919
  for key in stale_keys:
3920
  conv = _MIGRATE_CONVERT.get(key)
@@ -3945,9 +4043,14 @@ def migrate_ig_tables(rt, log=print, only=""):
3945
  cur[table_key] = t
3946
  return cur
3947
 
3948
- rt.update(UT_STORE_KEY, _up)
3949
- if stats["tables"]:
3950
- log(f"[ig-migrate] {stats}")
 
 
 
 
 
3951
  return stats
3952
 
3953
 
@@ -4879,10 +4982,14 @@ def capture_rows(res, pulled):
4879
  "bio": _s(prof.get("bio"), 500),
4880
  "external_url": _s(prof.get("external_url"), 300),
4881
  }
4882
- for fd in SNAPSHOT_FIELDS:
4883
- k = fd["key"]
4884
- if k not in snap_row:
4885
- snap_row[k] = _s(prof.get(k), 400)
 
 
 
 
4886
  idents, metrics_rows = [], []
4887
  for p in (res or {}).get("posts") or []:
4888
  ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
 
2003
  #: (R11), both enforced in `BD_NEVER_MAP`.
2004
  #: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose
2005
  #: almost none of them, which is what the `source` column is for.
2006
+ SNAPSHOT_FIELDS = [
2007
+ field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"),
2008
+ field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"),
2009
+ field_def("following", "Following", "int"), field_def("posts_count", "Posts", "int"),
2010
+ field_def("full_name", "Name"), field_def("bio", "Bio"),
2011
+ field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"),
2012
+ field_def("approx", "Counts are approximate", "checkbox"),
2013
  # The link in bio — the paid rung's field, and commercially the most useful single string an
2014
  # influencer row can carry (it is the shop/affiliate destination).
2015
  field_def("external_url", "Link in bio", "url"),
2016
  # --- the rest of the vendor's Profiles schema, minus the PII and the comment text.
2017
  field_def("ig_id", "Instagram id"),
2018
  field_def("profile_url", "Profile", "url"),
2019
+ field_def("avg_engagement", "Avg engagement", "pct"),
2020
+ field_def("category", "Category"),
2021
+ field_def("business_category", "Business category"),
2022
+ field_def("is_business", "Business account", "checkbox"),
2023
+ field_def("is_professional", "Professional account", "checkbox"),
2024
+ field_def("is_private", "Private", "checkbox"),
2025
+ field_def("highlights_count", "Story highlights", "int"),
2026
  field_def("bio_hashtags", "Bio hashtags"),
2027
  field_def("pronouns", "Pronouns"),
2028
  # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
 
2030
  # leaving them out would be the "captured but unrecorded" half of the same loss the note at
2031
  # the top of this list is about.
2032
  field_def("profile_name", "Profile name"),
2033
+ field_def("is_joined_recently", "Joined recently", "checkbox"),
2034
+ field_def("has_channel", "Has channel", "checkbox"),
2035
  field_def("partner_id", "Partner id"),
2036
  field_def("external_url_title", "Link title"),
2037
  field_def("fbid", "Facebook id"),
2038
  field_def("related_accounts", "Related accounts"),
2039
  field_def("country_code", "Country"),
2040
  ]
2041
+ POST_FIELDS = [
2042
+ field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
2043
+ field_def("posted_at", "Posted at", "date"),
2044
+ field_def("type", "Type", "select", options=["image", "video", "carousel"]),
2045
+ field_def("caption", "Caption"), field_def("url", "URL", "url"),
2046
  # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous
2047
  # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
2048
+ field_def("paid_partnership", "Paid partnership", "checkbox"),
2049
  field_def("partner", "Partner brand"),
2050
  field_def("hashtags", "Hashtags"),
2051
  field_def("alt_text", "Alt text"),
 
2127
  # What is left is the SHAPE of a thread — how many, how recent, how engaged — which is the
2128
  # part that informs an influencer decision without ingesting a stranger.
2129
  ]
2130
+ POST_SNAPSHOT_FIELDS = [
2131
  field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
2132
  field_def("influencer_key", "Influencer"),
2133
+ field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"),
2134
+ field_def("comments", "Comments", "int"),
2135
  # Plays/views move independently of likes on video, so it is its own series rather than a
2136
  # thing to derive. Paid rung only; blank means not read.
2137
  field_def("views", "Views", "int"),
 
3572
  item = dict(field)
3573
  if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY:
3574
  item.pop("profile", None)
 
3575
  out.append(item)
3576
  return out
3577
 
3578
 
3579
+ def _locked_ig_field(field):
3580
+ """One canonical Instagram field with the immutable preset declaration attached."""
3581
+ item = dict(field)
3582
+ automation = dict(item.get("automation") or {})
3583
+ automation["preset"] = True
3584
+ item["automation"] = automation
3585
+ return item
3586
+
3587
+
3588
+ def _profile_binding(table):
3589
+ """The declared Profile identity field, falling back only to the canonical handle."""
3590
+ fields = list((table or {}).get("fields") or [])
3591
+ flagged = next((str(f.get("key") or "") for f in fields
3592
+ if isinstance(f.get("profile"), dict)), "")
3593
+ if flagged:
3594
+ return flagged
3595
+ return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else ""
3596
+
3597
+
3598
+ def _ig_schema_contract(rt, profile_tables=None):
3599
+ """The complete per-tenant Instagram graph contract, including every Profile backlink.
3600
+
3601
+ The fixed child tables are shared. Profile tables are discovered structurally, so an older
3602
+ user-named target and the built-in candidate database receive the same preset columns, Links,
3603
+ Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted.
3604
+ """
3605
+ tables = ut_all(rt)
3606
+ profile_keys = sorted(set(profile_tables if profile_tables is not None
3607
+ else _ig_profile_tables(rt)))
3608
+ backlinks, wanted, drops = [], {}, {
3609
+ IG_SNAPSHOTS_TABLE: {"post_hashtags"},
3610
+ }
3611
+ for table_key in profile_keys:
3612
+ table = tables.get(table_key) or {}
3613
+ bound = _profile_binding(table)
3614
+ if not bound:
3615
+ continue
3616
+ label = str(table.get("label") or table_key)
3617
+ backlinks.append(_profile_backlink_field(table_key, label, bound))
3618
+ wanted[table_key] = _profile_schema_for(bound)
3619
+ # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
3620
+ # to disagree, so it is retired by key just like the old nested Posts JSON.
3621
+ drops[table_key] = {"posts", "post_hashtags", "tracked"}
3622
+ wanted.update({
3623
+ IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
3624
+ IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
3625
+ IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks],
3626
+ IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks],
3627
+ })
3628
+ return wanted, drops
3629
+
3630
+
3631
+ # Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use
3632
+ # the scalar strings their renderers expect; engagement is the exception because Bright Data's
3633
+ # 0-1 fraction becomes this product's 0-100 percentage.
3634
+ _IG_RETYPE_CONVERTERS = {
3635
+ (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100,
3636
+ }
3637
+
3638
+
3639
  def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
3640
  """Repair machine-owned IG field declarations in one coalesced store update.
3641
 
 
3645
  unrelated/user fields, and removes only explicitly retired preset columns plus their cells.
3646
  """
3647
  drop_by_table = drop_by_table or {}
3648
+ wanted_by_table = {
3649
+ key: [_locked_ig_field(f) for f in fields]
3650
+ for key, fields in wanted_by_table.items()
3651
+ }
3652
  changed = False
3653
 
3654
  def _up(cur):
 
3671
  fields.append(stored)
3672
  continue
3673
  repaired = dict(stored)
3674
+ for dkey, value in desired.items():
3675
+ if dkey != "automation":
3676
+ repaired[dkey] = value
3677
+ automation = dict(stored.get("automation") or {})
3678
+ automation.update(desired.get("automation") or {})
3679
+ repaired["automation"] = automation
3680
  # These bags define behaviour, not decoration. If the desired field does not use
3681
  # one, a stale bag from an earlier type must not remain attached to it.
3682
+ for bag in ("link", "rollup", "profile", "pinned"):
3683
  if bag not in desired:
3684
  repaired.pop(bag, None)
3685
+ if desired.get("type") not in ("select", "multiselect"):
3686
+ repaired.pop("options", None)
3687
+ converter = _IG_RETYPE_CONVERTERS.get((table_key, key))
3688
+ if (converter and stored.get("type") in ("text", "", None)
3689
+ and repaired.get("type") != stored.get("type")):
3690
+ for row in (table.get("rows") or {}).values():
3691
+ if str(row.get(key) or "").strip():
3692
+ converted = converter(row.get(key))
3693
+ if converted:
3694
+ row[key] = converted
3695
+ changed = True
3696
  if repaired != stored:
3697
  changed = True
3698
  fields.append(repaired)
 
3714
  snapshot = ut_all(rt)
3715
  needs = False
3716
  for table_key, wanted_fields in wanted_by_table.items():
3717
+ if table_key not in snapshot:
3718
+ continue
3719
  table = snapshot.get(table_key) or {}
3720
  by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []}
3721
  drops = set(drop_by_table.get(table_key) or ())
 
3724
  break
3725
  for desired in wanted_fields:
3726
  stored = by_key.get(str(desired.get("key") or ""))
3727
+ mismatch = stored is None
3728
+ if stored is not None:
3729
+ for key, value in desired.items():
3730
+ if key == "automation":
3731
+ if any((stored.get("automation") or {}).get(k) != v
3732
+ for k, v in value.items()):
3733
+ mismatch = True
3734
+ break
3735
+ elif stored.get(key) != value:
3736
+ mismatch = True
3737
+ break
3738
+ if not mismatch:
3739
+ stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired)
3740
+ mismatch = any(k in stored for k in stale_bags)
3741
+ if (not mismatch and desired.get("type") not in ("select", "multiselect")
3742
+ and "options" in stored):
3743
+ mismatch = True
3744
+ if mismatch:
3745
  needs = True
3746
  break
3747
  if needs:
 
3778
  keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag,
3779
  record_mode=AUTOMATION_RECORD_MODE, lock_fields=True)
3780
 
 
 
 
 
 
 
 
3781
  if profile_table and profile is not None and bound:
3782
  profile_fields = _profile_schema_for(bound)
3783
  ut_ensure(rt, profile_label or profile_table, profile_fields, username,
3784
  key=profile_table, flow_tag=flow_tag, lock_fields=True)
3785
+ # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older
3786
+ # Profile database and IG candidates on the same universal schema while all of them point to
3787
+ # the same four children. Existing flow provenance is merged, never replaced.
3788
+ wanted, drops = _ig_schema_contract(rt)
3789
  _reconcile_ig_graph_fields(rt, wanted, drops)
3790
  return keys
3791
 
 
3923
  return out, merged
3924
 
3925
 
3926
+ def migrate_ig_tables(rt, log=print, only=""):
3927
+ """Bring every Instagram Profile and canonical child table onto one current contract.
3928
+
3929
+ Returns counted changes rather than a reassuring line. Profile cells are converted before
3930
+ their type declarations change; then the graph reconciliation adds any missing preset
3931
+ Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes
3932
+ only retired machine fields.
3933
 
3934
  `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides
3935
  the WRITE PATH, so a table is brought forward immediately before anything appends to it and no
 
3938
  stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0,
3939
  # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from
3940
  # `stamped` because they answer different questions: that one is "how many ROWS got a
3941
+ # platform", these are "how many TABLES changed shape".
3942
  "pinned": 0, "flagged": 0, "droppedName": 0,
3943
+ "droppedRecentPosts": 0, "droppedPostHashtags": 0,
3944
+ "droppedTracked": 0, "schema": 0}
3945
  want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
3946
  # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red
3947
  # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
 
3986
  want_flag = (flag_f is not None and other_profile is None
3987
  and not isinstance(flag_f.get("profile"), dict))
3988
  vestigial = _vestigial_name_field(fields, rows)
3989
+ retired = {f.get("key") for f in fields} & {"posts", "post_hashtags", "tracked"}
3990
  if (not stale_keys and has_platform and not needs_stamp
3991
  and not want_pin and not want_flag and vestigial is None and not retired):
3992
  continue
 
4012
  r.pop(key, None)
4013
  stats["droppedRecentPosts"] += int("posts" in retired)
4014
  stats["droppedPostHashtags"] += int("post_hashtags" in retired)
4015
+ stats["droppedTracked"] += int("tracked" in retired)
4016
 
4017
  for key in stale_keys:
4018
  conv = _MIGRATE_CONVERT.get(key)
 
4043
  cur[table_key] = t
4044
  return cur
4045
 
4046
+ rt.update(UT_STORE_KEY, _up)
4047
+ # The profile loop owns value conversions and deduplication. The schema pass owns the
4048
+ # declarations across ALL related databases and is safe only after those conversions, because
4049
+ # the stored old type is the idempotency key for unit-changing values such as engagement.
4050
+ wanted, drops = _ig_schema_contract(rt)
4051
+ stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
4052
+ if stats["tables"] or stats["schema"]:
4053
+ log(f"[ig-migrate] {stats}")
4054
  return stats
4055
 
4056
 
 
4982
  "bio": _s(prof.get("bio"), 500),
4983
  "external_url": _s(prof.get("external_url"), 300),
4984
  }
4985
+ for fd in SNAPSHOT_FIELDS:
4986
+ k = fd["key"]
4987
+ if k not in snap_row:
4988
+ value = prof.get(k)
4989
+ # Profile history declares the same percentage dialect as the current Profile row.
4990
+ # Converting only the latest projection left the append series at 0-1 while its field
4991
+ # now said pct (0-100), so the two views of one measurement disagreed by 100x.
4992
+ snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400)
4993
  idents, metrics_rows = [], []
4994
  for p in (res or {}).get("posts") or []:
4995
  ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),