"""Normalise heterogeneous ingest events into one internal representation. The provided sample_events.jsonl is a realistic *multi-source* stream — three event families with different field names: * entry / exit -> id_token, store_code, event_timestamp, gender_pred, age_pred, group_id, group_size, is_face_hidden * zone_entered/_exited -> track_id, store_id (ST####), event_time, zone_name, zone_type, is_revenue_zone, zone_hotspot_x/y, gender, age * queue_completed/ queue_abandoned -> queue_event_id, queue_join_ts, wait_seconds, queue_position_at_join, abandoned This module maps all of them (plus our own emitted schema) onto a single canonical event dict the rest of the API understands, so the held-out scoring set ingests cleanly regardless of which family it uses. """ from __future__ import annotations import hashlib import re from datetime import datetime, timezone from typing import Any # official (lowercase) + our (UPPER) event types -> canonical UPPER type _TYPE_MAP = { "entry": "ENTRY", "exit": "EXIT", "zone_entered": "ZONE_ENTER", "zone_exited": "ZONE_EXIT", "queue_completed": "QUEUE_COMPLETED", "queue_abandoned": "BILLING_QUEUE_ABANDON", "zone_dwell": "ZONE_DWELL", "reentry": "REENTRY", "billing_queue_join": "BILLING_QUEUE_JOIN", "billing_queue_abandon": "BILLING_QUEUE_ABANDON", } CANONICAL_TYPES = { "ENTRY", "EXIT", "ZONE_ENTER", "ZONE_EXIT", "ZONE_DWELL", "BILLING_QUEUE_JOIN", "BILLING_QUEUE_ABANDON", "QUEUE_COMPLETED", "REENTRY", } BILLING_ZONE_TYPES = {"BILLING", "QUEUE", "CHECKOUT"} def canon_store(raw: str) -> str: """Unify the two official store-id formats (store_1076 == ST1076) without mangling ids that don't match that pattern (e.g. STORE_BLR_002 stays).""" if not raw: return raw m = re.fullmatch(r"(?:store_|st)(\d+)", raw.strip(), re.IGNORECASE) return f"ST{m.group(1)}" if m else raw def _first(d: dict, *keys, default=None): for k in keys: v = d.get(k) if v is not None and v != "": return v return default def parse_ts(value: Any) -> datetime | None: if value is None: return None s = str(value).strip().replace("Z", "+00:00") try: dt = datetime.fromisoformat(s) except ValueError: return None return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) def _to_bool(v: Any) -> bool: if isinstance(v, bool): return v return str(v).strip().lower() in {"yes", "true", "1", "y"} def _synth_id(store, visitor, etype, ts, zone) -> str: raw = f"{store}|{visitor}|{etype}|{ts}|{zone}" return "ev_" + hashlib.sha1(raw.encode()).hexdigest()[:24] class NormalizeError(ValueError): pass def normalize_event(raw: dict) -> dict: """Return a canonical event dict, or raise NormalizeError for malformed input.""" if not isinstance(raw, dict): raise NormalizeError("event is not an object") raw_type = str(_first(raw, "event_type", default="")).strip() etype = _TYPE_MAP.get(raw_type.lower(), raw_type.upper()) if etype not in CANONICAL_TYPES: raise NormalizeError(f"unknown event_type {raw_type!r}") store_id = canon_store(str(_first(raw, "store_code", "store_id", default=""))) if not store_id: raise NormalizeError("missing store_code/store_id") # visitor identity: id_token (entry/exit) | track_id (zone/queue) | our visitor_id visitor = _first(raw, "visitor_id", "id_token") if visitor is None and raw.get("track_id") is not None: visitor = f"T{raw['track_id']}" if visitor is None: raise NormalizeError("missing visitor identity (id_token/track_id/visitor_id)") visitor = str(visitor) ts = parse_ts(_first(raw, "timestamp", "event_timestamp", "event_time", "queue_join_ts")) if ts is None: raise NormalizeError("missing/invalid timestamp") meta = raw.get("metadata") or {} zone_id = _first(raw, "zone_id") queue_depth = _first(raw, "queue_position_at_join", default=meta.get("queue_depth")) if queue_depth is not None: try: queue_depth = int(queue_depth) except (TypeError, ValueError): queue_depth = None event_id = _first(raw, "event_id", "queue_event_id") or _synth_id( store_id, visitor, etype, ts.isoformat(), zone_id) zone_type = (_first(raw, "zone_type") or "") if zone_type: zone_type = str(zone_type).upper() # QUEUE_COMPLETED implies billing even if zone_type missing if etype in ("QUEUE_COMPLETED", "BILLING_QUEUE_ABANDON", "BILLING_QUEUE_JOIN") and not zone_type: zone_type = "BILLING" return { "event_id": str(event_id), "store_id": store_id, "camera_id": str(_first(raw, "camera_id", default="")), "visitor_id": visitor, "event_type": etype, "ts": ts, "zone_id": str(zone_id) if zone_id is not None else None, "zone_name": _first(raw, "zone_name"), "zone_type": zone_type or None, "is_revenue_zone": _to_bool(raw["is_revenue_zone"]) if "is_revenue_zone" in raw else None, "dwell_ms": int(_first(raw, "dwell_ms", default=meta.get("dwell_ms") or 0) or 0), "is_staff": _to_bool(raw.get("is_staff", False)), "confidence": float(_first(raw, "confidence", default=1.0)), "queue_depth": queue_depth, "sku_zone": _first(raw, "sku_zone", default=meta.get("sku_zone")), "session_seq": int(_first(raw, "session_seq", default=meta.get("session_seq") or 0) or 0), # demographics / grouping (official extras) "gender": _first(raw, "gender", "gender_pred"), "age": _first(raw, "age", "age_pred"), "age_bucket": _first(raw, "age_bucket"), "group_id": _first(raw, "group_id"), "group_size": _first(raw, "group_size"), "wait_seconds": _first(raw, "wait_seconds"), "abandoned": _to_bool(raw["abandoned"]) if "abandoned" in raw else None, }