"""Corpus generators for nlp-ui model training. Produces: - field-role examples: (key, valueKind, uniqueness, cardinality, meanLength, hasMoneySamples) -> role. Features are computed with the exact TS ports so trained weights are consumable by the browser runtime. - layout examples: 14 boolean flags -> LayoutKind. The corpus mixes a curated, intended-behaviour seed (so shipped predictions honour the SDK's contracts and tests) with synthetic perturbations of the scalar/flag features (so priors and probabilities are smooth rather than hand-tuned). """ from __future__ import annotations import numpy as np from features import field_role_features, LAYOUT_CLASSES, LAYOUT_FLAG_NAMES # --------------------------------------------------------------------------- # Field-role seeds: key -> (role, valueKind, expected scalar profile) # --------------------------------------------------------------------------- # Scalar profile: (uniqueness, cardinality, meanLength, hasMoney) base ranges. _ROLE_PROFILE: dict[str, tuple[str, tuple[float, float], tuple[int, int], tuple[float, float], bool]] = { "action": ("string", (0.85, 1.0), (28, 40), (18, 44), False), "boolean": ("boolean", (0.0, 0.2), (2, 2), (4, 6), False), "category": ("string", (0.04, 0.35), (2, 12), (6, 16), False), "currency": ("number", (0.3, 0.9), (14, 36), (5, 9), True), "date": ("date", (0.5, 1.0), (16, 40), (10, 24), False), "description": ("string", (0.7, 1.0), (28, 40), (60, 400), False), "email": ("email", (0.85, 1.0), (34, 40), (20, 28), False), "identifier": ("string", (0.95, 1.0), (30, 40), (6, 40), False), "image": ("image", (0.75, 1.0), (30, 40), (40, 60), False), "location": ("string", (0.15, 0.85), (6, 34), (6, 30), False), "metric": ("number", (0.2, 0.9), (8, 40), (2, 6), False), "person": ("string", (0.35, 0.9), (12, 34), (8, 18), False), "status": ("string", (0.05, 0.2), (2, 8), (6, 14), False), "text": ("string", (0.6, 0.95), (30, 40), (30, 120), False), "title": ("string", (0.55, 1.0), (24, 40), (8, 32), False), "url": ("url", (0.85, 1.0), (34, 40), (24, 50), False), } # Curated keys. These anchor intended behaviour (incl. every key asserted in # the SDK test suite). Format: key -> role. FIELD_ROLE_SEED: dict[str, str] = { # identifiers "id": "identifier", "uuid": "identifier", "guid": "identifier", "sku": "identifier", "isbn": "identifier", "ref": "identifier", "orderRef": "identifier", "productCode": "identifier", "slug": "identifier", # title / name "name": "title", "title": "title", "productName": "title", "headline": "title", "label": "title", "company": "title", "pair": "title", "symbol": "title", "ticker": "title", "listing": "title", "station": "title", "site": "title", "match": "title", "race": "title", "team": "title", "emitter": "title", "meter": "title", "flight": "title", "customer": "title", # description "description": "description", "bio": "description", "summary": "description", "notes": "description", "about": "description", "body": "description", # category "category": "category", "type": "category", "kind": "category", "genre": "category", "department": "category", "tags": "category", "party": "category", "league": "category", "propertyType": "category", "sector": "category", "tariff": "category", "airline": "category", # metric "count": "metric", "quantity": "metric", "qty": "metric", "score": "metric", "rating": "metric", "views": "metric", "stock": "metric", "weightKg": "metric", "voteShare": "metric", "turnout": "metric", "margin": "metric", "votesFor": "metric", "votesAgainst": "metric", "homeScore": "metric", "awayScore": "metric", "minute": "metric", "tempC": "metric", "tempF": "metric", "windKph": "metric", "humidity": "metric", "kwh": "metric", "tCO2e": "metric", "emissions": "metric", "delayMin": "metric", "wins": "metric", "losses": "metric", "points": "metric", "size": "metric", "customers": "metric", "beds": "metric", "sqm": "metric", "year": "metric", "share": "metric", # currency "price": "currency", "unitPrice": "currency", "cost": "currency", "fee": "currency", "salary": "currency", "total": "currency", "balance": "currency", "revenue": "currency", "fare": "currency", "amount": "currency", "value": "currency", "last": "currency", "bid": "currency", "ask": "currency", "notional": "currency", "pnl": "currency", # date "createdAt": "date", "updated_at": "date", "timestamp": "date", "publishedOn": "date", "dueDate": "date", "eta": "date", "date": "date", "started": "date", "year": "metric", # email / url / image "email": "email", "contactEmail": "email", "url": "url", "website": "url", "href": "url", "image": "image", "avatarUrl": "image", "thumbnail": "image", # boolean "active": "boolean", "enabled": "boolean", "featured": "boolean", "isPublished": "boolean", "published": "boolean", # status "status": "status", "state": "status", "stage": "status", "condition": "status", # location "city": "location", "country": "location", "address": "location", "region": "location", "zipCode": "location", "location": "location", "chamber": "location", "constituency": "location", "district": "location", "venue": "location", "area": "location", "site": "title", # person "author": "person", "owner": "person", "assignee": "person", "customer": "person", "firstName": "person", "lastName": "person", "displayName": "person", "sponsor": "person", "candidate": "person", "legislator": "person", "senator": "person", # text / action "message": "text", "comment": "text", "cta": "action", "actionUrl": "action", } def _variant_keys(key: str) -> list[str]: """Produce snake/camel/plural variants of a seed key (same role).""" out = [key] lower = key.lower() if lower != key: out.append(lower) # crude snake_case of camelCase s = lower.replace("_", " ") parts = s.split() if len(parts) >= 2: out.append("_".join(parts)) # common plural for nouns that end with a consonant (ids/codes excluded) if key in ("category", "sector", "tariff", "candidate", "emitter", "customer"): out.append(key + "s") return list(dict.fromkeys(out)) def _sample_scalars(rng: np.random.Generator, role: str): kind, (lo_u, hi_u), (lo_c, hi_c), (lo_m, hi_m), money = _ROLE_PROFILE[role] uniqueness = float(rng.uniform(lo_u, hi_u)) cardinality = int(rng.integers(lo_c, hi_c + 1)) mean_length = float(rng.uniform(lo_m, hi_m)) # mostly clean columns, sometimes gappy null_rate = float(rng.uniform(0.15, 0.4)) if rng.random() < 0.12 else float(rng.uniform(0.0, 0.05)) # a few open-text columns arrive with mixed value kinds if role in ("text", "category", "description") and rng.random() < 0.06: kind = "mixed" return kind, uniqueness, cardinality, mean_length, money, null_rate def generate_field_role_corpus(seed: int = 7, per_seed: int = 6, perturb: int = 3) -> list: """Return list of feature examples labelled by role. Each example: {"label", "features": ExtractedFeatures-shaped dict}. """ rng = np.random.default_rng(seed) examples = [] for key, role in FIELD_ROLE_SEED.items(): if role is None: continue for variant in _variant_keys(key): kind, u, c, m, money, null_rate = _sample_scalars(rng, role) tokens, ngrams, flags = field_role_features(variant, kind, u, c, m, money, null_rate) examples.append({"label": role, "features": {"tokens": tokens, "ngrams": ngrams, "flags": flags}, "meta": {"key": variant, "value_kind": kind, "uniqueness": u, "cardinality": c, "mean_length": m, "null_rate": null_rate}}) # perturbed replicas: same key, jittered scalars for _ in range(perturb): kind, u, c, m, money, null_rate = _sample_scalars(rng, role) tokens, ngrams, flags = field_role_features(variant, kind, u, c, m, money, null_rate) examples.append({"label": role, "features": {"tokens": tokens, "ngrams": ngrams, "flags": flags}, "meta": {"key": variant, "value_kind": kind, "uniqueness": u, "cardinality": c, "mean_length": m, "null_rate": null_rate}}) return examples # --------------------------------------------------------------------------- # Layout corpus: 14-flag vectors -> LayoutKind # --------------------------------------------------------------------------- # Prototype: flags + a per-flag flip probability used to synthesize variants. _LayoutSeed = tuple[list[bool], float] _LAYOUT_PROTOTYPES: dict[str, list[_LayoutSeed]] = { "catalog": [ ([1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0], 0.05), # image+currency+title+desc ([0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0], 0.05), # price+title+desc ([1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0], 0.05), # product cards + status ([0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0], 0.05), # housing listings: currency+status+metrics+desc+location ], "dashboard": [ ([0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0], 0.05), # many metrics + title ([0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0], 0.05), # sports: status+metrics+scorePair ([0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0], 0.05), # date+status+metrics ([0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0], 0.05), # scored leaders w/ people+location (polls) ], "directory": [ ([0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0], 0.05), # person+email+location ([1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0], 0.05), # person+image+location ([0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0], 0.05), ], "log": [ ([0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1], 0.05), # id+status+title (flights) ([0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1], 0.05), # id+date+status ([0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1], 0.05), # outage: id+date+status+metrics ([0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1], 0.05), # shipments: currency+person+desc ], "ledger": [ ([0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1], 0.05), # currency+date+status+id ([0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], 0.05), ([0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0], 0.05), # currency+person ], } def generate_layout_corpus(seed: int = 11, per_prototype: int = 300) -> list: rng = np.random.default_rng(seed) examples = [] for label, prototypes in _LAYOUT_PROTOTYPES.items(): for flags, flip in prototypes: for _ in range(per_prototype): vec = [bool(b if rng.random() > flip else 1 - b) for b in flags] tokens = [name for name, on in zip(LAYOUT_FLAG_NAMES, vec) if on] examples.append({ "label": label, "features": {"tokens": tokens, "ngrams": [0] * 8, "flags": vec}, }) return examples if __name__ == "__main__": fr = generate_field_role_corpus() lay = generate_layout_corpus() from collections import Counter print("field-role examples:", len(fr), Counter(e["label"] for e in fr)) print("layout examples:", len(lay), Counter(e["label"] for e in lay))