Hugging Face's logo

davidpere
/
nlp-ui-small

Text Classification
ONNX
Safetensors
custom
nlp-ui
field-role
layout
system-one
calibrated-decisions
classification
commercial-use
Model card Files
xet
 Community
nlp-ui-small / data.py
davidpere's picture
davidpere
Upload folder using huggingface_hub
3145730 verified
Raw History Blame Contribute Delete
11.7 kB
"""Corpus generators for nlp-ui model training.
Produces:
- field-role examples: (key, valueKind, uniqueness, cardinality,
meanLength, hasMoneySamples) -> role. Features are computed with the
exact TS ports so trained weights are consumable by the browser runtime.
- layout examples: 14 boolean flags -> LayoutKind.
The corpus mixes a curated, intended-behaviour seed (so shipped predictions
honour the SDK's contracts and tests) with synthetic perturbations of the
scalar/flag features (so priors and probabilities are smooth rather than
hand-tuned).
"""
from __future__ import annotations
import numpy as np
from features import field_role_features, LAYOUT_CLASSES, LAYOUT_FLAG_NAMES
# ---------------------------------------------------------------------------
# Field-role seeds: key -> (role, valueKind, expected scalar profile)
# ---------------------------------------------------------------------------
# Scalar profile: (uniqueness, cardinality, meanLength, hasMoney) base ranges.
_ROLE_PROFILE: dict[str, tuple[str, tuple[float, float], tuple[int, int], tuple[float, float], bool]] = {
"action": ("string", (0.85, 1.0), (28, 40), (18, 44), False),
"boolean": ("boolean", (0.0, 0.2), (2, 2), (4, 6), False),
"category": ("string", (0.04, 0.35), (2, 12), (6, 16), False),
"currency": ("number", (0.3, 0.9), (14, 36), (5, 9), True),
"date": ("date", (0.5, 1.0), (16, 40), (10, 24), False),
"description": ("string", (0.7, 1.0), (28, 40), (60, 400), False),
"email": ("email", (0.85, 1.0), (34, 40), (20, 28), False),
"identifier": ("string", (0.95, 1.0), (30, 40), (6, 40), False),
"image": ("image", (0.75, 1.0), (30, 40), (40, 60), False),
"location": ("string", (0.15, 0.85), (6, 34), (6, 30), False),
"metric": ("number", (0.2, 0.9), (8, 40), (2, 6), False),
"person": ("string", (0.35, 0.9), (12, 34), (8, 18), False),
"status": ("string", (0.05, 0.2), (2, 8), (6, 14), False),
"text": ("string", (0.6, 0.95), (30, 40), (30, 120), False),
"title": ("string", (0.55, 1.0), (24, 40), (8, 32), False),
"url": ("url", (0.85, 1.0), (34, 40), (24, 50), False),
}
# Curated keys. These anchor intended behaviour (incl. every key asserted in
# the SDK test suite). Format: key -> role.
FIELD_ROLE_SEED: dict[str, str] = {
# identifiers
"id": "identifier", "uuid": "identifier", "guid": "identifier",
"sku": "identifier", "isbn": "identifier", "ref": "identifier",
"orderRef": "identifier", "productCode": "identifier", "slug": "identifier",
# title / name
"name": "title", "title": "title", "productName": "title", "headline": "title",
"label": "title", "company": "title", "pair": "title", "symbol": "title",
"ticker": "title", "listing": "title", "station": "title", "site": "title",
"match": "title", "race": "title", "team": "title", "emitter": "title",
"meter": "title", "flight": "title", "customer": "title",
# description
"description": "description", "bio": "description", "summary": "description",
"notes": "description", "about": "description", "body": "description",
# category
"category": "category", "type": "category", "kind": "category",
"genre": "category", "department": "category", "tags": "category",
"party": "category", "league": "category", "propertyType": "category",
"sector": "category", "tariff": "category", "airline": "category",
# metric
"count": "metric", "quantity": "metric", "qty": "metric", "score": "metric",
"rating": "metric", "views": "metric", "stock": "metric", "weightKg": "metric",
"voteShare": "metric", "turnout": "metric", "margin": "metric",
"votesFor": "metric", "votesAgainst": "metric", "homeScore": "metric",
"awayScore": "metric", "minute": "metric", "tempC": "metric", "tempF": "metric",
"windKph": "metric", "humidity": "metric", "kwh": "metric", "tCO2e": "metric",
"emissions": "metric", "delayMin": "metric", "wins": "metric", "losses": "metric",
"points": "metric", "size": "metric", "customers": "metric", "beds": "metric",
"sqm": "metric", "year": "metric", "share": "metric",
# currency
"price": "currency", "unitPrice": "currency", "cost": "currency",
"fee": "currency", "salary": "currency", "total": "currency",
"balance": "currency", "revenue": "currency", "fare": "currency",
"amount": "currency", "value": "currency", "last": "currency", "bid": "currency",
"ask": "currency", "notional": "currency", "pnl": "currency",
# date
"createdAt": "date", "updated_at": "date", "timestamp": "date",
"publishedOn": "date", "dueDate": "date", "eta": "date",
"date": "date", "started": "date", "year": "metric",
# email / url / image
"email": "email", "contactEmail": "email", "url": "url", "website": "url",
"href": "url", "image": "image", "avatarUrl": "image", "thumbnail": "image",
# boolean
"active": "boolean", "enabled": "boolean", "featured": "boolean",
"isPublished": "boolean", "published": "boolean",
# status
"status": "status", "state": "status", "stage": "status",
"condition": "status",
# location
"city": "location", "country": "location", "address": "location",
"region": "location", "zipCode": "location", "location": "location",
"chamber": "location", "constituency": "location", "district": "location",
"venue": "location", "area": "location", "site": "title",
# person
"author": "person", "owner": "person", "assignee": "person",
"customer": "person", "firstName": "person", "lastName": "person",
"displayName": "person", "sponsor": "person", "candidate": "person",
"legislator": "person", "senator": "person",
# text / action
"message": "text", "comment": "text", "cta": "action", "actionUrl": "action",
}
def _variant_keys(key: str) -> list[str]:
"""Produce snake/camel/plural variants of a seed key (same role)."""
out = [key]
lower = key.lower()
if lower != key:
out.append(lower)
# crude snake_case of camelCase
s = lower.replace("_", " ")
parts = s.split()
if len(parts) >= 2:
out.append("_".join(parts))
# common plural for nouns that end with a consonant (ids/codes excluded)
if key in ("category", "sector", "tariff", "candidate", "emitter", "customer"):
out.append(key + "s")
return list(dict.fromkeys(out))
def _sample_scalars(rng: np.random.Generator, role: str):
kind, (lo_u, hi_u), (lo_c, hi_c), (lo_m, hi_m), money = _ROLE_PROFILE[role]
uniqueness = float(rng.uniform(lo_u, hi_u))
cardinality = int(rng.integers(lo_c, hi_c + 1))
mean_length = float(rng.uniform(lo_m, hi_m))
# mostly clean columns, sometimes gappy
null_rate = float(rng.uniform(0.15, 0.4)) if rng.random() < 0.12 else float(rng.uniform(0.0, 0.05))
# a few open-text columns arrive with mixed value kinds
if role in ("text", "category", "description") and rng.random() < 0.06:
kind = "mixed"
return kind, uniqueness, cardinality, mean_length, money, null_rate
def generate_field_role_corpus(seed: int = 7, per_seed: int = 6, perturb: int = 3) -> list:
"""Return list of feature examples labelled by role.
Each example: {"label", "features": ExtractedFeatures-shaped dict}.
"""
rng = np.random.default_rng(seed)
examples = []
for key, role in FIELD_ROLE_SEED.items():
if role is None:
continue
for variant in _variant_keys(key):
kind, u, c, m, money, null_rate = _sample_scalars(rng, role)
tokens, ngrams, flags = field_role_features(variant, kind, u, c, m, money, null_rate)
examples.append({"label": role, "features": {"tokens": tokens, "ngrams": ngrams, "flags": flags},
"meta": {"key": variant, "value_kind": kind, "uniqueness": u,
"cardinality": c, "mean_length": m, "null_rate": null_rate}})
# perturbed replicas: same key, jittered scalars
for _ in range(perturb):
kind, u, c, m, money, null_rate = _sample_scalars(rng, role)
tokens, ngrams, flags = field_role_features(variant, kind, u, c, m, money, null_rate)
examples.append({"label": role, "features": {"tokens": tokens, "ngrams": ngrams, "flags": flags},
"meta": {"key": variant, "value_kind": kind, "uniqueness": u,
"cardinality": c, "mean_length": m, "null_rate": null_rate}})
return examples
# ---------------------------------------------------------------------------
# Layout corpus: 14-flag vectors -> LayoutKind
# ---------------------------------------------------------------------------
# Prototype: flags + a per-flag flip probability used to synthesize variants.
_LayoutSeed = tuple[list[bool], float]
_LAYOUT_PROTOTYPES: dict[str, list[_LayoutSeed]] = {
"catalog": [
([1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0], 0.05), # image+currency+title+desc
([0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0], 0.05), # price+title+desc
([1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0], 0.05), # product cards + status
([0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0], 0.05), # housing listings: currency+status+metrics+desc+location
],
"dashboard": [
([0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0], 0.05), # many metrics + title
([0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0], 0.05), # sports: status+metrics+scorePair
([0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0], 0.05), # date+status+metrics
([0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0], 0.05), # scored leaders w/ people+location (polls)
],
"directory": [
([0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0], 0.05), # person+email+location
([1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0], 0.05), # person+image+location
([0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0], 0.05),
],
"log": [
([0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1], 0.05), # id+status+title (flights)
([0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1], 0.05), # id+date+status
([0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1], 0.05), # outage: id+date+status+metrics
([0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1], 0.05), # shipments: currency+person+desc
],
"ledger": [
([0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1], 0.05), # currency+date+status+id
([0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], 0.05),
([0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0], 0.05), # currency+person
],
}
def generate_layout_corpus(seed: int = 11, per_prototype: int = 300) -> list:
rng = np.random.default_rng(seed)
examples = []
for label, prototypes in _LAYOUT_PROTOTYPES.items():
for flags, flip in prototypes:
for _ in range(per_prototype):
vec = [bool(b if rng.random() > flip else 1 - b) for b in flags]
tokens = [name for name, on in zip(LAYOUT_FLAG_NAMES, vec) if on]
examples.append({
"label": label,
"features": {"tokens": tokens, "ngrams": [0] * 8, "flags": vec},
})
return examples
if __name__ == "__main__":
fr = generate_field_role_corpus()
lay = generate_layout_corpus()
from collections import Counter
print("field-role examples:", len(fr), Counter(e["label"] for e in fr))
print("layout examples:", len(lay), Counter(e["label"] for e in lay))