fix: require researched retail data contracts

demoprep_app/dataset/company_contract.py

@@ -0,0 +1,189 @@
+"""Validate researched company-specific dimensions before dataset generation."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from demoprep_app.scenario.extractor import ExtractedScenario
+
+
+@dataclass(slots=True)
+class CompanyDataContract:
+    dimensions: dict[str, list[str]] = field(default_factory=dict)
+    warnings: list[str] = field(default_factory=list)
+    blockers: list[str] = field(default_factory=list)
+    sources: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return not self.blockers
+
+
+VISIBLE_DIMENSIONS_BY_SCENARIO = {
+    "retail_sales": {"PRODUCTS", "CHANNELS", "STORES"},
+}
+
+GENERIC_PRODUCT_VALUES = {
+    "accessories",
+    "products",
+    "product",
+    "streaming",
+    "mobile app",
+    "app",
+    "online",
+    "retail",
+    "stores",
+    "store",
+    "service",
+    "services",
+}
+
+GENERIC_PRODUCT_PHRASE_TERMS = {
+    "analytics",
+    "demo",
+    "retail sales",
+    "sales retail",
+    "sales demo",
+    "use case",
+}
+
+APPAREL_DOMAIN_TERMS = {
+    "apparel",
+    "fashion",
+    "footwear",
+    "shoe",
+    "shoes",
+    "sneaker",
+    "sneakers",
+    "clothing",
+    "garment",
+    "sportswear",
+    "athletic",
+    "running",
+    "basketball",
+    "training",
+}
+
+APPAREL_INCOMPATIBLE_PRODUCT_TERMS = {
+    "soundbar",
+    "speaker",
+    "streaming",
+    "smart tv",
+    "television",
+    "remote",
+}
+
+
+def build_company_data_contract(
+    *,
+    company_name: str,
+    scenario_type: str,
+    extraction: ExtractedScenario | None,
+    use_case: str,
+    research_context: str | None = None,
+) -> CompanyDataContract:
+    """Return validated visible dimensions sourced from research/extraction.
+
+    The contract gate deliberately avoids company-specific production lists.
+    Its job is to decide whether extraction produced enough credible company
+    terms for generation to proceed.
+    """
+    dimensions = _clean_dimensions(extraction.dimensions if extraction else {})
+    contract = CompanyDataContract(dimensions=dimensions)
+    for name in VISIBLE_DIMENSIONS_BY_SCENARIO.get(scenario_type, set()):
+        if name in dimensions:
+            contract.sources[name] = extraction.source if extraction else "extraction"
+
+    if scenario_type == "retail_sales":
+        _validate_retail_sales_contract(
+            contract,
+            company_name=company_name,
+            use_case=use_case,
+            research_context=research_context,
+            extraction=extraction,
+        )
+    return contract
+
+
+def _validate_retail_sales_contract(
+    contract: CompanyDataContract,
+    *,
+    company_name: str,
+    use_case: str,
+    research_context: str | None,
+    extraction: ExtractedScenario | None,
+) -> None:
+    text = f"{company_name} {use_case} {research_context or ''} {extraction.business_domain if extraction else ''}".lower()
+    products = _specific_product_values(contract.dimensions.get("PRODUCTS", []))
+
+    if _is_apparel_domain(text):
+        incompatible = _values_with_terms(products, APPAREL_INCOMPATIBLE_PRODUCT_TERMS)
+        if incompatible:
+            contract.blockers.append(
+                "Retail/apparel product contract contains incompatible product terms from extraction: "
+                + ", ".join(incompatible[:8])
+            )
+
+    if len(products) < 4:
+        source = extraction.source if extraction else "none"
+        contract.blockers.append(
+            "Retail sales contract lacks enough researched company-specific products "
+            f"({len(products)} found from {source})."
+        )
+        return
+
+    if products != contract.dimensions.get("PRODUCTS", []):
+        contract.warnings.append("Removed generic product labels from researched product contract.")
+        contract.dimensions["PRODUCTS"] = products
+
+
+def _clean_dimensions(dimensions: dict[str, list[str]]) -> dict[str, list[str]]:
+    cleaned: dict[str, list[str]] = {}
+    for name, values in (dimensions or {}).items():
+        key = str(name).upper()
+        cleaned_values = []
+        seen = set()
+        for value in values or []:
+            text = str(value).strip()
+            if not text:
+                continue
+            text = re.sub(r"\s+", " ", text)
+            low = text.lower()
+            if low in seen:
+                continue
+            cleaned_values.append(text)
+            seen.add(low)
+        if cleaned_values:
+            cleaned[key] = cleaned_values[:36]
+    return cleaned
+
+
+def _specific_product_values(values: list[str]) -> list[str]:
+    specific = []
+    for value in values:
+        low = value.lower().strip()
+        if low in GENERIC_PRODUCT_VALUES:
+            continue
+        if any(term in low for term in GENERIC_PRODUCT_PHRASE_TERMS):
+            continue
+        if "sales" in low and not any(term in low for term in ("shoe", "apparel", "hoodie", "short", "jersey", "sneaker")):
+            continue
+        if re.search(r"\bvariant\s+\d{1,3}\b", low):
+            continue
+        if re.search(r"\b(?:product|item|sku)\s+\d{1,3}\b", low):
+            continue
+        specific.append(value)
+    return specific
+
+
+def _values_with_terms(values: list[str], terms: set[str]) -> list[str]:
+    return [
+        value
+        for value in values
+        if any(re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", value.lower()) for term in terms)
+    ]
+
+
+def _is_apparel_domain(text: str) -> bool:
+    return any(re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", text) for term in APPAREL_DOMAIN_TERMS)

demoprep_app/dataset/generators/retail_sales.py

@@ -361,9 +361,53 @@ def _product_tuples_from_values(values: list[str]) -> list[tuple[str, str, float
     if not base:
         return []
     expanded = list(base)
+    descriptors = [
+        "Core",
+        "Pro",
+        "Max",
+        "Mini",
+        "Plus",
+        "Ultra",
+        "Studio",
+        "Essential",
+        "Premium",
+        "Signature",
+        "Performance",
+        "Lifestyle",
+        "Home",
+        "Portable",
+        "Wireless",
+        "Connected",
+        "Outdoor",
+        "Theater",
+        "Compact",
+        "Flagship",
+        "Select",
+        "Elite",
+        "Limited",
+        "Smart",
+        "Voice Enabled",
+        "Surround",
+        "Travel",
+        "Everyday",
+        "Active",
+        "Classic",
+        "Modern",
+        "Heritage",
+        "Seasonal",
+        "Retail Exclusive",
+        "Direct",
+        "Partner",
+    ]
+    descriptor_idx = 0
     while len(expanded) < 36:
         seed = base[len(expanded) % len(base)]
-        expanded.append(f"{seed} Variant {len(expanded) + 1:02d}")
+        descriptor = descriptors[descriptor_idx % len(descriptors)]
+        descriptor_idx += 1
+        candidate = f"{descriptor} {seed}"
+        if candidate.lower() in {value.lower() for value in expanded}:
+            candidate = f"{seed} {descriptor}"
+        expanded.append(candidate)
     return [(_product_name(value, idx), _product_category(value), _product_price(value, idx)) for idx, value in enumerate(expanded[:36])]
 
 

demoprep_app/pipeline/dataset_first.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import hashlib
 from dataclasses import dataclass
 
+from demoprep_app.dataset.company_contract import build_company_data_contract
 from demoprep_app.dataset.contracts import DatasetBundle
 from demoprep_app.dataset.generators.router import DatasetGeneratorRouter
 from demoprep_app.ddl import DatasetDdlCompiler
@@ -189,7 +190,16 @@ def build_dataset_first_demo(
             metadata={"seed": seed},
         )
     elif scenario_type == "retail_sales":
-        extracted_dimensions = extraction.dimensions if extraction else {}
+        company_contract = build_company_data_contract(
+            company_name=company_name,
+            scenario_type=scenario_type,
+            extraction=extraction,
+            use_case=use_case,
+            research_context=research_context,
+        )
+        if not company_contract.ok:
+            raise ValueError("Company data contract failed before dataset generation:\n" + "\n".join(company_contract.blockers))
+        extracted_dimensions = company_contract.dimensions
         dimensions = [
             DimensionSpec(name=name, semantic_role=name.lower(), values=values)
             for name, values in extracted_dimensions.items()
@@ -225,6 +235,8 @@ def build_dataset_first_demo(
                 "business_domain": extraction.business_domain if extraction else "",
                 "extracted_dimensions": extracted_dimensions,
                 "extracted_measures": extraction.measures if extraction else [],
+                "company_contract_sources": company_contract.sources,
+                "company_contract_warnings": company_contract.warnings,
             },
         )
     else:

demoprep_app/scenario/extractor.py

@@ -212,6 +212,20 @@ def _infer_from_text(text: str) -> str | None:
 def _context_dimension_values(name: str, text: str) -> list[str]:
     phrases = _extract_named_phrases(text)
     product_terms = (
+        "running shoes",
+        "basketball shoes",
+        "training shoes",
+        "soccer boots",
+        "sneakers",
+        "footwear",
+        "apparel",
+        "performance apparel",
+        "dri-fit",
+        "hoodies",
+        "shorts",
+        "jerseys",
+        "sportswear",
+        "equipment",
         "smart tv",
         "smart tvs",
         "tv",
@@ -220,6 +234,10 @@ def _context_dimension_values(name: str, text: str) -> list[str]:
         "televisions",
         "soundbar",
         "soundbars",
+        "speaker",
+        "speakers",
+        "portable audio",
+        "home audio",
         "home theater",
         "watchfree",
         "sports zone",

tests/test_dataset_first_builders.py

@@ -5,8 +5,10 @@ dataset-first builders produce coherent table bundles and relationship-bearing
 DDL before the slower browser quality suite runs against the test Space.
 """
 
+from demoprep_app.dataset.generators.retail_sales import RetailSalesDatasetGenerator
 from demoprep_app.pipeline.dataset_first import build_dataset_first_demo, infer_scenario_type
 from demoprep_app.dataset.quality import validate_dataset_quality
+from demoprep_app.scenario.contract import DimensionSpec, ScenarioContract
 from demoprep_app.scenario.families import SCENARIO_FAMILIES
 
 
@@ -49,6 +51,13 @@ def test_saas_sales_specialized_builder_has_expected_shape():
 def test_all_scenario_families_generate_relationship_ddl():
     failures = []
     for scenario_type in SCENARIO_FAMILIES:
+        research_context = ""
+        if scenario_type == "retail_sales":
+            research_context = (
+                "Retail sales products include running shoes, basketball shoes, training apparel, "
+                "hoodies, shorts, jerseys, sneakers, backpacks, and performance socks. "
+                "Channels include retail stores, online, mobile app, outlet, and retail partners."
+            )
         build = build_dataset_first_demo(
             company_name="Acme",
             company_url="https://example.com",
@@ -56,6 +65,7 @@ def test_all_scenario_families_generate_relationship_ddl():
             vertical="",
             function="",
             row_count_guidance=25,
+            research_context=research_context,
         )
         if build is None:
             failures.append(f"{scenario_type}: no build")
@@ -222,6 +232,10 @@ def test_retail_sales_store_names_are_demo_safe_and_not_numbered_placeholders():
         vertical="Retail & Consumer Goods",
         function="Sales",
         row_count_guidance=100,
+        research_context=(
+            "Nike retail sales products include running shoes, basketball shoes, training shoes, "
+            "Dri-FIT performance apparel, hoodies, shorts, jerseys, sneakers, backpacks, and performance socks."
+        ),
     )
 
     assert build is not None
@@ -235,6 +249,93 @@ def test_retail_sales_store_names_are_demo_safe_and_not_numbered_placeholders():
     assert validate_dataset_quality(build.dataset).ok
 
 
+def test_retail_product_expansion_uses_named_variants_not_numbered_placeholders():
+    scenario = ScenarioContract(
+        company_name="Sonos",
+        company_url="https://sonos.com",
+        use_case="Consumer Electronics Sales",
+        scenario_type="retail_sales",
+        fact_grain="store-product-channel-day",
+        dimensions=[
+            DimensionSpec(name="PRODUCTS", semantic_role="product", values=["Soundbars", "Speakers", "Accessories", "Streaming"]),
+        ],
+        metadata={"seed": 123},
+    )
+
+    dataset = RetailSalesDatasetGenerator().generate(scenario, row_count=100)
+    values = [row["PRODUCT_NAME"] for row in dataset.table_map()["PRODUCTS"].rows]
+
+    assert len(values) == 36
+    assert not any("Variant " in value for value in values)
+    assert not any(value.endswith(tuple(str(i).zfill(2) for i in range(1, 40))) for value in values)
+    assert validate_dataset_quality(dataset).ok
+
+
+def test_retail_contract_uses_researched_products_before_generation():
+    build = build_dataset_first_demo(
+        company_name="Nike",
+        company_url="https://nike.com",
+        use_case="Retail Sales",
+        vertical="Retail & Consumer Goods",
+        function="Sales",
+        row_count_guidance=100,
+        research_context=(
+            "Nike retail sales demo products include running shoes, basketball shoes, "
+            "training shoes, soccer boots, Dri-FIT performance apparel, hoodies, shorts, and jerseys. "
+            "Channels include Nike stores, Nike.com, mobile app, outlet, and retail partners."
+        ),
+    )
+
+    assert build is not None
+    products = [row["PRODUCT_NAME"] for row in build.dataset.table_map()["PRODUCTS"].rows]
+
+    assert "Running Shoes" in products
+    assert "Basketball Shoes" in products
+    assert "Dri-FIT" in products
+    assert build.scenario.metadata["company_contract_sources"]["PRODUCTS"] == "fallback"
+    assert validate_dataset_quality(build.dataset).ok
+
+
+def test_retail_contract_blocks_incompatible_research_terms_before_generation():
+    try:
+        build_dataset_first_demo(
+            company_name="Nike",
+            company_url="https://nike.com",
+            use_case="Fashion/Apparel Sales",
+            vertical="Retail & Consumer Goods",
+            function="Sales",
+            row_count_guidance=100,
+            research_context="Bad extracted terms should not drive products: soundbars, streaming, smart tv.",
+        )
+    except ValueError as exc:
+        message = str(exc)
+    else:
+        raise AssertionError("Expected company contract failure")
+
+    assert "Company data contract failed" in message
+    assert "incompatible product terms" in message or "lacks enough researched" in message
+
+
+def test_sonos_retail_contract_allows_consumer_electronics_products():
+    build = build_dataset_first_demo(
+        company_name="Sonos",
+        company_url="https://sonos.com",
+        use_case="Consumer Electronics Sales",
+        vertical="Retail & Consumer Goods",
+        function="Sales",
+        row_count_guidance=100,
+        research_context="Sonos sells speakers, soundbars, portable audio, home theater, and accessories.",
+    )
+
+    assert build is not None
+    products = [row["PRODUCT_NAME"] for row in build.dataset.table_map()["PRODUCTS"].rows]
+
+    assert "Soundbars" in products
+    assert "Speakers" in products
+    assert build.scenario.metadata["company_contract_sources"]["PRODUCTS"] == "fallback"
+    assert validate_dataset_quality(build.dataset).ok
+
+
 def test_transportation_function_routing_keeps_sales_and_finance_distinct():
     cases = [
         ("Shipping", "Sales", "shipping_sales"),