Deploy public authenticated AutoML gateway

gateway_app.py

@@ -12,11 +12,11 @@ import tempfile
 import time
 import uuid
 from collections import Counter
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError
 from dataclasses import dataclass, replace
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Mapping
+from typing import Any, Callable, Mapping
 
 import gradio as gr
 import pandas as pd
@@ -83,6 +83,7 @@ from gateway_contract import (
     check_upload_size,
     inspect_package_manifest,
     load_gateway_catalog,
+    model_leaderboard_markdown,
     predictions_frame,
 )
 from gateway_jobs import ROUTED_JOB_ADMISSION, JobRejected
@@ -1016,6 +1017,36 @@ def _context_item(label: str, value: str) -> str:
     )
 
 
+def run_progress_html(status_text: str, *, duration: str = "") -> str:
+    """Show honest indeterminate progress until the backend returns a result."""
+
+    state = _run_state(status_text)
+    if state == "complete":
+        progress = '<progress value="1" max="1" aria-label="Run completed" style="width:100%;height:10px"></progress>'
+        detail = "Results are ready in the Runs tab."
+    elif state in {"failed", "cancelled"}:
+        progress = ""
+        detail = "Review the run status before starting another run."
+    else:
+        progress = '<progress max="1" aria-label="Waiting for the training result" style="width:100%;height:10px"></progress>'
+        detail = "Waiting for the private backend to return results."
+    elapsed = f" Elapsed: {html.escape(duration)}." if duration else ""
+    return (
+        '<section class="run-progress-feedback" aria-label="Run progress">'
+        f'<p role="status">{html.escape(str(status_text))}</p>{progress}'
+        f'<p>{detail}{elapsed}</p></section>'
+    )
+
+
+def _run_progress_update(context_html: str):
+    match = re.search(
+        r'<section class="run-progress-feedback"[^>]*>.*?</section>',
+        str(context_html or ""),
+        re.DOTALL,
+    )
+    return gr.update(value=match.group(0) if match else "", visible=bool(match))
+
+
 def run_context_html(
     context: RunContext,
     status_text: str,
@@ -1066,6 +1097,7 @@ def run_context_html(
         '<div class="run-alias-row"><span class="run-context-label">Run alias</span>'
         f'<code class="run-alias">{alias}</code>'
         f'<button type="button" class="run-id-copy" data-copy-value="{copy_value}">Copy</button></div>'
+        f'{run_progress_html(status_text, duration=duration)}'
         f'<div class="run-context-list">{visible}</div>{technical}</section>'
     )
 
@@ -1152,9 +1184,13 @@ def format_metrics_markdown(text: str) -> str:
 
 
 def candidate_comparison_markdown(summary: str) -> str:
+    return _summary_section(summary, "Candidate comparison")
+
+
+def _summary_section(summary: str, heading: str) -> str:
     lines = str(summary or "").splitlines()
     start = next(
-        (index for index, line in enumerate(lines) if re.match(r"^#{1,6}\s+Candidate comparison\s*$", line.strip(), re.I)),
+        (index for index, line in enumerate(lines) if re.match(r"^#{1,6}\s+" + re.escape(heading) + r"\s*$", line.strip(), re.I)),
         None,
     )
     if start is None:
@@ -1539,6 +1575,14 @@ def render_training_result(
         str(envelope.get("summary") or "")
     )
     candidate_comparison = candidate_comparison_markdown(summary)
+    leaderboard = model_leaderboard_markdown(envelope.get("model_leaderboard"))
+    leaderboard = leaderboard or _summary_section(summary, "AutoGluon model leaderboard")
+    if leaderboard:
+        candidate_comparison = "\n\n".join(
+            section for section in (leaderboard, candidate_comparison) if section
+        )
+    if _run_state(status_text) != "complete":
+        status_text = "Training complete. Your results are ready for review."
     download = None
     if package_path:
         try:
@@ -1599,6 +1643,53 @@ def _route_label(profile: str) -> str:
     return f"Private {str(profile or '').upper()}"
 
 
+RUN_HEARTBEAT_SECONDS = 2.0
+
+
+def _wait_for_backend(
+    operation: Callable[[], Any],
+    *,
+    context: RunContext,
+    status_text: str,
+    started: float,
+):
+    """Submit once and stream elapsed-time feedback while a blocking call runs."""
+
+    delivered = False
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(operation)
+        try:
+            while True:
+                try:
+                    result = future.result(timeout=RUN_HEARTBEAT_SECONDS)
+                except FutureTimeoutError:
+                    if future.done():
+                        # A TimeoutError raised by the operation is a failure,
+                        # rather than an instruction to submit it again.
+                        result = future.result()
+                    else:
+                        yield _training_outputs(
+                            status_text,
+                            context_html=run_context_html(
+                                context, status_text,
+                                duration=_format_duration(time.monotonic() - started),
+                            ),
+                        )
+                        continue
+                delivered = True
+                return result
+        finally:
+            if not delivered:
+                # Closing a browser's generator must not orphan downloaded files
+                # or cause a second submission of the running backend request.
+                try:
+                    result = future.result()
+                    backend_result = getattr(result, "backend_result", result)
+                    _cleanup_files(getattr(backend_result, "cleanup_files", ()))
+                except Exception:
+                    pass
+
+
 def run_existing_remote(
     payload: Mapping[str, Any],
     session_token: str | None,
@@ -1649,11 +1740,13 @@ def run_existing_remote(
             waking_status,
             context_html=run_context_html(context, waking_status),
         )
-        execution = PRODUCTION_SCHEDULER.run_existing(
-            username,
-            request_payload,
-            display_name=_display_name(request_payload),
-            idempotency_key=context.idempotency_key,
+        execution = yield from _wait_for_backend(
+            lambda: PRODUCTION_SCHEDULER.run_existing(
+                username, request_payload,
+                display_name=_display_name(request_payload),
+                idempotency_key=context.idempotency_key,
+            ),
+            context=context, status_text=waking_status, started=started,
         )
         result = execution.backend_result
         saved_model_record = execution.saved_model_record
@@ -1737,28 +1830,29 @@ def run_uploaded_remote(
             queued_status,
             context_html=run_context_html(context, queued_status),
         )
-        with ROUTED_JOB_ADMISSION.lease(
-            decision.profile, username, timeout=None
-        ):
-            waking_status = f"Processing the private {profile_label} request (startup, training, and packaging)…"
-            yield _training_outputs(
-                waking_status,
-                context_html=run_context_html(context, waking_status),
-            )
-            client = GatewayBackendClient.from_env(decision.profile)
-            result = client.run_uploaded(
-                request_payload, train_path, test_path or None
-            )
-            prune_gateway_egress()
-            yield render_training_result(
-                username,
-                result.envelope,
-                result.package_path,
-                _display_name(request_payload),
-                context=context,
-                duration_seconds=time.monotonic() - started,
-                provider="local",
-            )
+        waking_status = f"Processing the private {profile_label} request (startup, training, and packaging)…"
+        yield _training_outputs(
+            waking_status,
+            context_html=run_context_html(context, waking_status),
+        )
+
+        def submit_uploaded():
+            with ROUTED_JOB_ADMISSION.lease(decision.profile, username, timeout=None):
+                client = GatewayBackendClient.from_env(decision.profile)
+                return client.run_uploaded(request_payload, train_path, test_path or None)
+
+        result = yield from _wait_for_backend(
+            submit_uploaded,
+            context=context, status_text=waking_status, started=started,
+        )
+        prune_gateway_egress()
+        yield render_training_result(
+            username, result.envelope, result.package_path,
+            _display_name(request_payload),
+            context=context,
+            duration_seconds=time.monotonic() - started,
+            provider="local",
+        )
     except BackendUnavailable as exc:
         if decision is not None and decision.profile == "cpu":
             yield _training_error(
@@ -2629,6 +2723,38 @@ def _guided_run_button_ready():
     return gr.update(value="Run selected pipeline", interactive=True)
 
 
+def _guided_run_button_finishing():
+    """Keep the run control disabled until results and navigation are delivered."""
+
+    return gr.update(value="Preparing results…", interactive=False)
+
+
+def _guided_run_complete_ui(
+    status_text: str, session_token: str | None, request: gr.Request | None = None
+):
+    if not auth_storage.current_username(request, session_token):
+        return gr.update(), gr.update(), gr.update(value="Run selected pipeline", interactive=False)
+    if _run_state(status_text) == "complete":
+        gr.Info("Training finished. Your results are ready in Runs → Latest run.", duration=8)
+        return gr.update(selected="runs"), gr.update(selected="latest"), _guided_run_button_ready()
+    gr.Warning("The run could not be completed. Review the visible run status.", duration=8)
+    return gr.update(), gr.update(), _guided_run_button_ready()
+
+
+def _guided_parallel_complete_ui(
+    first_status: str, second_status: str, session_token: str | None,
+    request: gr.Request | None = None,
+):
+    if not auth_storage.current_username(request, session_token):
+        return gr.update(), gr.update()
+    completed = sum(_run_state(status) == "complete" for status in (first_status, second_status))
+    if completed == 2:
+        gr.Info("Both variants finished. Review their results in Runs.", duration=8)
+    else:
+        gr.Warning(f"{completed} of 2 variants completed. Review each variant's run status.", duration=8)
+    return gr.update(selected="runs"), gr.update(selected="variant-one")
+
+
 def _workbench_state_for_variant(variant: Any) -> WorkbenchState:
     approved = dict(variant.approved_research or {})
     return state_from_mapping(
@@ -3360,7 +3486,7 @@ def _add_results_section(label: str = "Latest run"):
             elem_classes=["run-status-source"],
         )
         with gr.Tabs(elem_classes=["guided-result-tabs"]):
-            with gr.TabItem("Overview"):
+            with gr.TabItem("Overview", render_children=True):
                 context = gr.HTML(
                     RESULTS_PLACEHOLDER_HTML,
                     elem_classes=["run-context-output"],
@@ -3594,7 +3720,7 @@ def create_interface() -> gr.Blocks:
                 )
 
         with gr.Tabs(visible=False, elem_id="workspace-tabs") as main_tabs:
-            with gr.TabItem("Build"):
+            with gr.TabItem("Build", id="build"):
                 with gr.Row(
                     equal_height=False,
                     elem_classes=["guided-instrument-bench", "studio-workbench"],
@@ -3670,6 +3796,13 @@ def create_interface() -> gr.Blocks:
                                     label="Test data (CSV, optional)",
                                     file_types=[".csv"],
                                 )
+                            gr.Markdown(
+                                "**Training and testing:** Training data fits the model. "
+                                "Separate test data measures performance on unseen rows. "
+                                "For tabular data, use a separate test CSV or enable "
+                                "**Create a test split** in Objective. Avoid using the same "
+                                "rows for both training and testing."
+                            )
 
                         with gr.Group(
                             visible=False,
@@ -3687,12 +3820,14 @@ def create_interface() -> gr.Blocks:
                                     value=default_target,
                                     label="Outcome to predict",
                                     allow_custom_value=True,
+                                    info="Select the outcome or label in your dataset, rather than an ID or an input predictor.",
                                 )
                             with gr.Group(visible=False) as upload_target_group:
                                 upload_target = gr.Dropdown(
                                     [],
                                     label="Outcome to predict",
                                     allow_custom_value=True,
+                                    info="Select the column you want the model to predict. Confirm that the labels match your research question.",
                                 )
                             with gr.Row():
                                 task_type = gr.Dropdown(
@@ -3744,6 +3879,13 @@ def create_interface() -> gr.Blocks:
                                 ),
                                 interactive=initial_image,
                             )
+                            gr.Markdown(
+                                "**Holdout:** For tabular data, a 0.20 test split reserves "
+                                "20% of the training file for testing and trains on the "
+                                "remaining 80%. Leave the split disabled when using an "
+                                "existing separate test set. Image workflows use the "
+                                "validation holdout setting shown above."
+                            )
 
                         with gr.Group(
                             visible=False,
@@ -3755,6 +3897,16 @@ def create_interface() -> gr.Blocks:
                                 '<p>The private research backend compares registered '
                                 'pipelines. Recommendations never start training.</p></div>'
                             )
+                            gr.Markdown(
+                                "**Why use research assistance?** Compare feature "
+                                "selection, transformations and training engines before "
+                                "spending the training budget. Optional LLM assistance "
+                                "can interpret your objective and help explain the "
+                                "recommendation. When it is disabled, planning uses "
+                                "deterministic rules; the research record shows the "
+                                "actual planning source. Review the evidence and "
+                                "approve a compatible candidate before running it."
+                            )
                             generate_plan_button = gr.Button(
                                 "Generate research plan",
                                 variant="secondary",
@@ -4190,8 +4342,11 @@ def create_interface() -> gr.Blocks:
                             interactive=False,
                             elem_classes=["guided-parallel-action"],
                         )
+                active_run_progress = gr.HTML(
+                    "", visible=False, elem_classes=["run-progress-output"],
+                )
 
-            with gr.TabItem("Runs"):
+            with gr.TabItem("Runs", id="runs", render_children=True):
                 gr.HTML(
                     """
                     <section class="guided-workbench-heading">
@@ -4200,15 +4355,15 @@ def create_interface() -> gr.Blocks:
                     </section>
                     """
                 )
-                with gr.Tabs():
-                    with gr.TabItem("Latest run"):
+                with gr.Tabs(selected="latest") as run_result_tabs:
+                    with gr.TabItem("Latest run", id="latest", render_children=True):
                         latest_outputs = _add_results_section("Latest run")
                         latest_visual_staging_state = gr.State(
                             _staged_visual_updates(())
                         )
-                    with gr.TabItem("Variant 1"):
+                    with gr.TabItem("Variant 1", id="variant-one"):
                         variant_one_outputs = _add_results_section("Variant 1")
-                    with gr.TabItem("Variant 2"):
+                    with gr.TabItem("Variant 2", id="variant-two"):
                         variant_two_outputs = _add_results_section("Variant 2")
 
             with gr.TabItem("Models") as replay_tab:
@@ -5036,6 +5191,8 @@ def create_interface() -> gr.Blocks:
             guided_run_ui_handler_staged,
             guided_run_inputs,
             [*latest_outputs[:16], latest_visual_staging_state],
+            show_progress="minimal",
+            show_progress_on=primary_run_button,
             api_name=False,
         )
         primary_run_event = primary_run_event.then(
@@ -5050,20 +5207,43 @@ def create_interface() -> gr.Blocks:
             latest_outputs[16],
             api_name=False,
         )
-        primary_run_event.then(
-            _guided_run_button_ready,
+        primary_run_finishing = primary_run_event.then(
+            _guided_run_button_finishing,
             inputs=[],
             outputs=[primary_run_button],
             queue=False,
             api_name=False,
         )
-        parallel_run_button.click(
+        primary_run_finishing.then(
+            _guided_run_complete_ui,
+            inputs=[latest_outputs[0], session_token],
+            outputs=[main_tabs, run_result_tabs, primary_run_button],
+            queue=True,
+            show_progress="hidden",
+            api_name=False,
+        )
+        latest_outputs[7].change(
+            _run_progress_update,
+            inputs=[latest_outputs[7]],
+            outputs=[active_run_progress],
+            queue=False,
+            api_name=False,
+        )
+        parallel_run_event = parallel_run_button.click(
             guided_parallel_plan_ui_handler,
             [workbench_state, session_token],
             [*variant_one_outputs, *variant_two_outputs],
             api_name=False,
             concurrency_limit=2,
         )
+        parallel_run_event.then(
+            _guided_parallel_complete_ui,
+            inputs=[variant_one_outputs[0], variant_two_outputs[0], session_token],
+            outputs=[main_tabs, run_result_tabs],
+            queue=True,
+            show_progress="hidden",
+            api_name=False,
+        )
         next_experiment_outputs = [
             workbench_state,
             user_objective,
@@ -5148,3 +5328,4 @@ __all__ = [
     "replay_route_preview",
     "validate_gateway_security_config",
 ]
+

gateway_catalog.json

@@ -30,6 +30,18 @@
         "col_0"
       ]
     },
+    "cervical_biopsy": {
+      "image": false,
+      "targets": [
+        "Age"
+      ]
+    },
+    "thyroid_recurrence": {
+      "image": false,
+      "targets": [
+        "Age"
+      ]
+    },
     "titanic": {
       "image": false,
       "targets": [

gateway_contract.py

@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import json
+import html
+import math
 import re
 import stat
 import zipfile
@@ -239,8 +241,8 @@ def _optional_visual_prediction(frame: pd.DataFrame) -> tuple[dict[str, Any], st
         )
 
 
-def _optional_execution_metadata(package: Any) -> dict[str, Any]:
-    """Read validated runtime metadata from one optional model-package key.
+def _optional_package_manifest(package: Any) -> dict[str, Any]:
+    """Read optional result metadata without changing the seven-value API.
 
     The backend's public training result remains the historical seven-value
     tuple.  Runtime transparency is carried by the package manifest so old
@@ -260,13 +262,147 @@ def _optional_execution_metadata(package: Any) -> dict[str, Any]:
         return {}
     try:
         manifest, _ = inspect_package_manifest(candidate)
-        from execution_metadata import execution_metadata_from_manifest
-
-        return execution_metadata_from_manifest(manifest)
+        return manifest
     except (ContractError, OSError, TypeError, ValueError):
         return {}
 
 
+def _optional_execution_metadata(
+    package: Any, *, manifest: Mapping[str, Any] | None = None
+) -> dict[str, Any]:
+    from execution_metadata import execution_metadata_from_manifest
+
+    try:
+        return execution_metadata_from_manifest(
+            manifest if manifest is not None else _optional_package_manifest(package)
+        )
+    except (TypeError, ValueError):
+        return {}
+
+
+MAX_LEADERBOARD_ROWS = 500
+_LEADERBOARD_NUMBERS = (
+    "score_val", "fit_time", "pred_time_val", "stack_level",
+)
+
+
+def _leaderboard_text(value: Any) -> str:
+    if not isinstance(value, str) or _SENSITIVE_MESSAGE.search(value):
+        return ""
+    return " ".join(value.split())[:160]
+
+
+def normalize_model_leaderboard(value: Any) -> dict[str, Any]:
+    """Allowlist optional leaderboard metadata before publishing a model result."""
+
+    if not isinstance(value, Mapping):
+        return {}
+    records = value.get("records")
+    if isinstance(records, pd.DataFrame):
+        records = records.to_dict("records")
+    if not isinstance(records, list):
+        return {}
+    rows = []
+    for record in records:
+        if not isinstance(record, Mapping):
+            continue
+        model = _leaderboard_text(record.get("model"))
+        if not model:
+            continue
+        row: dict[str, Any] = {"model": model}
+        for key in _LEADERBOARD_NUMBERS:
+            candidate = record.get(key)
+            try:
+                number = float(candidate) if not isinstance(candidate, bool) else math.nan
+            except (TypeError, ValueError, OverflowError):
+                number = math.nan
+            row[key] = number if math.isfinite(number) else None
+        inferable = record.get("can_infer")
+        row["can_infer"] = inferable if isinstance(inferable, bool) else None
+        rows.append(row)
+    if not rows:
+        return {}
+    # AutoGluon normalizes scores so higher is better, including negative losses.
+    rows.sort(key=lambda row: (
+        row["score_val"] is None,
+        -(row["score_val"] or 0.0),
+    ))
+    # Apply the display limit after ranking so an unsorted input cannot hide
+    # a higher-scoring model near the end of the backend's leaderboard.
+    truncated = bool(value.get("truncated")) or len(rows) > MAX_LEADERBOARD_ROWS
+    rows = rows[:MAX_LEADERBOARD_ROWS]
+    pipeline = value.get("pipeline")
+    pipeline = pipeline if isinstance(pipeline, Mapping) else {}
+    trial = value.get("selected_trial")
+    trial = trial if isinstance(trial, int) and not isinstance(trial, bool) and 1 <= trial <= 4 else None
+    return {
+        "schema_version": 1,
+        "selected_model": _leaderboard_text(value.get("selected_model")),
+        "eval_metric": _leaderboard_text(value.get("eval_metric")),
+        "selected_trial": trial,
+        "pipeline": {
+            key: _leaderboard_text(pipeline.get(key))
+            for key in ("feature_selection", "transformation", "training_engine")
+        },
+        "records": rows,
+        "truncated": truncated,
+    }
+
+
+def model_leaderboard_markdown(value: Any) -> str:
+    """Render the selected pipeline's real AutoGluon models as a public table."""
+
+    data = normalize_model_leaderboard(value)
+    if not data:
+        return ""
+
+    def cell(item: Any) -> str:
+        return html.escape(str(item)).replace("|", "\\|").replace("\n", " ")
+
+    def number(item: Any) -> str:
+        return f"{item:.6g}" if item is not None else "N/A"
+
+    selected = data["selected_model"]
+    lines = ["#### AutoGluon model leaderboard", ""]
+    if selected:
+        lines.extend([f"**Selected model:** {cell(selected)}", ""])
+    details = []
+    if data["selected_trial"] is not None:
+        details.append(f"Selected candidate run: {data['selected_trial']}")
+    details.extend(
+        f"{label}: {cell(data['pipeline'][key])}"
+        for key, label in (
+            ("feature_selection", "Feature selection"),
+            ("transformation", "Transformation"),
+            ("training_engine", "Training engine"),
+        )
+        if data["pipeline"][key]
+    )
+    if details:
+        lines.extend([" · ".join(details), ""])
+    metric = cell(data["eval_metric"] or "the configured evaluation metric")
+    lines.extend([
+        f"Validation scores use **{metric}**. Higher scores are better; "
+        "AutoGluon displays loss metrics as negative scores. These validation "
+        "scores are separate from the held-out test metrics above.",
+        "",
+        "| Rank | Model | Validation score | Fit time (s) | Validation prediction time (s) | Stack level | Can predict | Selection |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+    ])
+    for rank, row in enumerate(data["records"], start=1):
+        values = [
+            str(rank), cell(row["model"]), number(row["score_val"]),
+            number(row["fit_time"]), number(row["pred_time_val"]),
+            number(row["stack_level"]),
+            "Yes" if row["can_infer"] is True else "No" if row["can_infer"] is False else "N/A",
+            "**Selected**" if selected and row["model"] == selected else "",
+        ]
+        lines.append("| " + " | ".join(values) + " |")
+    if data["truncated"]:
+        lines.extend(["", f"Showing up to {MAX_LEADERBOARD_ROWS} models."])
+    return "\n".join(lines)
+
+
 def success_envelope(outputs: Sequence[Any]) -> dict[str, Any]:
     """Normalize the existing seven training outputs into JSON-safe fields."""
 
@@ -289,9 +425,15 @@ def success_envelope(outputs: Sequence[Any]) -> dict[str, Any]:
         "reasoning": str(reasoning or ""),
         "package_available": bool(package),
     }
-    execution_metadata = _optional_execution_metadata(package)
+    manifest = _optional_package_manifest(package)
+    execution_metadata = _optional_execution_metadata(package, manifest=manifest)
     if execution_metadata:
         envelope["execution_metadata"] = execution_metadata
+    leaderboard = normalize_model_leaderboard(
+        manifest.get("model_leaderboard")
+    )
+    if leaderboard:
+        envelope["model_leaderboard"] = leaderboard
     if visual_prediction:
         envelope["visual_prediction"] = visual_prediction
     if visual_warning:
@@ -487,3 +629,4 @@ __all__ = [
     "inspect_package_manifest",
     "load_gateway_catalog",
 ]
+