Spaces:
Running on Zero
Running on Zero
Publish /chat via explicit components: gr.api discarded every token
Browse files
ui.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The Gradio surface for the chatMCD Space: the debug UI and the /chat API.
|
| 2 |
+
|
| 3 |
+
Separate from app.py for one reason: app.py loads Qwen3-8B at import, so nothing
|
| 4 |
+
that imports it can run on a laptop, and the API contract therefore could not be
|
| 5 |
+
tested anywhere except in production. It was wrong in production twice.
|
| 6 |
+
|
| 7 |
+
1. The Flask app sent six arguments to ChatInterface's five-parameter endpoint.
|
| 8 |
+
`history` landed in `temperature`. Empty answer, no error.
|
| 9 |
+
2. gr.api() derived the endpoint's OUTPUT signature from the function's return
|
| 10 |
+
annotation. A streaming generator has none, so /chat published
|
| 11 |
+
`returns: []`, burned a GPU call, discarded every yield and returned an
|
| 12 |
+
empty tuple in 1.5 s. Again no error, anywhere.
|
| 13 |
+
|
| 14 |
+
Both were contract bugs, invisible to every test that did not read the published
|
| 15 |
+
API schema. test_api_contract.py now reads it, against this exact code, with a
|
| 16 |
+
stub generator in place of the model.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
from typing import Callable
|
| 22 |
+
|
| 23 |
+
import gradio as gr
|
| 24 |
+
|
| 25 |
+
# The six positional arguments chatmcd/hf_client.py sends, in order. The test
|
| 26 |
+
# asserts the published endpoint matches this exactly.
|
| 27 |
+
CHAT_API_PARAMS = ("message", "history", "temperature", "top_p",
|
| 28 |
+
"repetition_penalty", "max_new_tokens")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def build_demo(chat_fn: Callable, *, base_model: str, adapter: str,
|
| 32 |
+
retrieval_on: bool) -> gr.Blocks:
|
| 33 |
+
with gr.Blocks(title="chatMCD API", analytics_enabled=False) as demo:
|
| 34 |
+
gr.Markdown(
|
| 35 |
+
"## chatMCD — inference endpoint\n"
|
| 36 |
+
f"`{base_model}` + `{adapter or 'no adapter'}`"
|
| 37 |
+
f"{' · retrieval ON' if retrieval_on else ''}\n\n"
|
| 38 |
+
"This is the debug surface. The product lives at "
|
| 39 |
+
"[chatmcd.mdeller.com](https://chatmcd.mdeller.com)."
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
# The UI. ChatInterface owns history internally and does NOT expose it as
|
| 43 |
+
# an API parameter, so its endpoint is
|
| 44 |
+
# (message, temperature, top_p, repetition_penalty, max_new_tokens).
|
| 45 |
+
# The Flask app needs history, so it calls /chat below instead.
|
| 46 |
+
gr.ChatInterface(
|
| 47 |
+
fn=chat_fn,
|
| 48 |
+
type="messages",
|
| 49 |
+
additional_inputs=[
|
| 50 |
+
gr.Slider(0.0, 1.5, value=0.7, step=0.05, label="temperature"),
|
| 51 |
+
gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="top_p"),
|
| 52 |
+
gr.Slider(1.0, 1.5, value=1.05, step=0.01, label="repetition_penalty"),
|
| 53 |
+
gr.Slider(64, 1024, value=512, step=64, label="max_new_tokens"),
|
| 54 |
+
],
|
| 55 |
+
examples=[
|
| 56 |
+
["Give me Marc's quick résumé."],
|
| 57 |
+
["What is Elora Therapeutics?"],
|
| 58 |
+
["Why is BoltzMaker called BoltzMaker?"],
|
| 59 |
+
],
|
| 60 |
+
api_name="chat_ui",
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
# ---------------------------------------------------------------- /chat
|
| 64 |
+
# The real API. Explicit components, not gr.api(): components state the
|
| 65 |
+
# contract instead of inferring it from type hints, and this is the path
|
| 66 |
+
# Gradio actually streams a generator over.
|
| 67 |
+
with gr.Row(visible=False):
|
| 68 |
+
api_message = gr.Textbox(label="message")
|
| 69 |
+
api_history = gr.JSON(label="history")
|
| 70 |
+
api_temperature = gr.Number(value=0.7, label="temperature")
|
| 71 |
+
api_top_p = gr.Number(value=0.9, label="top_p")
|
| 72 |
+
api_repetition_penalty = gr.Number(value=1.05, label="repetition_penalty")
|
| 73 |
+
api_max_new_tokens = gr.Number(value=512, precision=0, label="max_new_tokens")
|
| 74 |
+
api_answer = gr.Textbox(label="answer")
|
| 75 |
+
api_submit = gr.Button("submit")
|
| 76 |
+
|
| 77 |
+
api_submit.click(
|
| 78 |
+
fn=chat_fn,
|
| 79 |
+
inputs=[api_message, api_history, api_temperature, api_top_p,
|
| 80 |
+
api_repetition_penalty, api_max_new_tokens],
|
| 81 |
+
outputs=api_answer,
|
| 82 |
+
api_name="chat",
|
| 83 |
+
show_progress="hidden",
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
return demo
|