multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
5140215 verified
Raw History Blame Contribute Delete
10.2 kB
"""Parsed, Plausible, Wrong — live demo of the paper's WORDS pizza parsers.
Runs the paper's two released GGUF semantic parsers (Qwen3.5-0.8B and
LFM2.5-350M fine-tunes, run in llama.cpp) under the exact prompt, stop
string, token cap and greedy decoding their export manifests were verified
under, then shows the WORDS ticket, the EXR s-expression the benchmark
scores, and a readable ticket. Example orders are the authors' own probes,
and the refusals are live: type any off-menu request and the model answers
"?" instead of inventing a ticket.
"""
import json
import os
import time
from pathlib import Path
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
from pizza_words import render_ticket_html, words_to_exr
DATA = Path(__file__).parent / "data"
MODELS = {
"Qwen3.5-0.8B (paper's main model)": {
"repo": "umarsheikh303/pizza-words-gguf",
"file": "pizza-words-q8_0.gguf",
"manifest": "pizza-words-q8_0.json",
},
"LFM2.5-350M (small sibling)": {
"repo": "umarsheikh303/pizza-lfm2-words-gguf",
"file": "pizza-lfm2-words-q8_0.gguf",
"manifest": "pizza-lfm2-words-q8_0.json",
},
}
DEFAULT_MODEL = "Qwen3.5-0.8B (paper's main model)"
# ---------------------------------------------------------------------------
# Model loading. Each GGUF (~834 MB / ~379 MB) is pulled once into the
# Space's ephemeral disk at boot and mmap'd by llama.cpp; cpu-basic has
# 2 vCPU and 16 GB RAM, where 2 threads is fastest for these short greedy
# generations (measured: 2 ≈ 4 threads, 8 slower).
# ---------------------------------------------------------------------------
def load_manifest(name):
with open(DATA / MODELS[name]["manifest"], encoding="utf-8") as f:
return json.load(f)
MANIFESTS = {name: load_manifest(name) for name in MODELS}
_LLAMAS = {}
def get_llama(name):
if name not in _LLAMAS:
m = MODELS[name]
path = hf_hub_download(
repo_id=m["repo"], filename=m["file"], repo_type="model"
)
_LLAMAS[name] = Llama(
model_path=path,
n_ctx=512,
n_threads=max(1, min(2, os.cpu_count() or 2)),
verbose=False,
)
return _LLAMAS[name]
# Preload the paper's primary model so the first click is not a cold start.
get_llama(DEFAULT_MODEL)
def parse_order(order, model_name=DEFAULT_MODEL, progress=gr.Progress()):
"""One parse, exactly as the export manifest specifies.
Preprocessing, decoding and post-processing follow the authors' own
site code (site/src/lib/model.ts): trim+lowercase the input, fill the
manifest prompt, greedy with top_k=1, stop at the manifest stop string,
cap at the manifest cap, then cut at the stop string and trim.
"""
if not order or not order.strip():
return "Type an order first.", "", "", ""
if model_name not in MANIFESTS:
model_name = DEFAULT_MODEL
m = MANIFESTS[model_name]
llm = get_llama(model_name)
prompt = m["prompt"].replace("{}", order.strip().lower())
progress(0.2, desc="parsing")
t0 = time.perf_counter()
out = llm.create_completion(
prompt=prompt,
max_tokens=m["cap"],
temperature=0,
top_k=1,
stop=[m["stop"]],
)
ms = (time.perf_counter() - t0) * 1000
text = out["choices"][0]["text"].split(m["stop"])[0].strip()
progress(0.8, desc="decoding ticket")
refused = "?" in text.split()
exr_out = "" if refused else (words_to_exr(text) or "— does not decode —")
return text, exr_out, render_ticket_html(text), f"{ms:.0f} ms"
# ---------------------------------------------------------------------------
# Examples: the authors' own probes (site/data/{examples,guardrail}.json and
# the manifests' verification rows). Every cell is a concrete value.
# ---------------------------------------------------------------------------
EXAMPLES = [
["i want two medium pizzas with ham and no olives", DEFAULT_MODEL],
["i would like to try two medium tuna pizzas with extra cheese and no pesto",
DEFAULT_MODEL],
["i want to get a large extra cheese pepperoni pizza with no bacon",
DEFAULT_MODEL],
["a pizza with ham, actually make that bacon", DEFAULT_MODEL],
["i want a pizza with cheese, wait no cheese, only tomato sauce",
DEFAULT_MODEL],
["where is the restroom", DEFAULT_MODEL],
["two cats and dog", DEFAULT_MODEL],
["can i get a burger", DEFAULT_MODEL],
["i would like a small chicken and bacon pizza and a small diet pepsi",
"LFM2.5-350M (small sibling)"],
]
CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
.ticket { display: flex; flex-direction: column; gap: 8px; padding: 12px;
border: 1px solid var(--border-color-primary); border-radius: 8px;
background: var(--background-fill-secondary); min-height: 46px; }
.t-line { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
.t-head { font-weight: 600; min-width: 90px; color: var(--body-text-color); }
.t-reject { color: #b91c1c; font-weight: 600; }
.t-head-reject { color: #b91c1c; }
.chip { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 13px;
padding: 3px 8px; border-radius: 5px;
background: color-mix(in oklch, var(--color-accent) 15%, transparent);
border: 1px solid color-mix(in oklch, var(--color-accent) 40%, transparent); }
"""
# Gradio 6: theme and css moved from the Blocks constructor to launch().
with gr.Blocks(title="Parsed, Plausible, Wrong — pizza order parser") as demo:
with gr.Column(elem_id="col-container"):
gr.HTML(
"""
<div style="text-align:center">
<h1 style="margin-bottom:4px">🍕 Parsed, Plausible, Wrong</h1>
<p style="margin-top:0">Output format controls silent errors in small-model semantic
parsing — live demo of the paper's <b>WORDS</b> pizza-order parsers. Type any
spoken-style order; the model answers a structured ticket, or <code>?</code> when
the order isn't from the menu, instead of inventing one.</p>
</div>
"""
)
with gr.Row():
with gr.Column(scale=5):
order_in = gr.Textbox(
label="Your order (spoken style)",
placeholder="e.g. i want two medium pizzas with ham and no olives",
lines=2,
autofocus=True,
)
model_in = gr.Dropdown(
choices=list(MODELS.keys()),
value=DEFAULT_MODEL,
label="Model",
filterable=False,
)
run_btn = gr.Button("Parse order", variant="primary")
with gr.Column(scale=6):
words_out = gr.Textbox(
label="WORDS output (what the model writes)", lines=2)
exr_out = gr.Textbox(
label="EXR — the s-expression the benchmark scores", lines=3)
ticket_out = gr.HTML(
value=render_ticket_html(""),
label="Readable ticket",
)
stats_out = gr.Textbox(label="Latency", interactive=False)
with gr.Accordion("About this demo", open=False):
gr.HTML(
"""
<p>Both released parsers run here in llama.cpp under the exact prompt, greedy
decoding, stop string and token cap of their export manifests (99.0% agreement
with the training run for Qwen, 93% for LFM2, measured over 200 orders). The
WORDS output is decoded to the benchmark's native s-expression by the paper's
own decoder, so what you see is the parse the paper measures.</p>
<p><b>The finding:</b> the same Qwen3.5-0.8B fine-tune writing the dataset's native
s-expression averages 85.8% exact match with <b>10.3% silent errors</b> — a parse
that looks fine and is wrong. Writing natural words instead
(<code>pizza 2 medium ham no olives</code>) keeps exact match at 87.5% while
cutting silent errors to <b>4.5%</b> and emitting 3.8× fewer tokens. Validity and
safety are not the same thing.</p>
<p><b>Refusals are a feature:</b> the parsers were trained with 30% negatives and
answer <code>?</code> to anything off the menu — a loud error instead of a silent
wrong pizza. Try <i>“two cats and a dog”</i> or <i>“can i get a burger”</i>. The
price of strictness: ~2–5% of real chatty orders are refused too.</p>
<p><b>Self-corrections are applied live:</b> <i>“a pizza with ham, actually make that
bacon”</i> gives <code>pizza 1 bacon</code>. Of the paper's 40 hand-written
correction probes, 38 are exactly right and none are silently wrong.</p>
<p>Models: <a href="https://huggingface.co/umarsheikh303/pizza-words-gguf">umarsheikh303/pizza-words-gguf</a>
(Qwen3.5-0.8B: 88.2% exact match / 3.8% silent errors on the 1,357 human-written
test orders) and
<a href="https://huggingface.co/umarsheikh303/pizza-lfm2-words-gguf">umarsheikh303/pizza-lfm2-words-gguf</a>
(LFM2.5-350M: 79.3% / 10.2%). Paper:
<a href="https://huggingface.co/papers/2611.17169">Parsed, Plausible, Wrong</a>
(arXiv:2611.17169), IDM Lab, ITMO University. Code:
<a href="https://github.com/UmarGit/coffee-llm">UmarGit/coffee-llm</a>. Trained on
the <a href="https://github.com/amazon-science/pizza-semantic-parsing-dataset">PIZZA
benchmark</a> (Arkoudas et al. 2022, CC-BY-NC-4.0 — non-commercial research use).</p>
"""
)
gr.Examples(
examples=EXAMPLES,
inputs=[order_in, model_in],
fn=parse_order,
outputs=[words_out, exr_out, ticket_out, stats_out],
cache_examples=True,
cache_mode="lazy",
label="Try the paper's own probes",
)
run_btn.click(
fn=parse_order,
inputs=[order_in, model_in],
outputs=[words_out, exr_out, ticket_out, stats_out],
)
order_in.submit(
fn=parse_order,
inputs=[order_in, model_in],
outputs=[words_out, exr_out, ticket_out, stats_out],
)
demo.launch(theme=gr.themes.Citrus(), css=CSS, show_error=True)