"""Parsed, Plausible, Wrong — live demo of the paper's WORDS pizza parsers. Runs the paper's two released GGUF semantic parsers (Qwen3.5-0.8B and LFM2.5-350M fine-tunes, run in llama.cpp) under the exact prompt, stop string, token cap and greedy decoding their export manifests were verified under, then shows the WORDS ticket, the EXR s-expression the benchmark scores, and a readable ticket. Example orders are the authors' own probes, and the refusals are live: type any off-menu request and the model answers "?" instead of inventing a ticket. """ import json import os import time from pathlib import Path import gradio as gr from huggingface_hub import hf_hub_download from llama_cpp import Llama from pizza_words import render_ticket_html, words_to_exr DATA = Path(__file__).parent / "data" MODELS = { "Qwen3.5-0.8B (paper's main model)": { "repo": "umarsheikh303/pizza-words-gguf", "file": "pizza-words-q8_0.gguf", "manifest": "pizza-words-q8_0.json", }, "LFM2.5-350M (small sibling)": { "repo": "umarsheikh303/pizza-lfm2-words-gguf", "file": "pizza-lfm2-words-q8_0.gguf", "manifest": "pizza-lfm2-words-q8_0.json", }, } DEFAULT_MODEL = "Qwen3.5-0.8B (paper's main model)" # --------------------------------------------------------------------------- # Model loading. Each GGUF (~834 MB / ~379 MB) is pulled once into the # Space's ephemeral disk at boot and mmap'd by llama.cpp; cpu-basic has # 2 vCPU and 16 GB RAM, where 2 threads is fastest for these short greedy # generations (measured: 2 ≈ 4 threads, 8 slower). # --------------------------------------------------------------------------- def load_manifest(name): with open(DATA / MODELS[name]["manifest"], encoding="utf-8") as f: return json.load(f) MANIFESTS = {name: load_manifest(name) for name in MODELS} _LLAMAS = {} def get_llama(name): if name not in _LLAMAS: m = MODELS[name] path = hf_hub_download( repo_id=m["repo"], filename=m["file"], repo_type="model" ) _LLAMAS[name] = Llama( model_path=path, n_ctx=512, n_threads=max(1, min(2, os.cpu_count() or 2)), verbose=False, ) return _LLAMAS[name] # Preload the paper's primary model so the first click is not a cold start. get_llama(DEFAULT_MODEL) def parse_order(order, model_name=DEFAULT_MODEL, progress=gr.Progress()): """One parse, exactly as the export manifest specifies. Preprocessing, decoding and post-processing follow the authors' own site code (site/src/lib/model.ts): trim+lowercase the input, fill the manifest prompt, greedy with top_k=1, stop at the manifest stop string, cap at the manifest cap, then cut at the stop string and trim. """ if not order or not order.strip(): return "Type an order first.", "", "", "" if model_name not in MANIFESTS: model_name = DEFAULT_MODEL m = MANIFESTS[model_name] llm = get_llama(model_name) prompt = m["prompt"].replace("{}", order.strip().lower()) progress(0.2, desc="parsing") t0 = time.perf_counter() out = llm.create_completion( prompt=prompt, max_tokens=m["cap"], temperature=0, top_k=1, stop=[m["stop"]], ) ms = (time.perf_counter() - t0) * 1000 text = out["choices"][0]["text"].split(m["stop"])[0].strip() progress(0.8, desc="decoding ticket") refused = "?" in text.split() exr_out = "" if refused else (words_to_exr(text) or "— does not decode —") return text, exr_out, render_ticket_html(text), f"{ms:.0f} ms" # --------------------------------------------------------------------------- # Examples: the authors' own probes (site/data/{examples,guardrail}.json and # the manifests' verification rows). Every cell is a concrete value. # --------------------------------------------------------------------------- EXAMPLES = [ ["i want two medium pizzas with ham and no olives", DEFAULT_MODEL], ["i would like to try two medium tuna pizzas with extra cheese and no pesto", DEFAULT_MODEL], ["i want to get a large extra cheese pepperoni pizza with no bacon", DEFAULT_MODEL], ["a pizza with ham, actually make that bacon", DEFAULT_MODEL], ["i want a pizza with cheese, wait no cheese, only tomato sauce", DEFAULT_MODEL], ["where is the restroom", DEFAULT_MODEL], ["two cats and dog", DEFAULT_MODEL], ["can i get a burger", DEFAULT_MODEL], ["i would like a small chicken and bacon pizza and a small diet pepsi", "LFM2.5-350M (small sibling)"], ] CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } .ticket { display: flex; flex-direction: column; gap: 8px; padding: 12px; border: 1px solid var(--border-color-primary); border-radius: 8px; background: var(--background-fill-secondary); min-height: 46px; } .t-line { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } .t-head { font-weight: 600; min-width: 90px; color: var(--body-text-color); } .t-reject { color: #b91c1c; font-weight: 600; } .t-head-reject { color: #b91c1c; } .chip { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 13px; padding: 3px 8px; border-radius: 5px; background: color-mix(in oklch, var(--color-accent) 15%, transparent); border: 1px solid color-mix(in oklch, var(--color-accent) 40%, transparent); } """ # Gradio 6: theme and css moved from the Blocks constructor to launch(). with gr.Blocks(title="Parsed, Plausible, Wrong — pizza order parser") as demo: with gr.Column(elem_id="col-container"): gr.HTML( """

🍕 Parsed, Plausible, Wrong

Output format controls silent errors in small-model semantic parsing — live demo of the paper's WORDS pizza-order parsers. Type any spoken-style order; the model answers a structured ticket, or ? when the order isn't from the menu, instead of inventing one.

""" ) with gr.Row(): with gr.Column(scale=5): order_in = gr.Textbox( label="Your order (spoken style)", placeholder="e.g. i want two medium pizzas with ham and no olives", lines=2, autofocus=True, ) model_in = gr.Dropdown( choices=list(MODELS.keys()), value=DEFAULT_MODEL, label="Model", filterable=False, ) run_btn = gr.Button("Parse order", variant="primary") with gr.Column(scale=6): words_out = gr.Textbox( label="WORDS output (what the model writes)", lines=2) exr_out = gr.Textbox( label="EXR — the s-expression the benchmark scores", lines=3) ticket_out = gr.HTML( value=render_ticket_html(""), label="Readable ticket", ) stats_out = gr.Textbox(label="Latency", interactive=False) with gr.Accordion("About this demo", open=False): gr.HTML( """

Both released parsers run here in llama.cpp under the exact prompt, greedy decoding, stop string and token cap of their export manifests (99.0% agreement with the training run for Qwen, 93% for LFM2, measured over 200 orders). The WORDS output is decoded to the benchmark's native s-expression by the paper's own decoder, so what you see is the parse the paper measures.

The finding: the same Qwen3.5-0.8B fine-tune writing the dataset's native s-expression averages 85.8% exact match with 10.3% silent errors — a parse that looks fine and is wrong. Writing natural words instead (pizza 2 medium ham no olives) keeps exact match at 87.5% while cutting silent errors to 4.5% and emitting 3.8× fewer tokens. Validity and safety are not the same thing.

Refusals are a feature: the parsers were trained with 30% negatives and answer ? to anything off the menu — a loud error instead of a silent wrong pizza. Try “two cats and a dog” or “can i get a burger”. The price of strictness: ~2–5% of real chatty orders are refused too.

Self-corrections are applied live: “a pizza with ham, actually make that bacon” gives pizza 1 bacon. Of the paper's 40 hand-written correction probes, 38 are exactly right and none are silently wrong.

Models: umarsheikh303/pizza-words-gguf (Qwen3.5-0.8B: 88.2% exact match / 3.8% silent errors on the 1,357 human-written test orders) and umarsheikh303/pizza-lfm2-words-gguf (LFM2.5-350M: 79.3% / 10.2%). Paper: Parsed, Plausible, Wrong (arXiv:2611.17169), IDM Lab, ITMO University. Code: UmarGit/coffee-llm. Trained on the PIZZA benchmark (Arkoudas et al. 2022, CC-BY-NC-4.0 — non-commercial research use).

""" ) gr.Examples( examples=EXAMPLES, inputs=[order_in, model_in], fn=parse_order, outputs=[words_out, exr_out, ticket_out, stats_out], cache_examples=True, cache_mode="lazy", label="Try the paper's own probes", ) run_btn.click( fn=parse_order, inputs=[order_in, model_in], outputs=[words_out, exr_out, ticket_out, stats_out], ) order_in.submit( fn=parse_order, inputs=[order_in, model_in], outputs=[words_out, exr_out, ticket_out, stats_out], ) demo.launch(theme=gr.themes.Citrus(), css=CSS, show_error=True)