Spaces:
Running on Zero
Running on Zero
kwabs22
Claude Opus 5.5
Add a pronoun rule option: most recent object or last clause's subject
6287002 | """VisualLearningIdea: translate English into 3D, then see how far 3D can simplify. | |
| One Space for people (this UI) and for models (every public event below is an | |
| MCP tool: demo.launch(mcp_server=True)). Helper events are api_visibility="private". | |
| The Space only visualises; discussion happens in the chat that calls it. Every | |
| picture comes back with a mini HTML page, because HTML is the final destination. | |
| The page is a URL by default: geometry is for the handoff, not the loop. | |
| """ | |
| import html as html_lib | |
| import json | |
| import os | |
| import re | |
| import tempfile | |
| import threading | |
| import spaces # ZeroGPU: import before any CUDA-touching library | |
| import gradio as gr | |
| from vli import assets, papercraft, pipeline, scene, simplify | |
| from vli import html as html_pages # `html` is the tools' output-mode argument | |
| from vli.grammar import check_words as _check_words | |
| from vli.lexicon import LEXICON, PRIMITIVE_SHAPES | |
| threading.Thread(target=LEXICON.ensure, daemon=True).start() # warm the dictionary | |
| def _fail(e): | |
| raise gr.Error(str(e)) | |
| def _json(text, name, default=None): | |
| """Parse a JSON argument; blank -> default, junk -> a readable error.""" | |
| if not (text or "").strip(): | |
| if default is None: | |
| raise ValueError(f"{name} is empty; translate a sentence first") | |
| return default | |
| try: | |
| return json.loads(text) | |
| except json.JSONDecodeError: | |
| raise ValueError(f"{name} is not valid JSON") | |
| def _seed(value): | |
| """Seeds arrive as strings (some MCP clients can't send numbers).""" | |
| try: | |
| return int(float(str(value).strip() or 0)) | |
| except ValueError: | |
| raise ValueError(f"seed must be a whole number, got {value!r}") | |
| def _count(value, name, lo, hi): | |
| """A whole number in [lo, hi], sent as a string like seeds.""" | |
| try: | |
| n = int(float(str(value).strip())) | |
| except ValueError: | |
| raise ValueError(f"{name} must be a whole number, got {value!r}") | |
| if not lo <= n <= hi: | |
| raise ValueError(f"{name} must be between {lo} and {hi}, got {n}") | |
| return n | |
| HTML_MODES = ["url", "inline", "skip"] | |
| def _html_out(page, mode): | |
| """The mini HTML as (file, text): a file for "url", file and text for "inline", neither for "skip". | |
| Inline geometry costs thousands of tokens, so a model iterating on the ledger | |
| gets the file URL and asks for the text only when it builds on the page. | |
| """ | |
| mode = (mode or "url").strip().lower() | |
| if mode not in HTML_MODES: | |
| raise ValueError(f"html must be one of {HTML_MODES}, got {mode!r}") | |
| if mode == "skip" or not page: | |
| return None, "" | |
| path = os.path.join(tempfile.mkdtemp(prefix="vli_html_"), "scene.html") | |
| with open(path, "w", encoding="utf-8") as f: | |
| f.write(page) | |
| return path, page if mode == "inline" else "" | |
| def generate_mesh(prompt): | |
| """GPU entry point for the generation tier (last in the fallback chain). | |
| ZeroGPU needs a @spaces.GPU function at startup. Nothing calls this until | |
| text-to-3D is wired in, so no quota is used. | |
| """ | |
| raise NotImplementedError("generation tier is not wired up yet") | |
| # ---------------- MCP tools ---------------- | |
| def check_words(sentence: str, primitives_json: str = "{}") -> str: | |
| """Check which words of a sentence exist in the 3D vocabulary, and complete the word being typed. | |
| A word is available only when a ready asset (SimReady or Objaverse) exists for it; | |
| "maps_to" names the asset it would get. Words with no asset are listed in "missing": | |
| rephrase them, or map each one to a primitive shape in primitives_json. Verbs (sat, | |
| stood, lay, shone, faced) and prepositions (on, under, beside, near, behind, in front | |
| of, above) are relations between meshes; whatever "shone" becomes the light source. | |
| Args: | |
| sentence: The English sentence, complete or partly typed. | |
| primitives_json: JSON object mapping missing words to shapes, e.g. {"sun": "sphere"}. | |
| Shapes: box, sphere, cylinder, cone, plane. | |
| Returns: | |
| JSON string: {"ok", "words": [{text, kind, available_in, maps_to | primitive}], "missing", | |
| "primitive_shapes", "completions"}. | |
| """ | |
| try: | |
| return json.dumps(_check_words(sentence, _json(primitives_json, "primitives_json", {}))) | |
| except Exception as e: | |
| _fail(e) | |
| def translate_to_scene(sentence: str, primitives_json: str = "{}", seed: str = "0", | |
| html: str = "url", pronouns: str = "recent") -> tuple[str, str, str, str, str]: | |
| """Translate an English sentence into a 3D scene. | |
| Produces the serial ledger (tokens in reading order, the items each word commits, | |
| and the unresolved questions), one configuration of that ledger chosen by `seed`, | |
| a rendered picture, a GLB model and a mini HTML page. The HTML is a self-contained | |
| Three.js scene with inline low-poly geometry: read it, edit it, build on it. Words | |
| with no asset must be mapped to a primitive in primitives_json (see check_words). | |
| Args: | |
| sentence: A sentence using available words, e.g. "the cat sat and the lamp glowed". | |
| primitives_json: JSON object mapping missing words to shapes, e.g. {"sun": "sphere"}. | |
| seed: Picks one of the configurations that satisfy the ledger (positions, facing, sun). | |
| html: "url" (default) returns the mini HTML as a file URL only; "inline" also returns | |
| its text (thousands of tokens: ask for it when building on the page); "skip" omits it. | |
| Iterate on the ledger and picture; the ledger can always re-create the page. | |
| pronouns: Default referent for it/he/she/they: "recent" (default) = the most recent | |
| object; "subject" = the subject of the most recent clause. Either way the choice is | |
| an implied question with the other objects as alternatives. Kept in the ledger. | |
| Returns: | |
| (ledger JSON string, rendered PNG, GLB scene file, mini HTML file, mini HTML text or ""). | |
| """ | |
| try: | |
| *rest, page = pipeline.translate(sentence, _json(primitives_json, "primitives_json", {}), _seed(seed), | |
| (pronouns or "recent").strip().lower()) | |
| return (*rest, *_html_out(page, html)) | |
| except Exception as e: | |
| _fail(e) | |
| def refine_scene(ledger_json: str, changes_json: str = "[]", seed: str = "0", | |
| html: str = "url") -> tuple[str, str, str, str, str]: | |
| """Refine a ledger: freeze some decisions and change the others, then re-render. | |
| Changes are a JSON list of: | |
| {"id": "n1", "freeze": true} lock an item (false unlocks) | |
| {"id": "n1", "use": 2} swap in alternatives[2]; the old choice takes slot 2, | |
| so using 2 again swaps back (other slots don't move) | |
| {"id": "n1", "reject": true} reject the tier: simready -> objaverse -> primitive | |
| (generation is not wired up: a primitive can't be rejected) | |
| {"id": "n1", "height_m": 0.4} set the object's height in metres | |
| {"id": "n1", "shape": "sphere"} use a primitive (box, sphere, cylinder, cone, plane) | |
| {"reroll": true} every unfrozen object takes its next alternative; | |
| repeated rerolls visit the whole pool, whatever the seed | |
| {"id": "n12", "bind": "n1"} n12 is the same thing as n1: merged into it, its relations move | |
| {"id": "q11", "bind": "n5"} the word behind reference question q11 ("it", "the cat") means n5 | |
| Frozen items cannot be changed. The configuration (placement, sun angle) is always | |
| re-chosen from `seed`: change the seed for another valid configuration. | |
| Args: | |
| ledger_json: Ledger JSON from translate_to_scene or a previous refine_scene. | |
| changes_json: JSON list of changes, as above. | |
| seed: Picks the configuration (placement, sun). Placement scales with each | |
| asset's size, so swapping an asset shifts offsets even at the same seed. | |
| html: "url" (default), "inline" or "skip", as in translate_to_scene. | |
| Returns: | |
| (ledger JSON string, rendered PNG, GLB scene file, mini HTML file, mini HTML text or ""). | |
| """ | |
| try: | |
| ledger = _json(ledger_json, "ledger_json") | |
| pipeline.apply_changes(ledger, _json(changes_json, "changes_json", [])) | |
| *rest, page = pipeline.outputs(ledger, _seed(seed)) | |
| return (*rest, *_html_out(page, html)) | |
| except Exception as e: | |
| _fail(e) | |
| def extend_scene(ledger_json: str, sentence: str, primitives_json: str = "{}", seed: str = "0", | |
| html: str = "url", pronouns: str = "") -> tuple[str, str, str, str, str]: | |
| """Add the next sentence of a paragraph to an existing scene. | |
| Paragraphs are sentences plus bindings. "the cat" when a cat is already in the ledger, | |
| and pronouns (it, he, she, they), refer back to an existing item instead of adding one; | |
| each such reference is an implied question ("which cat? the one already mentioned", | |
| "'it' = the cat?") whose `binds` names the item. Rebind with refine_scene | |
| {"id": "<question id>", "bind": "<item id>"}. "a/an/another cat" always adds a new one. | |
| Args: | |
| ledger_json: Ledger JSON from translate_to_scene, refine_scene or a previous extend_scene. | |
| sentence: The next sentence, e.g. "the apple lay beside it". | |
| primitives_json: JSON object mapping missing words to shapes, as in translate_to_scene. | |
| seed: Picks one of the configurations that satisfy the whole ledger. | |
| html: "url" (default), "inline" or "skip", as in translate_to_scene. | |
| pronouns: "recent" or "subject", as in translate_to_scene; blank = the ledger's rule. | |
| Returns: | |
| (ledger JSON string, rendered PNG, GLB scene file, mini HTML file, mini HTML text or ""). | |
| """ | |
| try: | |
| ledger = _json(ledger_json, "ledger_json") | |
| *rest, page = pipeline.extend(ledger, sentence, _json(primitives_json, "primitives_json", {}), _seed(seed), | |
| (pronouns or "").strip().lower() or None) | |
| return (*rest, *_html_out(page, html)) | |
| except Exception as e: | |
| _fail(e) | |
| def simplify_item(ledger_json: str, item_id: str, html: str = "url", rings: str = "8", | |
| samples: str = "16") -> tuple[str, list, list, str, str]: | |
| """Build the simplification ladder for one object: full asset -> fewer faces -> rings -> convex hull -> box. | |
| Look at where the object stops reading as its word: that rung is the least geometry | |
| that still carries the meaning, and the low rungs are the ones that can be made | |
| physically (fewer faces = fewer papercraft pieces). | |
| The rings rung is a second path: `rings` horizontal sections, each outline sampled at | |
| `samples` angles, joined into bands (one paper strip per band). The report lists every | |
| ring's height, centre and radii, so each vertex can be checked by hand. | |
| Args: | |
| ledger_json: Ledger JSON from translate_to_scene or refine_scene. | |
| item_id: The object item to simplify, e.g. "n1". | |
| html: "url" (default), "inline" or "skip", as in translate_to_scene. The ladder page | |
| holds several rungs of up to 1000 faces each, so inline is especially large. | |
| rings: Number of horizontal sections for the rings rung (2-64). | |
| samples: Points sampled around each section (3-128). | |
| Returns: | |
| (report JSON string, gallery of rung pictures, GLB file per rung, | |
| mini HTML file with the low rungs side by side, mini HTML text or ""). | |
| """ | |
| try: | |
| ledger = _json(ledger_json, "ledger_json") | |
| item = next((i for i in ledger["items"] if i["id"] == item_id), None) | |
| if item is None or item["kind"] != "object": | |
| raise ValueError(f"{item_id!r} is not an object item in this ledger") | |
| mesh = assets.settle(item) | |
| work = tempfile.mkdtemp(prefix="vli_ladder_") | |
| rungs = simplify.ladder(mesh, _count(rings, "rings", 2, 64), _count(samples, "samples", 3, 128)) | |
| report, gallery, files = {"item": item_id, "word": item["word"], "rungs": []}, [], [] | |
| for k, (label, rung) in enumerate(rungs): | |
| png = scene.render({item_id: rung}, os.path.join(work, f"{k}.png"), title=label) | |
| slug = re.sub(r"\W+", "_", label) | |
| glb = os.path.join(work, f"{k}_{slug}.glb") | |
| rung.export(glb) | |
| entry = {"rung": label, **simplify.stats(rung)} | |
| if "rings" in rung.metadata: | |
| entry["rings"] = rung.metadata["rings"] | |
| report["rungs"].append(entry) | |
| gallery.append((png, label)) | |
| files.append(glb) | |
| return (json.dumps(report, indent=1), gallery, files, | |
| *_html_out(html_pages.ladder_page(item, rungs), html)) | |
| except Exception as e: | |
| _fail(e) | |
| def papercraft_rings(ledger_json: str, item_id: str, rings: str = "8", samples: str = "16", | |
| height_cm: str = "") -> tuple[str, list, list]: | |
| """Make a printable papercraft net of one object from its rings rung, on A4 pages (see simplify_item). | |
| Every band between two rings unrolls exactly into one paper strip; bands that would | |
| curl past ~120 degrees on paper are split into parts (2a, 2b, ...). Glue tabs sit on | |
| one side of every joint and name the piece they glue to. Caps under 4 mm are left out | |
| (the band closes to a point there). Cut solid lines, fold dashed lines. | |
| Args: | |
| ledger_json: Ledger JSON from translate_to_scene or refine_scene. | |
| item_id: The object item to make, e.g. "n1". | |
| rings: Number of horizontal sections (2-64). Fewer rings = fewer, simpler pieces. | |
| samples: Points around each section (3-128). Fewer samples = fewer folds. | |
| height_cm: Height of the paper model in cm; blank = the largest model whose biggest | |
| piece still fits one A4 page. The report gives that A4 limit either way. | |
| Returns: | |
| (report JSON string with the pieces, pages and what glues to what, one PNG picture per A4 | |
| page at 150 dpi to look at, one SVG file per A4 page in mm to print). | |
| """ | |
| try: | |
| ledger = _json(ledger_json, "ledger_json") | |
| item = next((i for i in ledger["items"] if i["id"] == item_id), None) | |
| if item is None or item["kind"] != "object": | |
| raise ValueError(f"{item_id!r} is not an object item in this ledger") | |
| count, n = _count(rings, "rings", 2, 64), _count(samples, "samples", 3, 128) | |
| h = None | |
| if str(height_cm).strip(): | |
| try: | |
| h = float(str(height_cm).strip()) | |
| except ValueError: | |
| raise ValueError(f"height_cm must be a number, got {height_cm!r}") | |
| if not 1 <= h <= 200: | |
| raise ValueError(f"height_cm must be between 1 and 200, got {h}") | |
| _, profile = simplify.rings(assets.settle(item), count, n) | |
| if len(profile) < 2: | |
| raise ValueError(f"{item_id!r} has fewer than 2 sections; try more rings") | |
| real_cm = (profile[-1]["y"] - profile[0]["y"]) * 100 | |
| a4_cm = papercraft.a4_height_cm(profile, n) | |
| pieces, scale = papercraft.net(profile, n, h or a4_cm) | |
| work, slug = tempfile.mkdtemp(prefix="vli_paper_"), re.sub(r"\W+", "_", item["word"]) | |
| sheets = papercraft.pages(pieces) | |
| files, pictures, page_report = [], [], [] | |
| for k, placed in enumerate(sheets, 1): | |
| title = f"{item['word']}: {len(profile)} rings x {n}, page {k}/{len(sheets)}" | |
| text, size = papercraft.svg(placed, title) | |
| path = os.path.join(work, f"{slug}_net_p{k}.svg") | |
| with open(path, "w", encoding="utf-8") as f: | |
| f.write(text) | |
| files.append(path) | |
| pictures.append((papercraft.png(placed, title, os.path.join(work, f"{slug}_net_p{k}.png")), f"page {k}")) | |
| page_report.append({"page": k, "size_mm": size, "pieces": [pc["name"] for pc, _ in placed]}) | |
| report = {"item": item_id, "word": item["word"], "rings": len(profile), "samples": n, | |
| "model_height_cm": round((profile[-1]["y"] - profile[0]["y"]) * scale / 10, 1), | |
| "real_height_cm": round(real_cm, 1), "max_height_cm_for_a4": round(a4_cm, 1), | |
| "fits_a4": bool(h is None or h <= a4_cm), "pages": page_report, | |
| "pieces": [{"name": name, **j} for name, j in papercraft.joins(pieces).items()]} | |
| return json.dumps(report, indent=1), pictures, files | |
| except Exception as e: | |
| _fail(e) | |
| # ---------------- UI-only helpers ---------------- | |
| def _suggestions(words_json): | |
| data = json.loads(words_json or "{}") | |
| marks = [] | |
| for w in data.get("words", []): | |
| if w["kind"] == "unknown": | |
| marks.append(f"~~{w['text']}~~") | |
| elif w["kind"] == "noun": | |
| marks.append(f"**{w['text']}** → {w.get('maps_to', '?')}") | |
| elif w["kind"] == "primitive": | |
| marks.append(f"_{w['text']}_ ({w['primitive']})") | |
| else: | |
| marks.append(w["text"]) | |
| status = " · ".join(marks) or "_start typing_" | |
| if data.get("missing"): | |
| status += " \n_Struck-through words have no asset: pick a suggestion, rephrase, or choose a primitive._" | |
| missing = data.get("missing", []) | |
| return (status, gr.update(choices=data.get("completions", []), value=None), | |
| gr.update(choices=missing, value=missing[0] if missing else None)) | |
| def _complete(sentence, choice): | |
| if not choice: | |
| return sentence | |
| return re.sub(r"[A-Za-z]*$", "", sentence) + choice + " " | |
| def _use_primitive(primitives_json, word, shape): | |
| prims = _json(primitives_json, "primitives_json", {}) | |
| if word: | |
| prims[word] = shape | |
| return json.dumps(prims) | |
| def _html_preview(path): | |
| """Live preview (sandboxed iframe) of the mini HTML file.""" | |
| if not path: | |
| return "" | |
| with open(path, encoding="utf-8") as f: | |
| page = f.read() | |
| return (f'<iframe sandbox="allow-scripts" srcdoc="{html_lib.escape(page, quote=True)}" ' | |
| 'style="width:100%;height:460px;border:0;border-radius:8px"></iframe>') | |
| EXAMPLE_CHANGES = '[\n {"id": "n1", "freeze": true},\n {"reroll": true}\n]' | |
| DATASETS_MD = """ | |
| **Dictionary (public datasets, read-only, no account needed):** | |
| 1. [NVIDIA SimReady](https://github.com/NVIDIA-Omniverse/simready-central): metric OpenUSD assets, | |
| e.g. [nvidia/simready-assets](https://huggingface.co/datasets/nvidia/simready-assets). License per package | |
| (e.g. CC-BY-4.0). Only geometry and the thumbnail are downloaded, not the ~1 GB of textures. | |
| 2. [allenai/objaverse](https://huggingface.co/datasets/allenai/objaverse): the LVIS-labelled subset. | |
| License per object; non-commercial (NC) and no-derivatives (ND) objects are skipped. | |
| 3. Missing words: a primitive you choose (box, sphere, cylinder, cone, plane). Generation comes later. | |
| This Space only visualises and stores no token. Discuss the results in your chat | |
| (e.g. HuggingChat via MCP): every picture comes with a mini HTML page to build on, | |
| as a URL by default (`html="inline"` returns its text). | |
| """ | |
| with gr.Blocks(title="VisualLearningIdea") as demo: | |
| gr.Markdown("# English → 3D\nType with the words the 3D dictionary has. **Bold** = an object asset; " | |
| "verbs and prepositions are relations between meshes.") | |
| gr.Markdown(DATASETS_MD) | |
| with gr.Row(): | |
| sentence = gr.Textbox(label="Sentence", placeholder="the cat sat and the lamp glowed", scale=4) | |
| seed = gr.Textbox(label="Configuration seed", value="0", scale=1) | |
| pronoun_rule = gr.Radio(label="'it' means", choices=["recent", "subject"], value="recent", scale=1, | |
| info="recent: last object · subject: last clause's subject") | |
| status = gr.Markdown() | |
| suggest = gr.Radio(label="Suggestions", choices=[], interactive=True) | |
| with gr.Row(): | |
| missing = gr.Dropdown(label="Missing word", choices=[], interactive=True) | |
| shape = gr.Dropdown(label="Primitive", choices=PRIMITIVE_SHAPES, value="box", interactive=True) | |
| use_prim = gr.Button("Use primitive") | |
| primitives = gr.Code(label="Primitives", language="json", value="{}", interactive=True) | |
| words = gr.Code(label="Word check", language="json", visible=False) | |
| go = gr.Button("Translate", variant="primary") | |
| with gr.Row(): | |
| picture = gr.Image(label="Picture", type="filepath") | |
| model = gr.Model3D(label="3D") | |
| with gr.Accordion("Mini HTML", open=False): | |
| html_mode = gr.Radio(label="HTML text in the result", choices=HTML_MODES, value="url", | |
| info="url: file only · inline: also the text · skip: no page") | |
| scene_preview = gr.HTML() | |
| scene_file = gr.File(label="Download .html") | |
| scene_html = gr.Code(label="HTML", language="html") | |
| with gr.Accordion("Ledger: freeze and change (serial side)", open=False): | |
| ledger = gr.Code(label="Ledger", language="json", interactive=True) | |
| changes = gr.Code(label="Changes", language="json", value=EXAMPLE_CHANGES, interactive=True) | |
| refine = gr.Button("Refine") | |
| with gr.Row(): | |
| next_sentence = gr.Textbox(label="Next sentence (refers back: 'the cat', 'it')", scale=4) | |
| extend = gr.Button("Add sentence", scale=1) | |
| with gr.Accordion("Simplification ladder", open=False): | |
| with gr.Row(): | |
| item_id = gr.Textbox(label="Object item id", value="n1") | |
| ring_count = gr.Textbox(label="Rings", value="8") | |
| ring_samples = gr.Textbox(label="Samples per ring", value="16") | |
| simp = gr.Button("Simplify") | |
| ladder_pics = gr.Gallery(label="Rungs", columns=4) | |
| ladder_preview = gr.HTML() | |
| ladder_file = gr.File(label="Download .html") | |
| ladder_html = gr.Code(label="HTML", language="html") | |
| ladder_report = gr.Code(label="Report", language="json") | |
| ladder_files = gr.File(label="GLB per rung", file_count="multiple") | |
| with gr.Accordion("Papercraft net (rings)", open=False): | |
| paper_height = gr.Textbox(label="Model height in cm (blank = largest that fits A4)", value="") | |
| paper = gr.Button("Make net") | |
| paper_pages = gr.Gallery(label="A4 pages", columns=3) | |
| paper_file = gr.File(label="A4 pages to print (.svg, mm)", file_count="multiple") | |
| paper_report = gr.Code(label="Pieces", language="json") | |
| gr.on([sentence.change, primitives.change], check_words, [sentence, primitives], words, | |
| api_name="check_words", trigger_mode="always_last") \ | |
| .then(_suggestions, words, [status, suggest, missing], api_visibility="private") | |
| suggest.select(_complete, [sentence, suggest], sentence, api_visibility="private") | |
| use_prim.click(_use_primitive, [primitives, missing, shape], primitives, api_visibility="private") | |
| go.click(translate_to_scene, [sentence, primitives, seed, html_mode, pronoun_rule], | |
| [ledger, picture, model, scene_file, scene_html], api_name="translate_to_scene") \ | |
| .then(_html_preview, scene_file, scene_preview, api_visibility="private") | |
| refine.click(refine_scene, [ledger, changes, seed, html_mode], | |
| [ledger, picture, model, scene_file, scene_html], api_name="refine_scene") \ | |
| .then(_html_preview, scene_file, scene_preview, api_visibility="private") | |
| extend.click(extend_scene, [ledger, next_sentence, primitives, seed, html_mode, pronoun_rule], | |
| [ledger, picture, model, scene_file, scene_html], api_name="extend_scene") \ | |
| .then(_html_preview, scene_file, scene_preview, api_visibility="private") | |
| simp.click(simplify_item, [ledger, item_id, html_mode, ring_count, ring_samples], | |
| [ladder_report, ladder_pics, ladder_files, ladder_file, ladder_html], api_name="simplify_item") \ | |
| .then(_html_preview, ladder_file, ladder_preview, api_visibility="private") | |
| paper.click(papercraft_rings, [ledger, item_id, ring_count, ring_samples, paper_height], | |
| [paper_report, paper_pages, paper_file], api_name="papercraft_rings") | |
| if __name__ == "__main__": | |
| demo.launch(mcp_server=True) | |