Spaces:

KwabsHug
/

VisualLearningIdea

Running on Zero

kwabs22

Claude Opus 5.5

Weld before decimating so stalled meshes reach their budget; stop repeated ladder rungs

e0f9b43 9 days ago

8.74 kB

"""Mini HTML returned next to every picture.
HTML is the final destination: a chat model reads this page, edits it and
builds its own on it. The page is self-contained Three.js. Geometry is inlined
at low poly (a rung of the simplification ladder), so it outlives Gradio's
temporary file URLs. Even low poly costs thousands of tokens, so the tools
return the page as a URL unless the caller asks for its text.
"""
import json

import numpy as np

from .scene import GLOW, _sky
from .simplify import base_color, decimate

THREE = "https://unpkg.com/three@0.160.0/"
SCENE_FACE_BUDGET = 400   # per object in a scene page
LADDER_MAX_FACES = 1000   # rungs above this are left to the GLB files


def _hex(rgb):
    return "#%02x%02x%02x" % tuple(int(c) for c in rgb[:3])


def _sky_hex(sun):
    sky = _sky(sun)
    return sky if isinstance(sky, str) else _hex([c * 255 for c in sky])


def _geometry(mesh, budget):
    """Inline mesh; "floor" marks a mesh that wouldn't decimate down to the budget."""
    m = decimate(mesh, budget) if len(mesh.faces) > budget else mesh
    return {"faces": int(len(m.faces)), "budget": budget, "floor": bool(len(m.faces) > budget),
            "p": np.round(m.vertices, 2).ravel().tolist(),  # cm: plenty at low poly, fewer tokens
            "i": m.faces.ravel().astype(int).tolist()}


def _object(oid, word, mesh, position, yaw_deg, cand=None, glow=False, budget=SCENE_FACE_BUDGET, label=None):
    o = {"id": oid, "word": word, "label": label, "position": [round(float(c), 3) for c in position],
         "yaw_deg": round(float(yaw_deg), 1), "color": _hex(GLOW if glow else base_color(mesh)),
         "emits_light": glow}
    if cand and cand.get("source") == "primitive":
        o["primitive"] = {"shape": cand["shape"], "size": [round(float(e), 3) for e in mesh.extents]}
    else:
        o["mesh"] = _geometry(mesh, budget)
    return o


def scene_page(ledger, meshes, config):
    sun = config["sun"]
    objects = []
    for it in ledger["items"]:
        p = config["placements"].get(it["id"])
        if it["kind"] != "object" or p is None:
            continue
        dec = it["decision"]
        objects.append(_object(it["id"], it["word"], meshes[it["id"]], p["position"], p["yaw_deg"],
                               cand=dec["chosen"], glow=bool(p.get("light")), label=dec.get("label")))
    ground = None
    if any(i["kind"] == "environment" and i["role"] == "plane" for i in ledger["items"]):
        reach = max([abs(c) for o in objects if not o["emits_light"] for c in (o["position"][0], o["position"][2])]
                    + [0.3])
        ground = {"radius": round(1.6 * reach + 0.5, 3)}
    data = {
        "title": ledger["sentence"],
        "reading": [{"word": t["text"], "kind": t["kind"], "commits": t["commits"]} for t in ledger["tokens"]],
        "relations": [{"subject": i["subject"], "relation": i["relation"], "object": i["object"]}
                      for i in ledger["items"] if i["kind"] == "relation"],
        "unresolved": [{"question": q.get("question"), "default": q.get("default_chosen")}
                       for q in ledger["unresolved"]],
        "sky": _sky_hex(sun), "sun": sun, "ground": ground, "objects": objects,
    }
    return page(data)


def ladder_page(item, rungs):
    """Rungs left to right, from the most detailed embeddable one down to the box."""
    objects, x = [], 0.0
    for label, rung in rungs:
        if len(rung.faces) > LADDER_MAX_FACES:
            continue
        hw = 0.5 * max(rung.extents[0], rung.extents[2])
        objects.append(_object(f"rung{len(objects)}", item["word"], rung, [x + hw, 0, 0], 0,
                               budget=LADDER_MAX_FACES, label=label))
        x += 2 * hw + 0.3 * max(rung.extents[1], 0.1)
    for o in objects:
        o["position"][0] = round(o["position"][0] - x / 2, 3)
    data = {"title": f"{item['word']}: simplification ladder",
            "caption": "left → right: " + " · ".join(o["label"] for o in objects),
            "sky": "#2a3346", "sun": None, "ground": None, "objects": objects}
    return page(data)


def page(data):
    payload = json.dumps(data, separators=(",", ":")).replace("</", "<\\/")
    return TEMPLATE.replace("__THREE__", THREE).replace("__TITLE__", _escape(data["title"])) \
        .replace("__DATA__", payload)


def _escape(s):
    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")


TEMPLATE = """<!doctype html>
<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>__TITLE__</title>
<style>html,body{margin:0;height:100%;overflow:hidden;font:13px system-ui,sans-serif}
#cap{position:fixed;left:12px;bottom:12px;color:#fff;background:#0008;padding:6px 10px;border-radius:6px}</style>
<script type="importmap">{"imports":{"three":"__THREE__build/three.module.js","three/addons/":"__THREE__examples/jsm/"}}</script>
</head><body><div id="cap"></div>
<script type="module">
import * as THREE from "three";
import {OrbitControls} from "three/addons/controls/OrbitControls.js";
// Scene data from VisualLearningIdea: positions in metres, Y up, yaw about Y.
// Objects carry inline low-poly geometry ("mesh") or a primitive spec.
const D = __DATA__;
const renderer = new THREE.WebGLRenderer({antialias: true});
renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
renderer.setSize(innerWidth, innerHeight);
renderer.shadowMap.enabled = true;
document.body.appendChild(renderer.domElement);
const scene = new THREE.Scene();
scene.background = new THREE.Color(D.sky);
scene.add(new THREE.HemisphereLight(0xffffff, 0x445533, 0.9));
const camera = new THREE.PerspectiveCamera(45, innerWidth / innerHeight, 0.01, 1000);
function geometryOf(o) {
  if (o.primitive) {
    const [w, h, d] = o.primitive.size;
    const g = {sphere: () => new THREE.SphereGeometry(h / 2, 24, 16),
               cylinder: () => new THREE.CylinderGeometry(w / 2, w / 2, h, 24),
               cone: () => new THREE.ConeGeometry(w / 2, h, 24),
               plane: () => new THREE.BoxGeometry(w, h, d),
               box: () => new THREE.BoxGeometry(w, h, d)}[o.primitive.shape]();
    g.translate(0, h / 2, 0);  // rest on y = 0 like every asset
    return g;
  }
  const g = new THREE.BufferGeometry();
  g.setAttribute("position", new THREE.Float32BufferAttribute(o.mesh.p, 3));
  g.setIndex(o.mesh.i);
  g.computeVertexNormals();
  return g;
}
const frame = new THREE.Box3();
for (const o of D.objects) {
  const mat = o.emits_light
    ? new THREE.MeshBasicMaterial({color: o.color})
    : new THREE.MeshStandardMaterial({color: o.color, roughness: 0.85, flatShading: true, side: THREE.DoubleSide});
  const m = new THREE.Mesh(geometryOf(o), mat);
  m.name = o.id;
  m.position.fromArray(o.position);
  m.rotation.y = THREE.MathUtils.degToRad(o.yaw_deg);
  m.castShadow = !o.emits_light;
  m.receiveShadow = true;
  scene.add(m);
  if (!o.emits_light) frame.expandByObject(m);
}
if (frame.isEmpty()) frame.setFromCenterAndSize(new THREE.Vector3(), new THREE.Vector3(1, 1, 1));
const center = frame.getCenter(new THREE.Vector3());
const radius = Math.max(frame.getSize(new THREE.Vector3()).length(), 0.3);
if (D.ground) {
  const g = new THREE.Mesh(new THREE.CircleGeometry(D.ground.radius, 64),
                           new THREE.MeshStandardMaterial({color: 0x4d7c3a, roughness: 1}));
  g.rotation.x = -Math.PI / 2;
  g.receiveShadow = true;
  scene.add(g);
}
const sun = new THREE.DirectionalLight(0xfff1d0, 2.2);
if (D.sun) {
  const el = THREE.MathUtils.degToRad(D.sun.elevation_deg), az = THREE.MathUtils.degToRad(D.sun.azimuth_deg);
  const dir = new THREE.Vector3(Math.cos(el) * Math.sin(az), Math.sin(el), Math.cos(el) * Math.cos(az));
  sun.position.copy(center).addScaledVector(dir, radius * 3);
  sun.castShadow = !!D.sun.shadows;
} else {
  sun.position.copy(center).add(new THREE.Vector3(radius, radius * 2, radius * 1.2));
}
sun.target.position.copy(center);
Object.assign(sun.shadow.camera, {left: -radius * 1.5, right: radius * 1.5, top: radius * 1.5,
                                  bottom: -radius * 1.5, far: radius * 8});
sun.shadow.mapSize.set(1024, 1024);
scene.add(sun, sun.target);
camera.position.set(center.x + radius * 0.9, center.y + radius * 0.6, center.z + radius * 1.2);
const controls = new OrbitControls(camera, renderer.domElement);
controls.target.copy(center);
controls.update();
document.getElementById("cap").textContent = D.caption || D.title;
addEventListener("resize", () => {
  camera.aspect = innerWidth / innerHeight;
  camera.updateProjectionMatrix();
  renderer.setSize(innerWidth, innerHeight);
});
renderer.setAnimationLoop(() => { controls.update(); renderer.render(scene, camera); });
</script></body></html>
"""

	"""Mini HTML returned next to every picture.

	HTML is the final destination: a chat model reads this page, edits it and
	builds its own on it. The page is self-contained Three.js. Geometry is inlined
	at low poly (a rung of the simplification ladder), so it outlives Gradio's
	temporary file URLs. Even low poly costs thousands of tokens, so the tools
	return the page as a URL unless the caller asks for its text.
	"""
	import json

	import numpy as np

	from .scene import GLOW, _sky
	from .simplify import base_color, decimate

	THREE = "https://unpkg.com/three@0.160.0/"
	SCENE_FACE_BUDGET = 400 # per object in a scene page
	LADDER_MAX_FACES = 1000 # rungs above this are left to the GLB files


	def _hex(rgb):
	return "#%02x%02x%02x" % tuple(int(c) for c in rgb[:3])


	def _sky_hex(sun):
	sky = _sky(sun)
	return sky if isinstance(sky, str) else _hex([c * 255 for c in sky])


	def _geometry(mesh, budget):
	"""Inline mesh; "floor" marks a mesh that wouldn't decimate down to the budget."""
	m = decimate(mesh, budget) if len(mesh.faces) > budget else mesh
	return {"faces": int(len(m.faces)), "budget": budget, "floor": bool(len(m.faces) > budget),
	"p": np.round(m.vertices, 2).ravel().tolist(), # cm: plenty at low poly, fewer tokens
	"i": m.faces.ravel().astype(int).tolist()}


	def _object(oid, word, mesh, position, yaw_deg, cand=None, glow=False, budget=SCENE_FACE_BUDGET, label=None):
	o = {"id": oid, "word": word, "label": label, "position": [round(float(c), 3) for c in position],
	"yaw_deg": round(float(yaw_deg), 1), "color": _hex(GLOW if glow else base_color(mesh)),
	"emits_light": glow}
	if cand and cand.get("source") == "primitive":
	o["primitive"] = {"shape": cand["shape"], "size": [round(float(e), 3) for e in mesh.extents]}
	else:
	o["mesh"] = _geometry(mesh, budget)
	return o


	def scene_page(ledger, meshes, config):
	sun = config["sun"]
	objects = []
	for it in ledger["items"]:
	p = config["placements"].get(it["id"])
	if it["kind"] != "object" or p is None:
	continue
	dec = it["decision"]
	objects.append(_object(it["id"], it["word"], meshes[it["id"]], p["position"], p["yaw_deg"],
	cand=dec["chosen"], glow=bool(p.get("light")), label=dec.get("label")))
	ground = None
	if any(i["kind"] == "environment" and i["role"] == "plane" for i in ledger["items"]):
	reach = max([abs(c) for o in objects if not o["emits_light"] for c in (o["position"][0], o["position"][2])]
	+ [0.3])
	ground = {"radius": round(1.6 * reach + 0.5, 3)}
	data = {
	"title": ledger["sentence"],
	"reading": [{"word": t["text"], "kind": t["kind"], "commits": t["commits"]} for t in ledger["tokens"]],
	"relations": [{"subject": i["subject"], "relation": i["relation"], "object": i["object"]}
	for i in ledger["items"] if i["kind"] == "relation"],
	"unresolved": [{"question": q.get("question"), "default": q.get("default_chosen")}
	for q in ledger["unresolved"]],
	"sky": _sky_hex(sun), "sun": sun, "ground": ground, "objects": objects,
	}
	return page(data)


	def ladder_page(item, rungs):
	"""Rungs left to right, from the most detailed embeddable one down to the box."""
	objects, x = [], 0.0
	for label, rung in rungs:
	if len(rung.faces) > LADDER_MAX_FACES:
	continue
	hw = 0.5 * max(rung.extents[0], rung.extents[2])
	objects.append(_object(f"rung{len(objects)}", item["word"], rung, [x + hw, 0, 0], 0,
	budget=LADDER_MAX_FACES, label=label))
	x += 2 * hw + 0.3 * max(rung.extents[1], 0.1)
	for o in objects:
	o["position"][0] = round(o["position"][0] - x / 2, 3)
	data = {"title": f"{item['word']}: simplification ladder",
	"caption": "left → right: " + " · ".join(o["label"] for o in objects),
	"sky": "#2a3346", "sun": None, "ground": None, "objects": objects}
	return page(data)


	def page(data):
	payload = json.dumps(data, separators=(",", ":")).replace("</", "<\\/")
	return TEMPLATE.replace("__THREE__", THREE).replace("__TITLE__", _escape(data["title"])) \
	.replace("__DATA__", payload)


	def _escape(s):
	return s.replace("&", "&").replace("<", "<").replace(">", ">")


	TEMPLATE = """<!doctype html>
	<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
	<title>__TITLE__</title>
	<style>html,body{margin:0;height:100%;overflow:hidden;font:13px system-ui,sans-serif}
	#cap{position:fixed;left:12px;bottom:12px;color:#fff;background:#0008;padding:6px 10px;border-radius:6px}</style>
	<script type="importmap">{"imports":{"three":"__THREE__build/three.module.js","three/addons/":"__THREE__examples/jsm/"}}</script>
	</head><body><div id="cap"></div>
	<script type="module">
	import * as THREE from "three";
	import {OrbitControls} from "three/addons/controls/OrbitControls.js";

	// Scene data from VisualLearningIdea: positions in metres, Y up, yaw about Y.
	// Objects carry inline low-poly geometry ("mesh") or a primitive spec.
	const D = __DATA__;

	const renderer = new THREE.WebGLRenderer({antialias: true});
	renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
	renderer.setSize(innerWidth, innerHeight);
	renderer.shadowMap.enabled = true;
	document.body.appendChild(renderer.domElement);
	const scene = new THREE.Scene();
	scene.background = new THREE.Color(D.sky);
	scene.add(new THREE.HemisphereLight(0xffffff, 0x445533, 0.9));
	const camera = new THREE.PerspectiveCamera(45, innerWidth / innerHeight, 0.01, 1000);

	function geometryOf(o) {
	if (o.primitive) {
	const [w, h, d] = o.primitive.size;
	const g = {sphere: () => new THREE.SphereGeometry(h / 2, 24, 16),
	cylinder: () => new THREE.CylinderGeometry(w / 2, w / 2, h, 24),
	cone: () => new THREE.ConeGeometry(w / 2, h, 24),
	plane: () => new THREE.BoxGeometry(w, h, d),
	box: () => new THREE.BoxGeometry(w, h, d)}[o.primitive.shape]();
	g.translate(0, h / 2, 0); // rest on y = 0 like every asset
	return g;
	}
	const g = new THREE.BufferGeometry();
	g.setAttribute("position", new THREE.Float32BufferAttribute(o.mesh.p, 3));
	g.setIndex(o.mesh.i);
	g.computeVertexNormals();
	return g;
	}

	const frame = new THREE.Box3();
	for (const o of D.objects) {
	const mat = o.emits_light
	? new THREE.MeshBasicMaterial({color: o.color})
	: new THREE.MeshStandardMaterial({color: o.color, roughness: 0.85, flatShading: true, side: THREE.DoubleSide});
	const m = new THREE.Mesh(geometryOf(o), mat);
	m.name = o.id;
	m.position.fromArray(o.position);
	m.rotation.y = THREE.MathUtils.degToRad(o.yaw_deg);
	m.castShadow = !o.emits_light;
	m.receiveShadow = true;
	scene.add(m);
	if (!o.emits_light) frame.expandByObject(m);
	}
	if (frame.isEmpty()) frame.setFromCenterAndSize(new THREE.Vector3(), new THREE.Vector3(1, 1, 1));
	const center = frame.getCenter(new THREE.Vector3());
	const radius = Math.max(frame.getSize(new THREE.Vector3()).length(), 0.3);

	if (D.ground) {
	const g = new THREE.Mesh(new THREE.CircleGeometry(D.ground.radius, 64),
	new THREE.MeshStandardMaterial({color: 0x4d7c3a, roughness: 1}));
	g.rotation.x = -Math.PI / 2;
	g.receiveShadow = true;
	scene.add(g);
	}

	const sun = new THREE.DirectionalLight(0xfff1d0, 2.2);
	if (D.sun) {
	const el = THREE.MathUtils.degToRad(D.sun.elevation_deg), az = THREE.MathUtils.degToRad(D.sun.azimuth_deg);
	const dir = new THREE.Vector3(Math.cos(el) * Math.sin(az), Math.sin(el), Math.cos(el) * Math.cos(az));
	sun.position.copy(center).addScaledVector(dir, radius * 3);
	sun.castShadow = !!D.sun.shadows;
	} else {
	sun.position.copy(center).add(new THREE.Vector3(radius, radius * 2, radius * 1.2));
	}
	sun.target.position.copy(center);
	Object.assign(sun.shadow.camera, {left: -radius * 1.5, right: radius * 1.5, top: radius * 1.5,
	bottom: -radius * 1.5, far: radius * 8});
	sun.shadow.mapSize.set(1024, 1024);
	scene.add(sun, sun.target);

	camera.position.set(center.x + radius * 0.9, center.y + radius * 0.6, center.z + radius * 1.2);
	const controls = new OrbitControls(camera, renderer.domElement);
	controls.target.copy(center);
	controls.update();

	document.getElementById("cap").textContent = D.caption \|\| D.title;
	addEventListener("resize", () => {
	camera.aspect = innerWidth / innerHeight;
	camera.updateProjectionMatrix();
	renderer.setSize(innerWidth, innerHeight);
	});
	renderer.setAnimationLoop(() => { controls.update(); renderer.render(scene, camera); });
	</script></body></html>
	"""