"""Self-contained HTML visualization of structured scene descriptions. `CaptionVisualizer` renders an `SceneDescription` or a `SceneObjectList` beside its image: the description fields as collapsible cards, and the geometry (bounding boxes, masks, keypoints, text lines) as SVG overlays on the image. Overlays are toggled per object with CSS only, so the output needs no JavaScript and no network access, and it works inside a `gradio.HTML` component. visualizer = CaptionVisualizer() visualizer.save("description.html", image, description) """ from __future__ import annotations import base64 import html import io import json import uuid from pathlib import Path from typing import Any from data_model import ( BaseObject, BodyKeypoint, BoundingBox, Color, GeneralDescription, ObjectMask, Person, SceneDescription, SceneObjectList, Text, _parse_color_token, _parse_coord_token, ) from PIL import Image #: Overlay colors for objects without a color of their own. PALETTE = ("#FF6B6B", "#F7B801", "#6BCB77", "#4D96FF", "#845EC2", "#FF9671", "#00C9A7", "#F9C74F") FACE_COLOR = "#FF69B4" #: Fields shown by dedicated renderers, or as nested objects, instead of as plain values. _SPECIAL_FIELDS = frozenset( { "name", "kind", "description", "details", "artifacts", "relationships", "colors", "skin_color", "bounding_boxes", "face_bbox", "keypoints", "mask", "lines", "children", "clothing", "footwear", "accessories", "eyes", "hair", "facial_hair", } ) _NESTED_LISTS = ( ("children", "Children"), ("clothing", "Clothing"), ("footwear", "Footwear"), ("accessories", "Accessories"), ) _NESTED_SINGLE = (("eyes", "Eyes"), ("hair", "Hair"), ("facial_hair", "Facial hair")) _WEBP_QUALITY = 90 #: Scoped with `#ROOT`, which each render replaces with its own id, so several renders can #: share one page. _FRAGMENT_CSS = """ #ROOT { --cv-bg: #ffffff; --cv-card: #f6f7f9; --cv-card-head: #eceff4; --cv-text: #15181d; --cv-muted: #5b6270; --cv-border: rgba(120, 130, 150, 0.35); --cv-chip: #e6e9ef; --cv-accent: #3b5bdb; font-family: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; font-size: 14px; line-height: 1.5; color: var(--cv-text); } @media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) #ROOT { --cv-bg: #16181d; --cv-card: #1e2127; --cv-card-head: #262a32; --cv-text: #e7e9ee; --cv-muted: #9aa1ae; --cv-border: rgba(160, 170, 190, 0.25); --cv-chip: #2b3039; --cv-accent: #8ea4ff; } } /* An explicit theme on the page (`data-theme`, or gradio's `.dark`) wins over the system one. */ [data-theme="dark"] #ROOT, .dark #ROOT { --cv-bg: #16181d; --cv-card: #1e2127; --cv-card-head: #262a32; --cv-text: #e7e9ee; --cv-muted: #9aa1ae; --cv-border: rgba(160, 170, 190, 0.25); --cv-chip: #2b3039; --cv-accent: #8ea4ff; } #ROOT, #ROOT * { box-sizing: border-box; } #ROOT .cv-form { margin: 0; background: var(--cv-bg); border-radius: 12px; padding: 1rem; } #ROOT .cv-toggle { display: none; } #ROOT .cv-layout { display: flex; flex-direction: column; gap: 1.25rem; } #ROOT .cv-image-panel { width: 100%; min-width: 0; } #ROOT .cv-image-box { position: relative; border-radius: 10px; overflow: hidden; background: #0d0f13; } /* The pixel grid the coordinates refer to, without EXIF rotation. */ #ROOT .cv-image { display: block; width: 100%; height: auto; image-orientation: none; } #ROOT .cv-overlay { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; } #ROOT .cv-overlay g.cv-ov { display: none; } /* Keypoint names only for one open person: with all overlays shown they cover the image. */ #ROOT .cv-overlay g.cv-name-label { display: none; } #ROOT .cv-controls { display: flex; flex-wrap: wrap; gap: 0.5rem; margin-top: 0.75rem; } #ROOT .cv-button { font: inherit; font-size: 0.82rem; color: var(--cv-text); background: var(--cv-chip); border: 1px solid var(--cv-border); border-radius: 999px; padding: 0.25rem 0.8rem; cursor: pointer; user-select: none; } #ROOT .cv-button:hover { border-color: var(--cv-accent); } #ROOT .cv-button .cv-on { display: none; } #ROOT .cv-tree { display: flex; flex-direction: column; gap: 1rem; min-width: 0; } #ROOT .cv-section h3 { margin: 0 0 0.5rem; font-size: 1rem; } #ROOT .cv-general { margin: 0; padding: 0; list-style: none; display: flex; flex-direction: column; gap: 0.4rem; } #ROOT .cv-object { border: 1px solid var(--cv-border); border-radius: 8px; background: var(--cv-card); margin-bottom: 0.6rem; overflow: hidden; } #ROOT .cv-object:last-child { margin-bottom: 0; } #ROOT .cv-summary { display: flex; align-items: center; gap: 0.55rem; padding: 0.55rem 0.8rem; cursor: pointer; background: var(--cv-card-head); } #ROOT .cv-summary::-webkit-details-marker { display: none; } #ROOT .cv-dot { flex: none; width: 11px; height: 11px; border-radius: 50%; box-shadow: 0 0 0 2px var(--cv-bg); } #ROOT .cv-kind { flex: none; font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; color: var(--cv-muted); } #ROOT .cv-name { font-weight: 600; overflow-wrap: anywhere; } #ROOT .cv-body { padding: 0.55rem 0.8rem 0.7rem; display: flex; flex-direction: column; gap: 0.45rem; } #ROOT .cv-field { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.2rem 0.55rem; } #ROOT .cv-label { font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; font-weight: 600; color: var(--cv-muted); } #ROOT .cv-value { flex: 1 1 12rem; min-width: 0; overflow-wrap: anywhere; } #ROOT .cv-list { margin: 0; padding-left: 1.1rem; } #ROOT .cv-list--plain { list-style: none; padding-left: 0; } #ROOT .cv-chips { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 0.3rem; } #ROOT .cv-chip { display: inline-flex; align-items: center; gap: 0.35rem; padding: 0.15rem 0.5rem; border-radius: 999px; background: var(--cv-chip); font-size: 0.8rem; } #ROOT .cv-swatch { width: 14px; height: 14px; border-radius: 50%; border: 1px solid var(--cv-border); } #ROOT .cv-sub > h4 { margin: 0.2rem 0 0.35rem; font-size: 0.85rem; } #ROOT .cv-empty { margin: 0; color: var(--cv-muted); font-style: italic; } @media (min-width: 980px) { #ROOT .cv-layout { flex-direction: row; align-items: flex-start; } #ROOT .cv-image-panel { flex: 1 1 55%; position: sticky; top: 0.5rem; } /* A tall image fits the screen height, so it stays in view beside a long description. */ #ROOT .cv-image-box { width: min(100%, calc((100vh - 5rem) * var(--cv-ar, 1))); margin: 0 auto; } #ROOT .cv-tree { flex: 1 1 45%; } } """ _PAGE_CSS = """ :root { color-scheme: light dark; } body { margin: 0; padding: 1.25rem 1rem 2.5rem; background: #eef0f4; color: #15181d; font-family: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; } @media (prefers-color-scheme: dark) { body { background: #0d0f13; color: #e7e9ee; } } .cv-page { max-width: 1400px; margin: 0 auto; } .cv-page h1 { margin: 0 0 1rem; font-size: 1.2rem; } """ class CaptionVisualizer: """Renders structured scene descriptions as HTML with image overlays.""" def render( self, image: Image.Image, caption: SceneDescription | SceneObjectList | dict | str, title: str | None = None, ) -> str: """A standalone HTML page for `caption` over `image`.""" heading = f"

{html.escape(title)}

" if title else "" page_title = html.escape(title or "Structured scene description") return ( "\n" '\n\n\n' '\n' f"{page_title}\n\n\n\n" f'
{heading}{self.render_fragment(image, caption)}
\n' "\n\n" ) def render_fragment( self, image: Image.Image, caption: SceneDescription | SceneObjectList | dict | str, title: str | None = None, ) -> str: """The visualization as an HTML fragment with its own scoped styles, for embedding.""" caption = self._parse(caption) return _Render(image, caption, title).html() def save( self, path: str | Path, image: Image.Image, caption: SceneDescription | SceneObjectList | dict | str, title: str | None = None, ) -> Path: """Write the standalone page to `path` and return the path.""" path = Path(path) path.write_text(self.render(image, caption, title), encoding="utf-8") return path @staticmethod def _parse(caption: Any) -> SceneDescription | SceneObjectList: if isinstance(caption, (SceneDescription, SceneObjectList)): return caption data = json.loads(caption) if isinstance(caption, (str, bytes)) else caption if not isinstance(data, dict): raise TypeError(f"unsupported caption type {type(caption).__name__}") if "general" in data: return SceneDescription.model_validate(data) return SceneObjectList.model_validate(data) def _coord(value: Any) -> float: return _parse_coord_token(value) def _rgb(color: Color) -> tuple[int, int, int]: return _parse_color_token(color.red), _parse_color_token(color.green), _parse_color_token(color.blue) def _box(bbox: BoundingBox) -> tuple[float, float, float, float]: """(left, top, right, bottom) in percent, corners ordered.""" x0, y0 = _coord(bbox.top_left.x), _coord(bbox.top_left.y) x1, y1 = _coord(bbox.bottom_right.x), _coord(bbox.bottom_right.y) return min(x0, x1), min(y0, y1), max(x0, x1), max(y0, y1) def _number(value: float) -> str: return f"{value:.1f}" def _format_value(value: Any) -> str: if isinstance(value, list): if not value: return "" return '" if isinstance(value, dict): items = "".join( f'
  • {html.escape(str(key))} {_format_value(item)}
    • ' for key, item in value.items() ) return f'' if isinstance(value, float): return f"{value:.2f}".rstrip("0").rstrip(".") return html.escape(str(value)) def _image_data_url(image: Image.Image) -> str: buffer = io.BytesIO() image.convert("RGB").save(buffer, format="WEBP", quality=_WEBP_QUALITY) return "data:image/webp;base64," + base64.b64encode(buffer.getvalue()).decode("ascii") class _Render: """One render: collects the overlay groups while it walks the caption tree.""" def __init__(self, image: Image.Image, caption: SceneDescription | SceneObjectList, title: str | None): self.image = image self.caption = caption self.title = title self.root = f"cv-{uuid.uuid4().hex[:12]}" self.width, self.height = image.size self.unit = min(self.width, self.height) / 100.0 self.toggles: list[tuple[str, str, str]] = [] # (toggle id, overlay id, color) self.overlays: list[str] = [] self.counter = 0 # -- page ------------------------------------------------------------------ def html(self) -> str: if isinstance(self.caption, SceneDescription): general = self._general(self.caption.general) objects = self._section("Objects", self.caption.objects or []) else: general = "" objects = self._section("Objects", self.caption.objects or []) tree = general + objects or '

    The scene description has no content.

    ' all_toggle = f"{self.root}-all" inputs = f'' if self.toggles else "" inputs += "".join(f'' for toggle, _, _ in self.toggles) controls = "" if self.toggles: controls = ( '
    ' f'' '' "
    " ) overlay = ( f'' ) heading = f'

    {html.escape(self.title)}

    ' if self.title else "" return ( f'
    ' f"" f'
    {inputs}{heading}' '
    ' f'
    ' f'described image{overlay}' f"
    {controls}
    " f'
    {tree}
    ' "
    " ) def _toggle_css(self, all_toggle: str) -> str: root = f"#{self.root}" rules = [ f"{root} input#{all_toggle}:checked ~ .cv-layout .cv-overlay g.cv-ov {{ display: inline; }}", f'{root} input#{all_toggle}:checked ~ .cv-layout label[for="{all_toggle}"] .cv-off {{ display: none; }}', f'{root} input#{all_toggle}:checked ~ .cv-layout label[for="{all_toggle}"] .cv-on {{ display: inline; }}', ] for toggle, overlay_id, color in self.toggles: label = f'{root} input#{toggle}:checked ~ .cv-layout label[for="{toggle}"]' group = f'{root} input#{toggle}:checked ~ .cv-layout .cv-overlay g[data-id="{overlay_id}"]' rules += [ f"{group} {{ display: inline; }}", f"{group} .cv-name-label {{ display: inline; }}", f"{label} {{ border-color: {color}; box-shadow: inset 0 0 0 1px {color}; }}", f"{label} .cv-off {{ display: none; }}", f"{label} .cv-on {{ display: inline; }}", ] return "\n".join(rules) # -- sections -------------------------------------------------------------- def _general(self, general: GeneralDescription | None) -> str: if general is None: return "" data = general.model_dump(exclude_none=True) items = [] if general.color_palette: data.pop("color_palette", None) items.append(self._field("Color palette", self._chips(general.color_palette), tag="li")) for key, value in data.items(): items.append(self._field(key.replace("_", " ").capitalize(), _format_value(value), tag="li")) if not items: return "" return f'

    General

    ' def _section(self, title: str, objects: list[BaseObject]) -> str: if not objects: return "" body = "".join(self._object(obj, depth=0) for obj in objects) return f'

    {html.escape(title)} ({len(objects)})

    {body}
    ' # -- objects --------------------------------------------------------------- def _object(self, obj: BaseObject, depth: int) -> str: self.counter += 1 obj_id = f"{self.root}-o{self.counter}" color = self._object_color(obj) line_text = getattr(obj, "text", None) if line_text is None and getattr(obj, "lines", None): # A text object has no name; its lines say more than its description. line_text = " / ".join(line.text for line in obj.lines if line.text) if len(line_text) > 64: line_text = line_text[:61] + "..." name = obj.name or "" if not name and line_text: name = f"“{line_text}”" if not name and obj.description: snippet = obj.description.strip() name = snippet if len(snippet) <= 64 else snippet[:61] + "..." name = name or obj.kind.replace("_", " ") fields = [] if obj.description and name != obj.description.strip(): fields.append(self._field("Description", html.escape(obj.description))) if line_text is not None and obj.name: fields.append(self._field("Text", html.escape(line_text))) for label, key in (("Details", "details"), ("Artifacts", "artifacts")): if getattr(obj, key, None): fields.append(self._field(label, _format_value(getattr(obj, key)))) if obj.relationships: items = "".join( f"
  • {html.escape(rel.relationship)} → {html.escape(rel.to)}
    • " for rel in obj.relationships ) fields.append(self._field("Relationships", f'')) if obj.colors: fields.append(self._field("Colors", self._chips(obj.colors))) if isinstance(obj, Person) and obj.skin_color: fields.append(self._field("Skin color", self._chips([obj.skin_color]))) data = obj.model_dump(exclude_none=True) for key, value in data.items(): if key not in _SPECIAL_FIELDS: fields.append(self._field(key.replace("_", " ").capitalize(), _format_value(value))) fields += self._geometry_fields(obj, obj_id, color) nested = [] for key, title in _NESTED_SINGLE: item = getattr(obj, key, None) if item is not None: nested.append(self._subsection(title, self._object(item, depth + 1))) for key, title in _NESTED_LISTS: items = getattr(obj, key, None) or [] if items: nested.append(self._subsection(title, "".join(self._object(item, depth + 1) for item in items))) if isinstance(obj, Text) and obj.lines: nested.append(self._subsection("Lines", "".join(self._object(line, depth + 1) for line in obj.lines))) body = "".join(fields + nested) or '

    No other fields.

    ' open_attr = " open" if depth == 0 else "" return ( f'
    ' '' f'' f'{html.escape(obj.kind.replace("_", " "))}' f'{html.escape(name)}' "" f'
    {body}
    ' "
    " ) def _geometry_fields(self, obj: BaseObject, obj_id: str, color: str) -> list[str]: fields = [] boxes = obj.bounding_boxes or [] if boxes: shapes = "".join( self._box_shape(bbox, color, index + 1 if len(boxes) > 1 else None) for index, bbox in enumerate(boxes) ) items = "".join("
  • ({}, {}) → ({}, {})
    • ".format(*(_number(v) for v in _box(bbox))) for bbox in boxes) title = "Bounding boxes" if len(boxes) > 1 else "Bounding box" fields.append( self._overlay_field( title, f"{obj_id}-box", color, shapes, f'' ) ) if isinstance(obj, Person) and obj.face_bbox is not None: left, top, right, bottom = _box(obj.face_bbox) fields.append( self._overlay_field( "Face", f"{obj_id}-face", FACE_COLOR, self._box_shape(obj.face_bbox, FACE_COLOR, None), f"({_number(left)}, {_number(top)}) → ({_number(right)}, {_number(bottom)})", ) ) if isinstance(obj, Person) and obj.keypoints: shapes = "".join(self._keypoint_shape(point, color) for point in obj.keypoints) items = "".join( f"
  • {html.escape(point.name)} ({_number(_coord(point.point.x))}, " f"{_number(_coord(point.point.y))})
    • " for point in obj.keypoints ) fields.append( self._overlay_field( f"Keypoints ({len(obj.keypoints)})", f"{obj_id}-kp", color, shapes, f'
    list
    ', ) ) if obj.mask is not None: shape, summary = self._mask_shape(obj.mask, color) if shape: fields.append(self._overlay_field("Mask", f"{obj_id}-mask", color, shape, summary)) else: fields.append(self._field("Mask", summary)) return fields def _overlay_field(self, label: str, overlay_id: str, color: str, shapes: str, content: str) -> str: toggle = f"{overlay_id}-t" self.toggles.append((toggle, overlay_id, color)) self.overlays.append(f'{shapes}') button = ( f'' ) return ( '
    ' f'{html.escape(label)}{button}' f'
    {content}
    ' "
    " ) # -- overlay shapes (viewBox is the image in pixels) ----------------------- def _px(self, x: float, y: float) -> tuple[float, float]: return x / 100.0 * self.width, y / 100.0 * self.height def _box_shape(self, bbox: BoundingBox, color: str, index: int | None) -> str: left, top, right, bottom = _box(bbox) x0, y0 = self._px(left, top) x1, y1 = self._px(right, bottom) width, height = max(x1 - x0, 1.0), max(y1 - y0, 1.0) stroke = max(self.unit * 0.45, 1.5) # A white casing under the colored line keeps boxes visible on pixels of the same color. shape = ( f'' f'' ) if index is not None: shape += self._label(x0 + stroke, y0 + stroke, str(index)) return shape def _keypoint_shape(self, point: BodyKeypoint, color: str) -> str: x, y = self._px(_coord(point.point.x), _coord(point.point.y)) radius = max(self.unit * 0.7, 2.5) return ( f'{html.escape(point.name)}' '' + self._label(x + radius * 1.6, y - radius * 1.6, point.name, small=True) + "" ) def _label(self, x: float, y: float, text: str, small: bool = False) -> str: size = max(self.unit * (2.0 if small else 2.6), 9.0) return ( f'{html.escape(text)}' ) def _mask_shape(self, mask: ObjectMask, color: str) -> tuple[str, str]: rings = [] for ring in mask.polygons or []: values = [_coord(value) for value in ring] points = [self._px(x, y) for x, y in zip(values[0::2], values[1::2], strict=False)] if len(points) >= 3: rings.append("M" + " L".join(f"{x:.1f},{y:.1f}" for x, y in points) + " Z") if not rings: return "", "empty mask" vertices = sum(len(ring) // 2 for ring in mask.polygons or []) stroke = max(self.unit * 0.35, 1.2) path = " ".join(rings) shape = ( f'' f'' ) return shape, f"polygon mask, {len(rings)} ring(s), {vertices} vertices" # -- small parts ----------------------------------------------------------- @staticmethod def _field(label: str, content: str, tag: str = "div") -> str: return ( f'<{tag} class="cv-field">{html.escape(label)}' f'
    {content}
    ' ) @staticmethod def _subsection(title: str, content: str) -> str: return f'

    {html.escape(title)}

    {content}
    ' @staticmethod def _chips(colors: list[Color]) -> str: chips = [] for color in colors: red, green, blue = _rgb(color) label = f"{html.escape(color.name)} · " if color.name else "" chips.append( f'
  • ' f"{label}rgb({red}, {green}, {blue})
    • " ) return f'
      {"".join(chips)}
    ' def _object_color(self, obj: BaseObject) -> str: primary = (obj.colors or [None])[0] if primary is None and isinstance(obj, Person): primary = obj.skin_color if primary is not None: red, green, blue = _rgb(primary) return f"rgb({red},{green},{blue})" return PALETTE[self.counter % len(PALETTE)]