Image-Text-to-Text
Transformers
Safetensors
qwen3_5_moe
image-captioning
structured-output
json
object-detection
image-segmentation
keypoint-detection
ocr
vision-language
conversational
Instructions to use briaai/fibo-scene-analyzer with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use briaai/fibo-scene-analyzer with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="briaai/fibo-scene-analyzer") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("briaai/fibo-scene-analyzer") model = AutoModelForMultimodalLM.from_pretrained("briaai/fibo-scene-analyzer", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use briaai/fibo-scene-analyzer with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "briaai/fibo-scene-analyzer" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "briaai/fibo-scene-analyzer", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/briaai/fibo-scene-analyzer
- SGLang
How to use briaai/fibo-scene-analyzer with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "briaai/fibo-scene-analyzer" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "briaai/fibo-scene-analyzer", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "briaai/fibo-scene-analyzer" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "briaai/fibo-scene-analyzer", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use briaai/fibo-scene-analyzer with Docker Model Runner:
docker model run hf.co/briaai/fibo-scene-analyzer
Add the harness code: Transformers, vLLM and server clients, the schema and the visualizer
0a38c5b verified Download caption_visualizer.py from briaai/fibo-scene-analyzer: direct link, hf CLI and curl.
- Browser
- Download file 26.7 kB
-
https://huggingface.co/briaai/fibo-scene-analyzer/resolve/main/caption_visualizer.py
- Command line
-
hf download hf://briaai/fibo-scene-analyzer/caption_visualizer.py
-
curl -L -o caption_visualizer.py https://huggingface.co/briaai/fibo-scene-analyzer/resolve/main/caption_visualizer.py
26.7 kB
| """Self-contained HTML visualization of structured scene descriptions. | |
| `CaptionVisualizer` renders an `SceneDescription` or a `SceneObjectList` beside its image: | |
| the description fields as collapsible cards, and the geometry (bounding boxes, masks, | |
| keypoints, text lines) as SVG overlays on the image. Overlays are toggled per object with | |
| CSS only, so the output needs no JavaScript and no network access, and it works inside a | |
| `gradio.HTML` component. | |
| visualizer = CaptionVisualizer() | |
| visualizer.save("description.html", image, description) | |
| """ | |
| from __future__ import annotations | |
| import base64 | |
| import html | |
| import io | |
| import json | |
| import uuid | |
| from pathlib import Path | |
| from typing import Any | |
| from data_model import ( | |
| BaseObject, | |
| BodyKeypoint, | |
| BoundingBox, | |
| Color, | |
| GeneralDescription, | |
| ObjectMask, | |
| Person, | |
| SceneDescription, | |
| SceneObjectList, | |
| Text, | |
| _parse_color_token, | |
| _parse_coord_token, | |
| ) | |
| from PIL import Image | |
| #: Overlay colors for objects without a color of their own. | |
| PALETTE = ("#FF6B6B", "#F7B801", "#6BCB77", "#4D96FF", "#845EC2", "#FF9671", "#00C9A7", "#F9C74F") | |
| FACE_COLOR = "#FF69B4" | |
| #: Fields shown by dedicated renderers, or as nested objects, instead of as plain values. | |
| _SPECIAL_FIELDS = frozenset( | |
| { | |
| "name", | |
| "kind", | |
| "description", | |
| "details", | |
| "artifacts", | |
| "relationships", | |
| "colors", | |
| "skin_color", | |
| "bounding_boxes", | |
| "face_bbox", | |
| "keypoints", | |
| "mask", | |
| "lines", | |
| "children", | |
| "clothing", | |
| "footwear", | |
| "accessories", | |
| "eyes", | |
| "hair", | |
| "facial_hair", | |
| } | |
| ) | |
| _NESTED_LISTS = ( | |
| ("children", "Children"), | |
| ("clothing", "Clothing"), | |
| ("footwear", "Footwear"), | |
| ("accessories", "Accessories"), | |
| ) | |
| _NESTED_SINGLE = (("eyes", "Eyes"), ("hair", "Hair"), ("facial_hair", "Facial hair")) | |
| _WEBP_QUALITY = 90 | |
| #: Scoped with `#ROOT`, which each render replaces with its own id, so several renders can | |
| #: share one page. | |
| _FRAGMENT_CSS = """ | |
| #ROOT { --cv-bg: #ffffff; --cv-card: #f6f7f9; --cv-card-head: #eceff4; --cv-text: #15181d; --cv-muted: #5b6270; | |
| --cv-border: rgba(120, 130, 150, 0.35); --cv-chip: #e6e9ef; --cv-accent: #3b5bdb; | |
| font-family: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; | |
| font-size: 14px; line-height: 1.5; color: var(--cv-text); } | |
| @media (prefers-color-scheme: dark) { | |
| :root:not([data-theme="light"]) #ROOT { --cv-bg: #16181d; --cv-card: #1e2127; --cv-card-head: #262a32; | |
| --cv-text: #e7e9ee; --cv-muted: #9aa1ae; --cv-border: rgba(160, 170, 190, 0.25); --cv-chip: #2b3039; | |
| --cv-accent: #8ea4ff; } | |
| } | |
| /* An explicit theme on the page (`data-theme`, or gradio's `.dark`) wins over the system one. */ | |
| [data-theme="dark"] #ROOT, .dark #ROOT { --cv-bg: #16181d; --cv-card: #1e2127; --cv-card-head: #262a32; --cv-text: #e7e9ee; --cv-muted: #9aa1ae; | |
| --cv-border: rgba(160, 170, 190, 0.25); --cv-chip: #2b3039; --cv-accent: #8ea4ff; } | |
| #ROOT, #ROOT * { box-sizing: border-box; } | |
| #ROOT .cv-form { margin: 0; background: var(--cv-bg); border-radius: 12px; padding: 1rem; } | |
| #ROOT .cv-toggle { display: none; } | |
| #ROOT .cv-layout { display: flex; flex-direction: column; gap: 1.25rem; } | |
| #ROOT .cv-image-panel { width: 100%; min-width: 0; } | |
| #ROOT .cv-image-box { position: relative; border-radius: 10px; overflow: hidden; background: #0d0f13; } | |
| /* The pixel grid the coordinates refer to, without EXIF rotation. */ | |
| #ROOT .cv-image { display: block; width: 100%; height: auto; image-orientation: none; } | |
| #ROOT .cv-overlay { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; } | |
| #ROOT .cv-overlay g.cv-ov { display: none; } | |
| /* Keypoint names only for one open person: with all overlays shown they cover the image. */ | |
| #ROOT .cv-overlay g.cv-name-label { display: none; } | |
| #ROOT .cv-controls { display: flex; flex-wrap: wrap; gap: 0.5rem; margin-top: 0.75rem; } | |
| #ROOT .cv-button { font: inherit; font-size: 0.82rem; color: var(--cv-text); background: var(--cv-chip); | |
| border: 1px solid var(--cv-border); border-radius: 999px; padding: 0.25rem 0.8rem; cursor: pointer; | |
| user-select: none; } | |
| #ROOT .cv-button:hover { border-color: var(--cv-accent); } | |
| #ROOT .cv-button .cv-on { display: none; } | |
| #ROOT .cv-tree { display: flex; flex-direction: column; gap: 1rem; min-width: 0; } | |
| #ROOT .cv-section h3 { margin: 0 0 0.5rem; font-size: 1rem; } | |
| #ROOT .cv-general { margin: 0; padding: 0; list-style: none; display: flex; flex-direction: column; gap: 0.4rem; } | |
| #ROOT .cv-object { border: 1px solid var(--cv-border); border-radius: 8px; background: var(--cv-card); | |
| margin-bottom: 0.6rem; overflow: hidden; } | |
| #ROOT .cv-object:last-child { margin-bottom: 0; } | |
| #ROOT .cv-summary { display: flex; align-items: center; gap: 0.55rem; padding: 0.55rem 0.8rem; cursor: pointer; | |
| background: var(--cv-card-head); } | |
| #ROOT .cv-summary::-webkit-details-marker { display: none; } | |
| #ROOT .cv-dot { flex: none; width: 11px; height: 11px; border-radius: 50%; box-shadow: 0 0 0 2px var(--cv-bg); } | |
| #ROOT .cv-kind { flex: none; font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; | |
| color: var(--cv-muted); } | |
| #ROOT .cv-name { font-weight: 600; overflow-wrap: anywhere; } | |
| #ROOT .cv-body { padding: 0.55rem 0.8rem 0.7rem; display: flex; flex-direction: column; gap: 0.45rem; } | |
| #ROOT .cv-field { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.2rem 0.55rem; } | |
| #ROOT .cv-label { font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; font-weight: 600; | |
| color: var(--cv-muted); } | |
| #ROOT .cv-value { flex: 1 1 12rem; min-width: 0; overflow-wrap: anywhere; } | |
| #ROOT .cv-list { margin: 0; padding-left: 1.1rem; } | |
| #ROOT .cv-list--plain { list-style: none; padding-left: 0; } | |
| #ROOT .cv-chips { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 0.3rem; } | |
| #ROOT .cv-chip { display: inline-flex; align-items: center; gap: 0.35rem; padding: 0.15rem 0.5rem; border-radius: 999px; | |
| background: var(--cv-chip); font-size: 0.8rem; } | |
| #ROOT .cv-swatch { width: 14px; height: 14px; border-radius: 50%; border: 1px solid var(--cv-border); } | |
| #ROOT .cv-sub > h4 { margin: 0.2rem 0 0.35rem; font-size: 0.85rem; } | |
| #ROOT .cv-empty { margin: 0; color: var(--cv-muted); font-style: italic; } | |
| @media (min-width: 980px) { | |
| #ROOT .cv-layout { flex-direction: row; align-items: flex-start; } | |
| #ROOT .cv-image-panel { flex: 1 1 55%; position: sticky; top: 0.5rem; } | |
| /* A tall image fits the screen height, so it stays in view beside a long description. */ | |
| #ROOT .cv-image-box { width: min(100%, calc((100vh - 5rem) * var(--cv-ar, 1))); margin: 0 auto; } | |
| #ROOT .cv-tree { flex: 1 1 45%; } | |
| } | |
| """ | |
| _PAGE_CSS = """ | |
| :root { color-scheme: light dark; } | |
| body { margin: 0; padding: 1.25rem 1rem 2.5rem; background: #eef0f4; color: #15181d; | |
| font-family: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; } | |
| @media (prefers-color-scheme: dark) { body { background: #0d0f13; color: #e7e9ee; } } | |
| .cv-page { max-width: 1400px; margin: 0 auto; } | |
| .cv-page h1 { margin: 0 0 1rem; font-size: 1.2rem; } | |
| """ | |
| class CaptionVisualizer: | |
| """Renders structured scene descriptions as HTML with image overlays.""" | |
| def render( | |
| self, | |
| image: Image.Image, | |
| caption: SceneDescription | SceneObjectList | dict | str, | |
| title: str | None = None, | |
| ) -> str: | |
| """A standalone HTML page for `caption` over `image`.""" | |
| heading = f"<h1>{html.escape(title)}</h1>" if title else "" | |
| page_title = html.escape(title or "Structured scene description") | |
| return ( | |
| "<!DOCTYPE html>\n" | |
| '<html lang="en">\n<head>\n<meta charset="utf-8"/>\n' | |
| '<meta name="viewport" content="width=device-width, initial-scale=1"/>\n' | |
| f"<title>{page_title}</title>\n<style>{_PAGE_CSS}</style>\n</head>\n<body>\n" | |
| f'<div class="cv-page">{heading}{self.render_fragment(image, caption)}</div>\n' | |
| "</body>\n</html>\n" | |
| ) | |
| def render_fragment( | |
| self, | |
| image: Image.Image, | |
| caption: SceneDescription | SceneObjectList | dict | str, | |
| title: str | None = None, | |
| ) -> str: | |
| """The visualization as an HTML fragment with its own scoped styles, for embedding.""" | |
| caption = self._parse(caption) | |
| return _Render(image, caption, title).html() | |
| def save( | |
| self, | |
| path: str | Path, | |
| image: Image.Image, | |
| caption: SceneDescription | SceneObjectList | dict | str, | |
| title: str | None = None, | |
| ) -> Path: | |
| """Write the standalone page to `path` and return the path.""" | |
| path = Path(path) | |
| path.write_text(self.render(image, caption, title), encoding="utf-8") | |
| return path | |
| def _parse(caption: Any) -> SceneDescription | SceneObjectList: | |
| if isinstance(caption, (SceneDescription, SceneObjectList)): | |
| return caption | |
| data = json.loads(caption) if isinstance(caption, (str, bytes)) else caption | |
| if not isinstance(data, dict): | |
| raise TypeError(f"unsupported caption type {type(caption).__name__}") | |
| if "general" in data: | |
| return SceneDescription.model_validate(data) | |
| return SceneObjectList.model_validate(data) | |
| def _coord(value: Any) -> float: | |
| return _parse_coord_token(value) | |
| def _rgb(color: Color) -> tuple[int, int, int]: | |
| return _parse_color_token(color.red), _parse_color_token(color.green), _parse_color_token(color.blue) | |
| def _box(bbox: BoundingBox) -> tuple[float, float, float, float]: | |
| """(left, top, right, bottom) in percent, corners ordered.""" | |
| x0, y0 = _coord(bbox.top_left.x), _coord(bbox.top_left.y) | |
| x1, y1 = _coord(bbox.bottom_right.x), _coord(bbox.bottom_right.y) | |
| return min(x0, x1), min(y0, y1), max(x0, x1), max(y0, y1) | |
| def _number(value: float) -> str: | |
| return f"{value:.1f}" | |
| def _format_value(value: Any) -> str: | |
| if isinstance(value, list): | |
| if not value: | |
| return "" | |
| return '<ul class="cv-list">' + "".join(f"<li>{_format_value(item)}</li>" for item in value) + "</ul>" | |
| if isinstance(value, dict): | |
| items = "".join( | |
| f'<li><span class="cv-label">{html.escape(str(key))}</span> {_format_value(item)}</li>' | |
| for key, item in value.items() | |
| ) | |
| return f'<ul class="cv-list cv-list--plain">{items}</ul>' | |
| if isinstance(value, float): | |
| return f"{value:.2f}".rstrip("0").rstrip(".") | |
| return html.escape(str(value)) | |
| def _image_data_url(image: Image.Image) -> str: | |
| buffer = io.BytesIO() | |
| image.convert("RGB").save(buffer, format="WEBP", quality=_WEBP_QUALITY) | |
| return "data:image/webp;base64," + base64.b64encode(buffer.getvalue()).decode("ascii") | |
| class _Render: | |
| """One render: collects the overlay groups while it walks the caption tree.""" | |
| def __init__(self, image: Image.Image, caption: SceneDescription | SceneObjectList, title: str | None): | |
| self.image = image | |
| self.caption = caption | |
| self.title = title | |
| self.root = f"cv-{uuid.uuid4().hex[:12]}" | |
| self.width, self.height = image.size | |
| self.unit = min(self.width, self.height) / 100.0 | |
| self.toggles: list[tuple[str, str, str]] = [] # (toggle id, overlay id, color) | |
| self.overlays: list[str] = [] | |
| self.counter = 0 | |
| # -- page ------------------------------------------------------------------ | |
| def html(self) -> str: | |
| if isinstance(self.caption, SceneDescription): | |
| general = self._general(self.caption.general) | |
| objects = self._section("Objects", self.caption.objects or []) | |
| else: | |
| general = "" | |
| objects = self._section("Objects", self.caption.objects or []) | |
| tree = general + objects or '<p class="cv-empty">The scene description has no content.</p>' | |
| all_toggle = f"{self.root}-all" | |
| inputs = f'<input type="checkbox" class="cv-toggle" id="{all_toggle}"/>' if self.toggles else "" | |
| inputs += "".join(f'<input type="checkbox" class="cv-toggle" id="{toggle}"/>' for toggle, _, _ in self.toggles) | |
| controls = "" | |
| if self.toggles: | |
| controls = ( | |
| '<div class="cv-controls">' | |
| f'<label class="cv-button" for="{all_toggle}">' | |
| '<span class="cv-off">Show all overlays</span><span class="cv-on">Hide all overlays</span></label>' | |
| '<button type="reset" class="cv-button">Clear</button>' | |
| "</div>" | |
| ) | |
| overlay = ( | |
| f'<svg class="cv-overlay" viewBox="0 0 {self.width} {self.height}" preserveAspectRatio="none" ' | |
| f'xmlns="http://www.w3.org/2000/svg" aria-hidden="true">{"".join(self.overlays)}</svg>' | |
| ) | |
| heading = f'<h3 class="cv-title">{html.escape(self.title)}</h3>' if self.title else "" | |
| return ( | |
| f'<section id="{self.root}" class="cv-root">' | |
| f"<style>{_FRAGMENT_CSS.replace('#ROOT', '#' + self.root)}{self._toggle_css(all_toggle)}</style>" | |
| f'<form class="cv-form">{inputs}{heading}' | |
| '<div class="cv-layout">' | |
| f'<div class="cv-image-panel"><div class="cv-image-box" style="--cv-ar: {self.width / self.height:.4f}">' | |
| f'<img class="cv-image" src="{_image_data_url(self.image)}" alt="described image"/>{overlay}' | |
| f"</div>{controls}</div>" | |
| f'<div class="cv-tree">{tree}</div>' | |
| "</div></form></section>" | |
| ) | |
| def _toggle_css(self, all_toggle: str) -> str: | |
| root = f"#{self.root}" | |
| rules = [ | |
| f"{root} input#{all_toggle}:checked ~ .cv-layout .cv-overlay g.cv-ov {{ display: inline; }}", | |
| f'{root} input#{all_toggle}:checked ~ .cv-layout label[for="{all_toggle}"] .cv-off {{ display: none; }}', | |
| f'{root} input#{all_toggle}:checked ~ .cv-layout label[for="{all_toggle}"] .cv-on {{ display: inline; }}', | |
| ] | |
| for toggle, overlay_id, color in self.toggles: | |
| label = f'{root} input#{toggle}:checked ~ .cv-layout label[for="{toggle}"]' | |
| group = f'{root} input#{toggle}:checked ~ .cv-layout .cv-overlay g[data-id="{overlay_id}"]' | |
| rules += [ | |
| f"{group} {{ display: inline; }}", | |
| f"{group} .cv-name-label {{ display: inline; }}", | |
| f"{label} {{ border-color: {color}; box-shadow: inset 0 0 0 1px {color}; }}", | |
| f"{label} .cv-off {{ display: none; }}", | |
| f"{label} .cv-on {{ display: inline; }}", | |
| ] | |
| return "\n".join(rules) | |
| # -- sections -------------------------------------------------------------- | |
| def _general(self, general: GeneralDescription | None) -> str: | |
| if general is None: | |
| return "" | |
| data = general.model_dump(exclude_none=True) | |
| items = [] | |
| if general.color_palette: | |
| data.pop("color_palette", None) | |
| items.append(self._field("Color palette", self._chips(general.color_palette), tag="li")) | |
| for key, value in data.items(): | |
| items.append(self._field(key.replace("_", " ").capitalize(), _format_value(value), tag="li")) | |
| if not items: | |
| return "" | |
| return f'<section class="cv-section"><h3>General</h3><ul class="cv-general">{"".join(items)}</ul></section>' | |
| def _section(self, title: str, objects: list[BaseObject]) -> str: | |
| if not objects: | |
| return "" | |
| body = "".join(self._object(obj, depth=0) for obj in objects) | |
| return f'<section class="cv-section"><h3>{html.escape(title)} ({len(objects)})</h3>{body}</section>' | |
| # -- objects --------------------------------------------------------------- | |
| def _object(self, obj: BaseObject, depth: int) -> str: | |
| self.counter += 1 | |
| obj_id = f"{self.root}-o{self.counter}" | |
| color = self._object_color(obj) | |
| line_text = getattr(obj, "text", None) | |
| if line_text is None and getattr(obj, "lines", None): | |
| # A text object has no name; its lines say more than its description. | |
| line_text = " / ".join(line.text for line in obj.lines if line.text) | |
| if len(line_text) > 64: | |
| line_text = line_text[:61] + "..." | |
| name = obj.name or "" | |
| if not name and line_text: | |
| name = f"“{line_text}”" | |
| if not name and obj.description: | |
| snippet = obj.description.strip() | |
| name = snippet if len(snippet) <= 64 else snippet[:61] + "..." | |
| name = name or obj.kind.replace("_", " ") | |
| fields = [] | |
| if obj.description and name != obj.description.strip(): | |
| fields.append(self._field("Description", html.escape(obj.description))) | |
| if line_text is not None and obj.name: | |
| fields.append(self._field("Text", html.escape(line_text))) | |
| for label, key in (("Details", "details"), ("Artifacts", "artifacts")): | |
| if getattr(obj, key, None): | |
| fields.append(self._field(label, _format_value(getattr(obj, key)))) | |
| if obj.relationships: | |
| items = "".join( | |
| f"<li><strong>{html.escape(rel.relationship)}</strong> → {html.escape(rel.to)}</li>" | |
| for rel in obj.relationships | |
| ) | |
| fields.append(self._field("Relationships", f'<ul class="cv-list">{items}</ul>')) | |
| if obj.colors: | |
| fields.append(self._field("Colors", self._chips(obj.colors))) | |
| if isinstance(obj, Person) and obj.skin_color: | |
| fields.append(self._field("Skin color", self._chips([obj.skin_color]))) | |
| data = obj.model_dump(exclude_none=True) | |
| for key, value in data.items(): | |
| if key not in _SPECIAL_FIELDS: | |
| fields.append(self._field(key.replace("_", " ").capitalize(), _format_value(value))) | |
| fields += self._geometry_fields(obj, obj_id, color) | |
| nested = [] | |
| for key, title in _NESTED_SINGLE: | |
| item = getattr(obj, key, None) | |
| if item is not None: | |
| nested.append(self._subsection(title, self._object(item, depth + 1))) | |
| for key, title in _NESTED_LISTS: | |
| items = getattr(obj, key, None) or [] | |
| if items: | |
| nested.append(self._subsection(title, "".join(self._object(item, depth + 1) for item in items))) | |
| if isinstance(obj, Text) and obj.lines: | |
| nested.append(self._subsection("Lines", "".join(self._object(line, depth + 1) for line in obj.lines))) | |
| body = "".join(fields + nested) or '<p class="cv-empty">No other fields.</p>' | |
| open_attr = " open" if depth == 0 else "" | |
| return ( | |
| f'<details class="cv-object"{open_attr}>' | |
| '<summary class="cv-summary">' | |
| f'<span class="cv-dot" style="background:{color}"></span>' | |
| f'<span class="cv-kind">{html.escape(obj.kind.replace("_", " "))}</span>' | |
| f'<span class="cv-name">{html.escape(name)}</span>' | |
| "</summary>" | |
| f'<div class="cv-body">{body}</div>' | |
| "</details>" | |
| ) | |
| def _geometry_fields(self, obj: BaseObject, obj_id: str, color: str) -> list[str]: | |
| fields = [] | |
| boxes = obj.bounding_boxes or [] | |
| if boxes: | |
| shapes = "".join( | |
| self._box_shape(bbox, color, index + 1 if len(boxes) > 1 else None) for index, bbox in enumerate(boxes) | |
| ) | |
| items = "".join("<li>({}, {}) → ({}, {})</li>".format(*(_number(v) for v in _box(bbox))) for bbox in boxes) | |
| title = "Bounding boxes" if len(boxes) > 1 else "Bounding box" | |
| fields.append( | |
| self._overlay_field( | |
| title, f"{obj_id}-box", color, shapes, f'<ul class="cv-list cv-list--plain">{items}</ul>' | |
| ) | |
| ) | |
| if isinstance(obj, Person) and obj.face_bbox is not None: | |
| left, top, right, bottom = _box(obj.face_bbox) | |
| fields.append( | |
| self._overlay_field( | |
| "Face", | |
| f"{obj_id}-face", | |
| FACE_COLOR, | |
| self._box_shape(obj.face_bbox, FACE_COLOR, None), | |
| f"({_number(left)}, {_number(top)}) → ({_number(right)}, {_number(bottom)})", | |
| ) | |
| ) | |
| if isinstance(obj, Person) and obj.keypoints: | |
| shapes = "".join(self._keypoint_shape(point, color) for point in obj.keypoints) | |
| items = "".join( | |
| f"<li>{html.escape(point.name)} ({_number(_coord(point.point.x))}, " | |
| f"{_number(_coord(point.point.y))})</li>" | |
| for point in obj.keypoints | |
| ) | |
| fields.append( | |
| self._overlay_field( | |
| f"Keypoints ({len(obj.keypoints)})", | |
| f"{obj_id}-kp", | |
| color, | |
| shapes, | |
| f'<details><summary>list</summary><ul class="cv-list cv-list--plain">{items}</ul></details>', | |
| ) | |
| ) | |
| if obj.mask is not None: | |
| shape, summary = self._mask_shape(obj.mask, color) | |
| if shape: | |
| fields.append(self._overlay_field("Mask", f"{obj_id}-mask", color, shape, summary)) | |
| else: | |
| fields.append(self._field("Mask", summary)) | |
| return fields | |
| def _overlay_field(self, label: str, overlay_id: str, color: str, shapes: str, content: str) -> str: | |
| toggle = f"{overlay_id}-t" | |
| self.toggles.append((toggle, overlay_id, color)) | |
| self.overlays.append(f'<g class="cv-ov" data-id="{overlay_id}">{shapes}</g>') | |
| button = ( | |
| f'<label class="cv-button" for="{toggle}">' | |
| '<span class="cv-off">Show</span><span class="cv-on">Hide</span></label>' | |
| ) | |
| return ( | |
| '<div class="cv-field">' | |
| f'<span class="cv-label">{html.escape(label)}</span>{button}' | |
| f'<div class="cv-value">{content}</div>' | |
| "</div>" | |
| ) | |
| # -- overlay shapes (viewBox is the image in pixels) ----------------------- | |
| def _px(self, x: float, y: float) -> tuple[float, float]: | |
| return x / 100.0 * self.width, y / 100.0 * self.height | |
| def _box_shape(self, bbox: BoundingBox, color: str, index: int | None) -> str: | |
| left, top, right, bottom = _box(bbox) | |
| x0, y0 = self._px(left, top) | |
| x1, y1 = self._px(right, bottom) | |
| width, height = max(x1 - x0, 1.0), max(y1 - y0, 1.0) | |
| stroke = max(self.unit * 0.45, 1.5) | |
| # A white casing under the colored line keeps boxes visible on pixels of the same color. | |
| shape = ( | |
| f'<rect x="{x0:.1f}" y="{y0:.1f}" width="{width:.1f}" height="{height:.1f}" fill="none" ' | |
| f'stroke="#ffffff" stroke-opacity="0.9" stroke-width="{stroke * 2:.1f}"/>' | |
| f'<rect x="{x0:.1f}" y="{y0:.1f}" width="{width:.1f}" height="{height:.1f}" fill="{color}" ' | |
| f'fill-opacity="0.08" stroke="{color}" stroke-width="{stroke:.1f}"/>' | |
| ) | |
| if index is not None: | |
| shape += self._label(x0 + stroke, y0 + stroke, str(index)) | |
| return shape | |
| def _keypoint_shape(self, point: BodyKeypoint, color: str) -> str: | |
| x, y = self._px(_coord(point.point.x), _coord(point.point.y)) | |
| radius = max(self.unit * 0.7, 2.5) | |
| return ( | |
| f'<circle cx="{x:.1f}" cy="{y:.1f}" r="{radius:.1f}" fill="{color}" stroke="#ffffff" ' | |
| f'stroke-width="{radius * 0.45:.1f}"><title>{html.escape(point.name)}</title></circle>' | |
| '<g class="cv-name-label">' | |
| + self._label(x + radius * 1.6, y - radius * 1.6, point.name, small=True) | |
| + "</g>" | |
| ) | |
| def _label(self, x: float, y: float, text: str, small: bool = False) -> str: | |
| size = max(self.unit * (2.0 if small else 2.6), 9.0) | |
| return ( | |
| f'<text x="{x:.1f}" y="{y + size:.1f}" font-size="{size:.1f}" font-family="system-ui, sans-serif" ' | |
| f'font-weight="600" fill="#111418" stroke="#ffffff" stroke-width="{size * 0.25:.1f}" ' | |
| f'paint-order="stroke">{html.escape(text)}</text>' | |
| ) | |
| def _mask_shape(self, mask: ObjectMask, color: str) -> tuple[str, str]: | |
| rings = [] | |
| for ring in mask.polygons or []: | |
| values = [_coord(value) for value in ring] | |
| points = [self._px(x, y) for x, y in zip(values[0::2], values[1::2], strict=False)] | |
| if len(points) >= 3: | |
| rings.append("M" + " L".join(f"{x:.1f},{y:.1f}" for x, y in points) + " Z") | |
| if not rings: | |
| return "", "empty mask" | |
| vertices = sum(len(ring) // 2 for ring in mask.polygons or []) | |
| stroke = max(self.unit * 0.35, 1.2) | |
| path = " ".join(rings) | |
| shape = ( | |
| f'<path d="{path}" fill="none" stroke="#ffffff" stroke-opacity="0.9" stroke-width="{stroke * 2:.1f}"/>' | |
| f'<path d="{path}" fill="{color}" fill-opacity="0.35" fill-rule="evenodd" stroke="{color}" ' | |
| f'stroke-width="{stroke:.1f}"/>' | |
| ) | |
| return shape, f"polygon mask, {len(rings)} ring(s), {vertices} vertices" | |
| # -- small parts ----------------------------------------------------------- | |
| def _field(label: str, content: str, tag: str = "div") -> str: | |
| return ( | |
| f'<{tag} class="cv-field"><span class="cv-label">{html.escape(label)}</span>' | |
| f'<div class="cv-value">{content}</div></{tag}>' | |
| ) | |
| def _subsection(title: str, content: str) -> str: | |
| return f'<div class="cv-sub"><h4>{html.escape(title)}</h4>{content}</div>' | |
| def _chips(colors: list[Color]) -> str: | |
| chips = [] | |
| for color in colors: | |
| red, green, blue = _rgb(color) | |
| label = f"{html.escape(color.name)} · " if color.name else "" | |
| chips.append( | |
| f'<li class="cv-chip"><span class="cv-swatch" style="background:rgb({red},{green},{blue})"></span>' | |
| f"{label}rgb({red}, {green}, {blue})</li>" | |
| ) | |
| return f'<ul class="cv-chips">{"".join(chips)}</ul>' | |
| def _object_color(self, obj: BaseObject) -> str: | |
| primary = (obj.colors or [None])[0] | |
| if primary is None and isinstance(obj, Person): | |
| primary = obj.skin_color | |
| if primary is not None: | |
| red, green, blue = _rgb(primary) | |
| return f"rgb({red},{green},{blue})" | |
| return PALETTE[self.counter % len(PALETTE)] | |