"""Display-only projections. Raw model output is never rewritten.""" import base64 import html import io import json import re import bleach from bs4 import BeautifulSoup from markdown_it import MarkdownIt from PIL import ImageDraw, ImageFont from catalog import ROOT from presentation import TASK_VIEWS, JSON_TASKS from structured_json import strict_json_loads COLORS = ["#0d9488", "#e05a47", "#7659b4", "#b88712", "#2876ba"] RESULT_CSS = (ROOT / "assets/result.css").read_text().replace("__ASSETS__", str(ROOT / "assets")) CROP = re.compile(r"(?:page_(\d+)/)?images/bbox_(\d+)_(\d+)_(\d+)_(\d+)\.jpg") TAGS = {"p", "div", "span", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "b", "strong", "i", "em", "u", "s", "sub", "sup", "ul", "ol", "li", "blockquote", "pre", "code", "table", "thead", "tbody", "tfoot", "tr", "th", "td", "img"} def parse_json(raw): text = raw.strip() fence = re.fullmatch(r"```(?:json)?\s*\n(.*?)\n```", text, re.S | re.I) return strict_json_loads(fence[1] if fence else text) def output_structure_error(task, data): arrays = {"document_structured", "document_layout", "figure_captioning", "word_spotting", "reference_grounding", "crosspage_table"} expected = (dict, list) if task == "kie_schema" else (list,) if task in arrays else (dict,) if not isinstance(data, expected): name = "an object or array" if len(expected) == 2 else "an array" if list in expected else "an object" return f"Expected a JSON {name.removeprefix('an ')} for this task." if task in arrays and not all(isinstance(item, dict) for item in data): return "Expected a JSON array of objects for this task." return None def bbox(value): if isinstance(value, str): match = re.fullmatch(r"\s*[\[(]\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*[\])]\s*", value) if not match: raise ValueError("Invalid bounding box string.") value = list(map(int, match.groups())) if not isinstance(value, (list, tuple)) or len(value) != 4 or not all(type(v) is int for v in value): raise ValueError("Invalid bounding box.") x1, y1, x2, y2 = value if not (0 <= x1 < x2 <= 1000 and 0 <= y1 < y2 <= 1000): raise ValueError("Bounding box is out of range or empty.") return value def crop_image(image, value): x1, y1, x2, y2 = bbox(value) w, h = image.size return image.crop((round(x1*w/1000), round(y1*h/1000), round(x2*w/1000), round(y2*h/1000))) def image_uri(image): output = io.BytesIO() image.convert("RGB").save(output, format="JPEG", quality=88) return "data:image/jpeg;base64," + base64.b64encode(output.getvalue()).decode("ascii") def boxed_answer_rule(state, silent): if not state.src.startswith(r"\boxed", state.pos): return False opening = state.pos + len(r"\boxed") while opening < state.posMax and state.src[opening].isspace(): opening += 1 if opening >= state.posMax or state.src[opening] != "{": return False depth, end = 1, opening + 1 # Balance nested TeX groups without consuming escaped braces or code spans. while end < state.posMax: char = state.src[end] if char == "\\": end += 2 continue if char == "{": depth += 1 elif char == "}": depth -= 1 if depth == 0: if not silent: token = state.push("boxed_answer", "", 0) token.content = state.src[opening + 1:end] state.pos = end + 1 return True end += 1 return False def boxed_answer_html(tokens, index, options, env): content = tokens[index].content # Plain-language answers retain spaces and upright text; TeX remains TeX. if re.search(r"[\\{}_^]", content): content = r"\(" + content + r"\)" return '' + html.escape(content) + '' def markdown_html(text, *, boxed_answers=False): # Protect math before Markdown consumes backslashes and underscores. expressions = [] def protect(match): expressions.append(match[0]) return f"OVISMATHPLACEHOLDER{len(expressions)-1}END" text = re.sub(r"\$\$[\s\S]+?\$\$|\\\[[\s\S]+?\\\]|\\\([\s\S]+?\\\)|(? ''' + safe + "" document = document.replace("", "") if not math: document = re.sub(r"", "", document) return document def frame(content, images=(), math=None): document = frame_document(content, images, math) return '' def pending_output(task, raw="", state="Ready"): title = TASK_VIEWS[task][1] if raw: return frame('
' + html.escape(state) + '
' + html.escape(raw) + '
', math=False) label = "No result yet" if state == "Ready" else state return frame(f'

{html.escape(title)}

{html.escape(label)}

', math=False) def value_html(value): if isinstance(value, dict): rows = [] for key, item in value.items(): rows.append('' + html.escape(str(key)) + '' + value_html(item) + '') return '' + ''.join(rows) + '
' if isinstance(value, list): if not value: return '[]' if all(isinstance(item, dict) for item in value): keys = list(dict.fromkeys(key for item in value for key in item)) header = '' + ''.join('' + html.escape(str(key)) + '' for key in keys) + '' rows = ''.join('' + ''.join('' + (value_html(item[key]) if key in item else 'Not returned') + '' for key in keys) + '' for item in value) return '' + header + '' + rows + '
' return '
    ' + ''.join('
  1. ' + value_html(item) + '
  2. ' for item in value) + '
' text = json.dumps(value, ensure_ascii=False) if value is None or isinstance(value, bool) else str(value) return '' + html.escape(text) + '' def region_rows(data): records = data if isinstance(data, list) else [data] rows = [] for index, record in enumerate(records): if not isinstance(record, dict): rows.append('' + str(index + 1) + '' + value_html(record) + '') continue text = record.get("text", record.get("answer", record.get("category", record.get("reference", "")))) coordinates = html.escape(str(record.get("bbox", "Not returned"))) rows.append(f'{index+1:02d}{html.escape(str(text))}{coordinates}') return '' + ''.join(rows) + '
#Content / categoryCoordinates
' def annotated(image, data): result = image.copy() draw = ImageDraw.Draw(result) width = max(2, round(result.width / 500)) try: font = ImageFont.truetype("DejaVuSans.ttf", max(16, result.width // 70)) except OSError: font = ImageFont.load_default() records = data if isinstance(data, list) else [data] invalid = 0 for index, record in enumerate(records): if not isinstance(record, dict) or "bbox" not in record: continue try: x1, y1, x2, y2 = bbox(record["bbox"]) except ValueError: invalid += 1 continue color = COLORS[index % len(COLORS)] x1, x2 = x1*result.width/1000, x2*result.width/1000 y1, y2 = y1*result.height/1000, y2*result.height/1000 draw.rectangle((x1, y1, x2, y2), outline=color, width=width) label = str(index + 1) box = draw.textbbox((0, 0), label, font=font) tw, th = box[2]+8, box[3]+5 ly = max(0, y1-th) draw.rectangle((x1, ly, x1+tw, ly+th), fill=color) draw.text((x1+4, ly), label, fill="white", font=font) return result, invalid def render_output(task, raw, images): data, overlays, warnings = None, [], [] if task in JSON_TASKS: try: data = parse_json(raw) except (ValueError, TypeError) as exc: warnings.append(f"Invalid JSON: {exc}") else: structure_error = output_structure_error(task, data) if structure_error: warnings.append(structure_error) body = ('
Unexpected output structure' + html.escape(structure_error) + '
' + html.escape(raw) + '
') return frame(body, images), overlays, data, warnings if data is not None and task in {"document_structured", "document_layout", "figure_captioning", "word_spotting", "reference_grounding", "position_aware_vqa"}: image, invalid = annotated(images[0], data) overlays.append((image, "Detected regions")) if invalid: warnings.append(f"{invalid} invalid bounding box(es); retained in raw output.") meta = '
' + TASK_VIEWS[task][1] + '' + TASK_VIEWS[task][2] + '
' if data is not None and task in {"document_structured", "figure_captioning"} and isinstance(data, list): parts, context = [], [] for index, item in enumerate(data): if not isinstance(item, dict): context.append(value_html(item)) continue content = str(item.get("content", "")) crop = "" if "" in content and "bbox" in item: try: coordinates = bbox(item["bbox"]) ref = "images/bbox_" + "_".join(map(str, coordinates)) + ".jpg" crop = f'' except ValueError: pass is_caption = "" in content content = re.sub(r"([\s\S]*?)", lambda m: "\n```text\n" + m[1] + "\n```\n", content) content = content.replace("", "").replace("", "") if task == "figure_captioning" and is_caption: label = html.escape(str(item.get("category", "Figure"))) parts.append(f'
{index+1:02d} / {label}
{crop}{markdown_html(content)}
') elif task == "figure_captioning": context.append(markdown_html(content)) else: parts.append('
' + crop + markdown_html(content) + '
') body = meta + "\n".join(parts) if context: body += '
Page context
' + '\n'.join(context) + '
' elif data is not None and task == "crosspage_table" and isinstance(data, list): parts = [] for item in data: if not isinstance(item, dict): parts.append(value_html(item)) continue pages = item.get("source_pages", []) badges = ''.join('Page ' + html.escape(str(page)) + '' for page in pages) if isinstance(pages, list) else value_html(pages) parts.append('
' + badges + str(item.get("html", "")) + '
') body = meta + '\n'.join(parts) elif data is not None and task in {"kie_open", "kie_fields", "kie_schema"}: body = meta + value_html(data) elif data is not None and task == "chart_parsing" and isinstance(data, dict): title = html.escape(str(data.get("title", "Chart data"))) body = meta + '

' + title + '

' if "values" in data: body += value_html(data["values"]) # Show every returned field, including unrecognized chart schema fields. metadata = {key: value for key, value in data.items() if key not in {"values", "title"}} if metadata: body += '
' + value_html(metadata) + '
' elif data is not None and task == "position_aware_vqa" and isinstance(data, dict): body = meta + '
' + html.escape(str(data.get("answer", ""))) + '
' + region_rows(data) elif data is not None and task in {"document_layout", "word_spotting", "reference_grounding"}: body = meta + region_rows(data) if task == "document_layout" and overlays: images = [*images, overlays[0][0]] body = meta + f'' + region_rows(data) elif data is not None: body = meta + value_html(data) elif task in JSON_TASKS: body = '
Structured output unavailableInvalid JSON. Original model response:
' + html.escape(raw) + '
' elif task == "formula_recognition": expression = raw.strip() if not any(token in expression for token in ("$", r"\[", r"\(")): expression = r"\[" + expression + r"\]" body = meta + '
' + markdown_html(expression) + '
' elif task == "image_classification": body = meta + '' + html.escape(raw) + '' elif task == "visual_question_answering": body = meta + '
' + markdown_html(raw, boxed_answers=True) + '
' elif task == "document_markdown": body = markdown_html(raw) else: body = meta + '
' + html.escape(raw) + '
' return frame(body, images), overlays, data, warnings