| """ |
| visual_explainer.py — Caption/table understanding for Paper2Lab. |
| |
| This is caption-grounded visual understanding. It does not use image pixels yet. |
| For the hackathon MVP, this gives useful figure/table summaries without GPU cost. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import re |
| from typing import Any, Dict, List |
|
|
|
|
| _METRIC_WORDS = [ |
| "accuracy", "precision", "recall", "f1", "auc", "bleu", "rouge", "loss", "perplexity", |
| "score", "performance", "results", "comparison", "evaluation", "training cost", "p-value", |
| ] |
|
|
| _METHOD_WORDS = [ |
| "architecture", "pipeline", "framework", "workflow", "model", "method", "procedure", |
| "attention", "encoder", "decoder", "algorithm", "overview", |
| ] |
|
|
| _DATA_WORDS = [ |
| "dataset", "data", "samples", "patients", "images", "sentences", "articles", "studies", |
| "distribution", "statistics", "characteristics", |
| ] |
|
|
|
|
| def _clean(text: str) -> str: |
| text = text or "" |
| text = re.sub(r"\s+", " ", text) |
| return text.strip(" .;:\n\t") |
|
|
|
|
| def _label_type(label: str) -> str: |
| low = (label or "").lower() |
| if "table" in low or "tbl" in low: |
| return "table" |
| if "figure" in low or "fig" in low: |
| return "figure" |
| if "algorithm" in low: |
| return "algorithm" |
| if "scheme" in low: |
| return "scheme" |
| return "visual" |
|
|
|
|
| def _purpose(caption: str, visual_type: str) -> str: |
| low = caption.lower() |
| if any(w in low for w in _METHOD_WORDS): |
| return "method_or_architecture" |
| if any(w in low for w in _METRIC_WORDS): |
| return "results_or_evaluation" |
| if any(w in low for w in _DATA_WORDS): |
| return "data_or_dataset_description" |
| if visual_type == "table": |
| return "structured_results_or_metadata" |
| return "illustrative_figure" |
|
|
|
|
| def _summary_from_caption(label: str, caption: str, visual_type: str) -> str: |
| caption = _clean(caption) |
| if not caption: |
| return f"{label} is a {visual_type}, but no caption text was extracted." |
| |
| if len(caption) <= 220: |
| return caption |
| first_sentence = re.split(r"(?<=[.!?])\s+", caption)[0] |
| return _clean(first_sentence[:260]) |
|
|
|
|
| def _summarize_table_data(table: Dict[str, Any]) -> str | None: |
| data = table.get("data") |
| if not isinstance(data, list) or not data: |
| return None |
| rows = [r for r in data if isinstance(r, list)] |
| if not rows: |
| return None |
| n_rows = len(rows) |
| n_cols = max((len(r) for r in rows), default=0) |
| header = rows[0] if rows else [] |
| header_text = ", ".join(str(x).strip() for x in header if str(x).strip())[:180] |
| if header_text: |
| return f"Extracted table with approximately {n_rows} rows and {n_cols} columns. Header fields include: {header_text}." |
| return f"Extracted table with approximately {n_rows} rows and {n_cols} columns." |
|
|
|
|
| def explain_figures_and_tables(extracted: Dict[str, Any]) -> List[Dict[str, Any]]: |
| """Return concise explanations for extracted captions and tables.""" |
| outputs: List[Dict[str, Any]] = [] |
|
|
| for cap in extracted.get("captions", []) or []: |
| label = cap.get("label", "") |
| caption = _clean(cap.get("caption", "")) |
| visual_type = _label_type(label) |
| outputs.append({ |
| "label": label, |
| "type": visual_type, |
| "purpose": _purpose(caption, visual_type), |
| "summary": _summary_from_caption(label, caption, visual_type), |
| "evidence": caption, |
| "page_number": cap.get("page_number"), |
| }) |
|
|
| |
| existing_table_pages = {(o.get("page_number"), o.get("label")) for o in outputs if o.get("type") == "table"} |
| for table in extracted.get("tables", []) or []: |
| page = table.get("page_number") |
| caption = _clean(table.get("caption") or "") |
| label = f"Table extracted on page {page}" if page is not None else "Extracted table" |
| if (page, label) in existing_table_pages: |
| continue |
| data_summary = _summarize_table_data(table) |
| outputs.append({ |
| "label": label, |
| "type": "table", |
| "purpose": _purpose(caption or data_summary or "", "table"), |
| "summary": caption or data_summary or "A table was detected, but its content could not be summarized reliably.", |
| "evidence": caption or data_summary or "", |
| "page_number": page, |
| }) |
|
|
| return outputs[:20] |
|
|
